From 6445044ac645ed40c9a9d1b07dc26c7d83f943c4 Mon Sep 17 00:00:00 2001 From: Ricardo Date: Tue, 9 Jun 2026 22:55:58 +0100 Subject: [PATCH] Release v4928.1.4.2 stable --- .env.example | 91 + .gitignore | 7 + CLEANUP_SUMMARY.md | 28 + README.md | 144 + RELEASE_NOTES_v4926_4.md | 46 + RELEASE_NOTES_v4926_5.md | 55 + RELEASE_v4926_3.md | 35 + VERSION_NOTES_v4926_6_3.md | 24 + app/__init__.py | 0 app/action_catalog.py | 115 + app/action_decider.py | 94 + app/action_llm_client.py | 243 ++ app/action_mapper.py | 36 + app/action_prompt.py | 136 + app/admin_dashboard.py | 1784 +++++++++ app/admin_queries.py | 55 + app/admin_ui/__init__.py | 5 + app/admin_ui/components.py | 116 + app/admin_ui/guidance.py | 353 ++ app/admin_ui/htmx.py | 35 + app/admin_ui/labels.py | 109 + app/admin_ui/layout.py | 59 + app/admin_ui/navigation.py | 103 + app/admin_ui/pages/README.md | 15 + app/admin_ui/pages/__init__.py | 1 + app/admin_ui/pages/communications.py | 201 + app/admin_ui/pages/conversations.py | 24 + app/admin_ui/pages/customers.py | 199 + app/admin_ui/pages/dashboard.py | 102 + app/admin_ui/pages/events.py | 46 + app/admin_ui/pages/finance.py | 51 + app/admin_ui/pages/integrations.py | 198 + app/admin_ui/pages/operations.py | 185 + app/admin_ui/pages/opportunities.py | 1282 +++++++ app/admin_ui/pages/orders.py | 163 + app/admin_ui/pages/outbox.py | 242 ++ app/admin_ui/pages/products.py | 301 ++ app/admin_ui/pages/queues.py | 83 + app/admin_ui/pages/reconciliation.py | 746 ++++ app/admin_ui/pages/runs.py | 57 + app/admin_ui/pages/system.py | 300 ++ app/admin_ui/pages/tasks.py | 769 ++++ app/admin_ui/router.py | 52 + app/admin_ui/styles.py | 2439 ++++++++++++ app/admin_ui/view_models/__init__.py | 1 + app/admin_ui/view_models/operations.py | 235 ++ app/analyzer.py | 90 + app/api/__init__.py | 0 app/api/internal.py | 100 + app/chatwoot_client.py | 94 + app/commercial/__init__.py | 0 app/commercial_service.py | 885 +++++ app/communication_service.py | 322 ++ app/config.py | 107 + app/customers/__init__.py | 0 app/db.py | 324 ++ app/email_identity_cleanup_service.py | 117 + app/email_identity_extraction_service.py | 798 ++++ app/external_reconciliation_sync.py | 994 +++++ app/fiscal_enrichment_service.py | 1267 +++++++ app/integration_outbox_service.py | 592 +++ app/integrations/__init__.py | 0 app/jasmin_backfill_service.py | 733 ++++ app/jasmin_client.py | 234 ++ app/jasmin_service.py | 638 ++++ app/logistics/__init__.py | 0 app/main.py | 42 + app/mautic_client.py | 92 + app/message_cleaner.py | 65 + app/models.py | 8 + app/odoo_client.py | 126 + app/odoo_service.py | 669 ++++ app/operation_noise.py | 113 + app/operation_service.py | 149 + app/operations_service.py | 481 +++ app/operator_audit_service.py | 108 + app/opportunities/__init__.py | 0 app/opportunity_next_action_service.py | 194 + app/opportunity_service.py | 1137 ++++++ app/outbox/__init__.py | 0 app/packlink_client.py | 145 + app/packlink_service.py | 383 ++ app/persistence.py | 290 ++ app/posting_policy.py | 23 + app/preparation_service.py | 29 + app/preparation_view_model.py | 90 + app/product_service.py | 533 +++ app/prompt.py | 82 + app/reconciliation_decision_service.py | 181 + app/reconciliation_service.py | 3254 +++++++++++++++++ app/schemas.py | 60 + app/task_service.py | 1679 +++++++++ app/webhooks_chatwoot.py | 459 +++ app/workflow_guard.py | 299 ++ .../clientflow-fiscal-enrichment.service | 10 + .../clientflow-fiscal-enrichment.timer | 11 + .../systemd/clientflow-outbox-jasmin.service | 13 + deploy/systemd/clientflow-outbox-jasmin.timer | 11 + .../clientflow-outbox-packlink.service | 13 + .../systemd/clientflow-outbox-packlink.timer | 11 + docs/CLIENTFLOW_V31_NOTES.md | 109 + docs/CLIENTFLOW_V32_NOTES.md | 78 + docs/CLIENTFLOW_V33_NOTES.md | 120 + docs/CLIENTFLOW_V34_NOTES.md | 60 + docs/CLIENTFLOW_V3_CUSTOMERS_DOCUMENTS.md | 82 + docs/CLIENTFLOW_V40_NOTES.md | 72 + docs/CLIENTFLOW_V41_NOTES.md | 34 + docs/CLIENTFLOW_V42_NOTES.md | 59 + docs/CLIENTFLOW_V42_REVIEW_UI_NOTES.md | 42 + docs/CLIENTFLOW_V451_UI_ALIGNMENT.md | 89 + docs/CLIENTFLOW_V452_UI_ICONS_FIX.md | 19 + docs/CLIENTFLOW_V453_UI_POLISH.md | 49 + docs/CLIENTFLOW_V45_OPERATIONAL_CORE.md | 110 + docs/CLIENTFLOW_V45_UPDATE_GUIDE.md | 83 + ...IENTFLOW_V461_CUSTOMER_MISMATCH_WARNING.md | 22 + ..._V462_DISABLE_CUSTOMER_MISMATCH_WARNING.md | 32 + ...LIENTFLOW_V462_WORKFLOW_ACTIONS_CLEANUP.md | 43 + ...LIENTFLOW_V463_OPERATIONS_QUEUE_CLEANUP.md | 27 + docs/CLIENTFLOW_V465_OPERATIONS_WORK_QUEUE.md | 44 + ...LIENTFLOW_V466_SAFE_OPPORTUNITY_LINKING.md | 16 + ...W_V467_CHATWOOT_AUTOCOMPLETE_GUARDRAILS.md | 26 + ...W_V468_OPERATIONAL_SAFETY_CONSOLIDATION.md | 59 + ...ENTFLOW_V46_CHATWOOT_WORKFLOW_HARDENING.md | 99 + ...LIENTFLOW_V471_ADMIN_SIDEBAR_SCROLL_FIX.md | 30 + docs/CLIENTFLOW_V472_DOMAIN_ROUTES.md | 47 + docs/CLIENTFLOW_V473_HTMX_PARTIALS.md | 74 + docs/CLIENTFLOW_V474_HTMX_COMPLETION.md | 101 + docs/CLIENTFLOW_V47_ADMIN_UI_REFACTOR.md | 95 + docs/CLIENTFLOW_V4801_ADMIN_MENU_COLLAPSED.md | 43 + ...0_OPERATIONAL_AUTOMATION_AUDIT_RECOVERY.md | 109 + ...LIENTFLOW_V482_PRODUCTION_STABILIZATION.md | 76 + docs/CLIENTFLOW_V483_OPERATIONS_GUARDRAILS.md | 85 + ...CLIENTFLOW_V484_FISCAL_LINK_CONSISTENCY.md | 30 + docs/CLIENTFLOW_V485_BOUNCE_IGNORE_CARD_UX.md | 74 + ...W_V486_OPPORTUNITY_BOARD_LAYOUT_CLEANUP.md | 35 + ...NTFLOW_V487_OPERATIONS_WORKBENCH_LAYOUT.md | 28 + ...LIENTFLOW_V488_OPERATIONS_CLASSIC_CARDS.md | 39 + .../CLIENTFLOW_V489_OPERATIONS_CARD_POLISH.md | 35 + ...LIENTFLOW_V490_OPERATIONS_NOISE_CLEANUP.md | 48 + ...OW_V4912_ODOO_FULFILMENT_RECONCILIATION.md | 46 + ...TFLOW_V4913_APPLY_RECONSTRUCTED_PROCESS.md | 77 + ...17_RECONCILIATION_CROSS_SOURCE_GROUPING.md | 38 + ...ENTFLOW_V491_OPERATIONAL_RECONCILIATION.md | 124 + ...LOW_V4920_MULTI_PURCHASE_RECONCILIATION.md | 81 + docs/CLIENTFLOW_V4921_RESET_RECONCILIATION.md | 52 + ...ENTFLOW_V4922_REBUILD_RECONCILIATION_UI.md | 38 + ...LOW_V4924_RECONCILIATION_PROCESS_REVIEW.md | 32 + ...IENTFLOW_V4925_FISCAL_ENRICHMENT_WORKER.md | 98 + docs/CLIENTFLOW_V4926_6_NOTES.md | 21 + docs/CLIENTFLOW_V4926_7_1_NOTES.md | 30 + docs/CLIENTFLOW_V4926_7_NOTES.md | 17 + docs/CLIENTFLOW_V4926_8_1_NOTES.md | 40 + docs/CLIENTFLOW_V4926_8_2_NOTES.md | 52 + docs/CLIENTFLOW_V4926_8_3_NOTES.md | 44 + docs/CLIENTFLOW_V4926_8_4_NOTES.md | 25 + docs/CLIENTFLOW_V4926_8_NOTES.md | 83 + docs/CLIENTFLOW_V4927_1_HOTFIX.md | 21 + docs/CLIENTFLOW_V4927_2_HOTFIX.md | 34 + docs/CLIENTFLOW_V4927_ROADMAP_STEP1.md | 30 + docs/CLIENTFLOW_V4928_1_1_SYNTAX_HOTFIX.md | 25 + docs/CLIENTFLOW_V4928_1_2_UI_COHERENCE.md | 28 + docs/CLIENTFLOW_V4928_1_3_UI_FREEZE_FIXES.md | 30 + docs/CLIENTFLOW_V4928_1_LEGACY_COHERENCE.md | 33 + docs/CLIENTFLOW_V4928_DECISIONAL_UI.md | 101 + docs/CLIENTFLOW_V492_EXTERNAL_API_SYNC.md | 130 + docs/CLIENTFLOW_V493_JASMIN_RECENT_SYNC.md | 39 + ...TFLOW_V494_RECONCILIATION_RECENT_WINDOW.md | 45 + ...95_RECONCILIATION_OPERATION_SUGGESTIONS.md | 42 + ...LOW_V496_RECONCILIATION_NIF_SUGGESTIONS.md | 19 + ...ENTFLOW_V497_RECONCILIATION_PAGE_HOTFIX.md | 31 + ...8_RECONCILIATION_MATCHING_BOARD_CLEANUP.md | 35 + ...OW_V499_PROCESS_TIMELINE_RECONSTRUCTION.md | 33 + docs/JASMIN_CLIENTFLOW.md | 135 + docs/PACKLINK_CLIENTFLOW.md | 118 + docs/RENOMEAR_CCE_PARA_CLIENTFLOW.md | 21 + docs/env.odoo.example | 8 + migrations/001_core_indexes.sql | 17 + ..._customers_documents_shipments_indexes.sql | 25 + migrations/003_product_external_codes.sql | 12 + migrations/004_outbox_safety.sql | 12 + migrations/005_operations_health_indexes.sql | 19 + migrations/006_v45_operational_core.sql | 73 + requirements.txt | 29 + scripts/apply_migrations.py | 86 + .../backfill_chatwoot_inbox_to_clientflow.py | 338 ++ .../backfill_jasmin_opportunity_details.py | 312 ++ .../backfill_opportunity_product_mappings.py | 89 + scripts/batch_validate_email_identity.py | 347 ++ scripts/batch_validate_email_identity_safe.py | 347 ++ scripts/check_clientflow_health.py | 82 + ...anup_invalid_email_identity_suggestions.py | 131 + .../cleanup_non_commercial_opportunities.py | 105 + scripts/cleanup_operations_noise.py | 158 + scripts/cleanup_stale_reconciliation_items.py | 79 + scripts/create_automatic_followups.py | 15 + scripts/enrich_fiscal_customers.py | 42 + scripts/evaluate_action_decider.py | 217 ++ scripts/extract_email_identity.py | 43 + scripts/ignore_manually_cleaned_outbox.py | 60 + scripts/inspect_customer_tax_id_conflict.py | 33 + ...spect_jasmin_candidates_for_opportunity.py | 42 + scripts/install_systemd_timers.sh | 38 + scripts/prepare_task.py | 512 +++ scripts/process_outbox.py | 318 ++ scripts/recover_stale_outbox.py | 41 + scripts/reopen_chatwoot_review_tasks.py | 71 + .../repair_odoo_sale_order_customer_names.py | 89 + ...replace_jasmin_document_for_opportunity.py | 44 + scripts/reset_reconciliation_generated.py | 133 + .../review_domain_auto_applied_suggestions.py | 89 + scripts/run_dev.sh | 4 + scripts/run_reconciliation_pipeline.py | 48 + scripts/sync_external_reconciliation.py | 77 + scripts/sync_reconciliation_candidates.py | 25 + scripts/test_action_core_http.py | 95 + scripts/test_jasmin_connection.py | 51 + scripts/test_operational_flow.py | 15 + scripts/test_packlink_connection.py | 101 + scripts/test_sample.sh | 5 + scripts/validate_v45_operational_core.py | 48 + test_clientflow_e2e.sh | 407 +++ test_odoo_integration.sh | 66 + test_odoo_order_status.sh | 104 + tests/sample_cases/sample_analyze.json | 13 + tests/test_clean_architecture_static.py | 38 + tests/test_v40_operational_files.py | 20 + tests/test_v42_review_fixes.py | 40 + tests/test_v451_ui_alignment.py | 46 + tests/test_v45_operational_core.py | 49 + tests/test_v461_customer_mismatch_warning.py | 22 + ..._v462_disable_customer_mismatch_warning.py | 27 + tests/test_v462_mark_no_interest.py | 36 + tests/test_v463_operations_queue_cleanup.py | 43 + tests/test_v465_operations_work_queue.py | 38 + tests/test_v466_safe_opportunity_linking.py | 47 + ...t_v467_chatwoot_autocomplete_guardrails.py | 40 + ...t_v468_operational_safety_consolidation.py | 60 + tests/test_v46_chatwoot_workflow.py | 53 + tests/test_v472_domain_routes.py | 72 + tests/test_v473_htmx_partials.py | 54 + tests/test_v474_htmx_completion.py | 51 + tests/test_v47_admin_ui_refactor.py | 63 + tests/test_v4801_admin_menu_collapsed.py | 13 + ...0_operational_automation_audit_recovery.py | 68 + tests/test_v482_production_stabilization.py | 58 + tests/test_v483_operations_guardrails.py | 63 + tests/test_v484_fiscal_link_consistency.py | 44 + tests/test_v485_bounce_ignore_card_ux.py | 50 + ...t_v486_opportunity_board_layout_cleanup.py | 36 + tests/test_v488_operations_classic_cards.py | 52 + tests/test_v489_operations_card_polish.py | 36 + tests/test_v490_operations_noise_cleanup.py | 50 + ...st_v4911_odoo_name_match_reconciliation.py | 89 + ...st_v4912_odoo_fulfilment_reconciliation.py | 129 + .../test_v4913_apply_reconstructed_process.py | 109 + .../test_v4914_reconciliation_link_hotfix.py | 19 + .../test_v4915_reconciliation_amount_cast.py | 8 + tests/test_v491_operational_reconciliation.py | 75 + ...st_v4920_reconciliation_multiple_orders.py | 104 + ...v4920_reconciliation_multiple_purchases.py | 112 + ...est_v4922_reconciliation_rebuild_button.py | 29 + ..._reconciliation_customer_first_pipeline.py | 49 + ...est_v4924_reconciliation_process_review.py | 96 + ...test_v4925_2_odoo_customer_field_compat.py | 24 + ...st_v4925_3_odoo_customer_identity_guard.py | 26 + tests/test_v4925_fiscal_enrichment_worker.py | 57 + tests/test_v4926_1_email_identity_quality.py | 124 + .../test_v4926_5_manual_opportunity_static.py | 31 + ...4926_6_2_jasmin_money_extraction_static.py | 39 + ...st_v4926_6_jasmin_reconciliation_static.py | 31 + ...st_v4926_7_identity_and_customer_guards.py | 20 + tests/test_v4928_decisional_ui_static.py | 33 + tests/test_v492_external_api_sync.py | 63 + tests/test_v493_jasmin_recent_sync.py | 38 + .../test_v494_reconciliation_recent_window.py | 27 + ...95_reconciliation_operation_suggestions.py | 35 + ...est_v496_reconciliation_nif_suggestions.py | 42 + ...8_reconciliation_matching_board_cleanup.py | 44 + ...st_v499_process_timeline_reconstruction.py | 39 + ...rocess_timeline_reconstruction_behavior.py | 308 ++ 280 files changed, 41775 insertions(+) create mode 100644 .env.example create mode 100644 .gitignore create mode 100644 CLEANUP_SUMMARY.md create mode 100644 README.md create mode 100644 RELEASE_NOTES_v4926_4.md create mode 100644 RELEASE_NOTES_v4926_5.md create mode 100644 RELEASE_v4926_3.md create mode 100644 VERSION_NOTES_v4926_6_3.md create mode 100644 app/__init__.py create mode 100644 app/action_catalog.py create mode 100644 app/action_decider.py create mode 100644 app/action_llm_client.py create mode 100644 app/action_mapper.py create mode 100644 app/action_prompt.py create mode 100644 app/admin_dashboard.py create mode 100644 app/admin_queries.py create mode 100644 app/admin_ui/__init__.py create mode 100644 app/admin_ui/components.py create mode 100644 app/admin_ui/guidance.py create mode 100644 app/admin_ui/htmx.py create mode 100644 app/admin_ui/labels.py create mode 100644 app/admin_ui/layout.py create mode 100644 app/admin_ui/navigation.py create mode 100644 app/admin_ui/pages/README.md create mode 100644 app/admin_ui/pages/__init__.py create mode 100644 app/admin_ui/pages/communications.py create mode 100644 app/admin_ui/pages/conversations.py create mode 100644 app/admin_ui/pages/customers.py create mode 100644 app/admin_ui/pages/dashboard.py create mode 100644 app/admin_ui/pages/events.py create mode 100644 app/admin_ui/pages/finance.py create mode 100644 app/admin_ui/pages/integrations.py create mode 100644 app/admin_ui/pages/operations.py create mode 100644 app/admin_ui/pages/opportunities.py create mode 100644 app/admin_ui/pages/orders.py create mode 100644 app/admin_ui/pages/outbox.py create mode 100644 app/admin_ui/pages/products.py create mode 100644 app/admin_ui/pages/queues.py create mode 100644 app/admin_ui/pages/reconciliation.py create mode 100644 app/admin_ui/pages/runs.py create mode 100644 app/admin_ui/pages/system.py create mode 100644 app/admin_ui/pages/tasks.py create mode 100644 app/admin_ui/router.py create mode 100644 app/admin_ui/styles.py create mode 100644 app/admin_ui/view_models/__init__.py create mode 100644 app/admin_ui/view_models/operations.py create mode 100644 app/analyzer.py create mode 100644 app/api/__init__.py create mode 100644 app/api/internal.py create mode 100644 app/chatwoot_client.py create mode 100644 app/commercial/__init__.py create mode 100644 app/commercial_service.py create mode 100644 app/communication_service.py create mode 100644 app/config.py create mode 100644 app/customers/__init__.py create mode 100644 app/db.py create mode 100644 app/email_identity_cleanup_service.py create mode 100644 app/email_identity_extraction_service.py create mode 100644 app/external_reconciliation_sync.py create mode 100644 app/fiscal_enrichment_service.py create mode 100644 app/integration_outbox_service.py create mode 100644 app/integrations/__init__.py create mode 100644 app/jasmin_backfill_service.py create mode 100644 app/jasmin_client.py create mode 100644 app/jasmin_service.py create mode 100644 app/logistics/__init__.py create mode 100644 app/main.py create mode 100644 app/mautic_client.py create mode 100644 app/message_cleaner.py create mode 100644 app/models.py create mode 100644 app/odoo_client.py create mode 100644 app/odoo_service.py create mode 100644 app/operation_noise.py create mode 100644 app/operation_service.py create mode 100644 app/operations_service.py create mode 100644 app/operator_audit_service.py create mode 100644 app/opportunities/__init__.py create mode 100644 app/opportunity_next_action_service.py create mode 100644 app/opportunity_service.py create mode 100644 app/outbox/__init__.py create mode 100644 app/packlink_client.py create mode 100644 app/packlink_service.py create mode 100644 app/persistence.py create mode 100644 app/posting_policy.py create mode 100644 app/preparation_service.py create mode 100644 app/preparation_view_model.py create mode 100644 app/product_service.py create mode 100644 app/prompt.py create mode 100644 app/reconciliation_decision_service.py create mode 100644 app/reconciliation_service.py create mode 100644 app/schemas.py create mode 100644 app/task_service.py create mode 100644 app/webhooks_chatwoot.py create mode 100644 app/workflow_guard.py create mode 100644 deploy/systemd/clientflow-fiscal-enrichment.service create mode 100644 deploy/systemd/clientflow-fiscal-enrichment.timer create mode 100644 deploy/systemd/clientflow-outbox-jasmin.service create mode 100644 deploy/systemd/clientflow-outbox-jasmin.timer create mode 100644 deploy/systemd/clientflow-outbox-packlink.service create mode 100644 deploy/systemd/clientflow-outbox-packlink.timer create mode 100644 docs/CLIENTFLOW_V31_NOTES.md create mode 100644 docs/CLIENTFLOW_V32_NOTES.md create mode 100644 docs/CLIENTFLOW_V33_NOTES.md create mode 100644 docs/CLIENTFLOW_V34_NOTES.md create mode 100644 docs/CLIENTFLOW_V3_CUSTOMERS_DOCUMENTS.md create mode 100644 docs/CLIENTFLOW_V40_NOTES.md create mode 100644 docs/CLIENTFLOW_V41_NOTES.md create mode 100644 docs/CLIENTFLOW_V42_NOTES.md create mode 100644 docs/CLIENTFLOW_V42_REVIEW_UI_NOTES.md create mode 100644 docs/CLIENTFLOW_V451_UI_ALIGNMENT.md create mode 100644 docs/CLIENTFLOW_V452_UI_ICONS_FIX.md create mode 100644 docs/CLIENTFLOW_V453_UI_POLISH.md create mode 100644 docs/CLIENTFLOW_V45_OPERATIONAL_CORE.md create mode 100644 docs/CLIENTFLOW_V45_UPDATE_GUIDE.md create mode 100644 docs/CLIENTFLOW_V461_CUSTOMER_MISMATCH_WARNING.md create mode 100644 docs/CLIENTFLOW_V462_DISABLE_CUSTOMER_MISMATCH_WARNING.md create mode 100644 docs/CLIENTFLOW_V462_WORKFLOW_ACTIONS_CLEANUP.md create mode 100644 docs/CLIENTFLOW_V463_OPERATIONS_QUEUE_CLEANUP.md create mode 100644 docs/CLIENTFLOW_V465_OPERATIONS_WORK_QUEUE.md create mode 100644 docs/CLIENTFLOW_V466_SAFE_OPPORTUNITY_LINKING.md create mode 100644 docs/CLIENTFLOW_V467_CHATWOOT_AUTOCOMPLETE_GUARDRAILS.md create mode 100644 docs/CLIENTFLOW_V468_OPERATIONAL_SAFETY_CONSOLIDATION.md create mode 100644 docs/CLIENTFLOW_V46_CHATWOOT_WORKFLOW_HARDENING.md create mode 100644 docs/CLIENTFLOW_V471_ADMIN_SIDEBAR_SCROLL_FIX.md create mode 100644 docs/CLIENTFLOW_V472_DOMAIN_ROUTES.md create mode 100644 docs/CLIENTFLOW_V473_HTMX_PARTIALS.md create mode 100644 docs/CLIENTFLOW_V474_HTMX_COMPLETION.md create mode 100644 docs/CLIENTFLOW_V47_ADMIN_UI_REFACTOR.md create mode 100644 docs/CLIENTFLOW_V4801_ADMIN_MENU_COLLAPSED.md create mode 100644 docs/CLIENTFLOW_V480_OPERATIONAL_AUTOMATION_AUDIT_RECOVERY.md create mode 100644 docs/CLIENTFLOW_V482_PRODUCTION_STABILIZATION.md create mode 100644 docs/CLIENTFLOW_V483_OPERATIONS_GUARDRAILS.md create mode 100644 docs/CLIENTFLOW_V484_FISCAL_LINK_CONSISTENCY.md create mode 100644 docs/CLIENTFLOW_V485_BOUNCE_IGNORE_CARD_UX.md create mode 100644 docs/CLIENTFLOW_V486_OPPORTUNITY_BOARD_LAYOUT_CLEANUP.md create mode 100644 docs/CLIENTFLOW_V487_OPERATIONS_WORKBENCH_LAYOUT.md create mode 100644 docs/CLIENTFLOW_V488_OPERATIONS_CLASSIC_CARDS.md create mode 100644 docs/CLIENTFLOW_V489_OPERATIONS_CARD_POLISH.md create mode 100644 docs/CLIENTFLOW_V490_OPERATIONS_NOISE_CLEANUP.md create mode 100644 docs/CLIENTFLOW_V4912_ODOO_FULFILMENT_RECONCILIATION.md create mode 100644 docs/CLIENTFLOW_V4913_APPLY_RECONSTRUCTED_PROCESS.md create mode 100644 docs/CLIENTFLOW_V4917_RECONCILIATION_CROSS_SOURCE_GROUPING.md create mode 100644 docs/CLIENTFLOW_V491_OPERATIONAL_RECONCILIATION.md create mode 100644 docs/CLIENTFLOW_V4920_MULTI_PURCHASE_RECONCILIATION.md create mode 100644 docs/CLIENTFLOW_V4921_RESET_RECONCILIATION.md create mode 100644 docs/CLIENTFLOW_V4922_REBUILD_RECONCILIATION_UI.md create mode 100644 docs/CLIENTFLOW_V4924_RECONCILIATION_PROCESS_REVIEW.md create mode 100644 docs/CLIENTFLOW_V4925_FISCAL_ENRICHMENT_WORKER.md create mode 100644 docs/CLIENTFLOW_V4926_6_NOTES.md create mode 100644 docs/CLIENTFLOW_V4926_7_1_NOTES.md create mode 100644 docs/CLIENTFLOW_V4926_7_NOTES.md create mode 100644 docs/CLIENTFLOW_V4926_8_1_NOTES.md create mode 100644 docs/CLIENTFLOW_V4926_8_2_NOTES.md create mode 100644 docs/CLIENTFLOW_V4926_8_3_NOTES.md create mode 100644 docs/CLIENTFLOW_V4926_8_4_NOTES.md create mode 100644 docs/CLIENTFLOW_V4926_8_NOTES.md create mode 100644 docs/CLIENTFLOW_V4927_1_HOTFIX.md create mode 100644 docs/CLIENTFLOW_V4927_2_HOTFIX.md create mode 100644 docs/CLIENTFLOW_V4927_ROADMAP_STEP1.md create mode 100644 docs/CLIENTFLOW_V4928_1_1_SYNTAX_HOTFIX.md create mode 100644 docs/CLIENTFLOW_V4928_1_2_UI_COHERENCE.md create mode 100644 docs/CLIENTFLOW_V4928_1_3_UI_FREEZE_FIXES.md create mode 100644 docs/CLIENTFLOW_V4928_1_LEGACY_COHERENCE.md create mode 100644 docs/CLIENTFLOW_V4928_DECISIONAL_UI.md create mode 100644 docs/CLIENTFLOW_V492_EXTERNAL_API_SYNC.md create mode 100644 docs/CLIENTFLOW_V493_JASMIN_RECENT_SYNC.md create mode 100644 docs/CLIENTFLOW_V494_RECONCILIATION_RECENT_WINDOW.md create mode 100644 docs/CLIENTFLOW_V495_RECONCILIATION_OPERATION_SUGGESTIONS.md create mode 100644 docs/CLIENTFLOW_V496_RECONCILIATION_NIF_SUGGESTIONS.md create mode 100644 docs/CLIENTFLOW_V497_RECONCILIATION_PAGE_HOTFIX.md create mode 100644 docs/CLIENTFLOW_V498_RECONCILIATION_MATCHING_BOARD_CLEANUP.md create mode 100644 docs/CLIENTFLOW_V499_PROCESS_TIMELINE_RECONSTRUCTION.md create mode 100644 docs/JASMIN_CLIENTFLOW.md create mode 100644 docs/PACKLINK_CLIENTFLOW.md create mode 100644 docs/RENOMEAR_CCE_PARA_CLIENTFLOW.md create mode 100644 docs/env.odoo.example create mode 100644 migrations/001_core_indexes.sql create mode 100644 migrations/002_customers_documents_shipments_indexes.sql create mode 100644 migrations/003_product_external_codes.sql create mode 100644 migrations/004_outbox_safety.sql create mode 100644 migrations/005_operations_health_indexes.sql create mode 100644 migrations/006_v45_operational_core.sql create mode 100644 requirements.txt create mode 100755 scripts/apply_migrations.py create mode 100755 scripts/backfill_chatwoot_inbox_to_clientflow.py create mode 100755 scripts/backfill_jasmin_opportunity_details.py create mode 100755 scripts/backfill_opportunity_product_mappings.py create mode 100755 scripts/batch_validate_email_identity.py create mode 100755 scripts/batch_validate_email_identity_safe.py create mode 100755 scripts/check_clientflow_health.py create mode 100755 scripts/cleanup_invalid_email_identity_suggestions.py create mode 100755 scripts/cleanup_non_commercial_opportunities.py create mode 100755 scripts/cleanup_operations_noise.py create mode 100755 scripts/cleanup_stale_reconciliation_items.py create mode 100755 scripts/create_automatic_followups.py create mode 100755 scripts/enrich_fiscal_customers.py create mode 100755 scripts/evaluate_action_decider.py create mode 100755 scripts/extract_email_identity.py create mode 100755 scripts/ignore_manually_cleaned_outbox.py create mode 100644 scripts/inspect_customer_tax_id_conflict.py create mode 100755 scripts/inspect_jasmin_candidates_for_opportunity.py create mode 100755 scripts/install_systemd_timers.sh create mode 100755 scripts/prepare_task.py create mode 100644 scripts/process_outbox.py create mode 100755 scripts/recover_stale_outbox.py create mode 100755 scripts/reopen_chatwoot_review_tasks.py create mode 100644 scripts/repair_odoo_sale_order_customer_names.py create mode 100755 scripts/replace_jasmin_document_for_opportunity.py create mode 100755 scripts/reset_reconciliation_generated.py create mode 100755 scripts/review_domain_auto_applied_suggestions.py create mode 100755 scripts/run_dev.sh create mode 100755 scripts/run_reconciliation_pipeline.py create mode 100755 scripts/sync_external_reconciliation.py create mode 100755 scripts/sync_reconciliation_candidates.py create mode 100644 scripts/test_action_core_http.py create mode 100755 scripts/test_jasmin_connection.py create mode 100644 scripts/test_operational_flow.py create mode 100755 scripts/test_packlink_connection.py create mode 100755 scripts/test_sample.sh create mode 100755 scripts/validate_v45_operational_core.py create mode 100755 test_clientflow_e2e.sh create mode 100755 test_odoo_integration.sh create mode 100755 test_odoo_order_status.sh create mode 100644 tests/sample_cases/sample_analyze.json create mode 100644 tests/test_clean_architecture_static.py create mode 100644 tests/test_v40_operational_files.py create mode 100644 tests/test_v42_review_fixes.py create mode 100644 tests/test_v451_ui_alignment.py create mode 100644 tests/test_v45_operational_core.py create mode 100644 tests/test_v461_customer_mismatch_warning.py create mode 100644 tests/test_v462_disable_customer_mismatch_warning.py create mode 100644 tests/test_v462_mark_no_interest.py create mode 100644 tests/test_v463_operations_queue_cleanup.py create mode 100644 tests/test_v465_operations_work_queue.py create mode 100644 tests/test_v466_safe_opportunity_linking.py create mode 100644 tests/test_v467_chatwoot_autocomplete_guardrails.py create mode 100644 tests/test_v468_operational_safety_consolidation.py create mode 100644 tests/test_v46_chatwoot_workflow.py create mode 100644 tests/test_v472_domain_routes.py create mode 100644 tests/test_v473_htmx_partials.py create mode 100644 tests/test_v474_htmx_completion.py create mode 100644 tests/test_v47_admin_ui_refactor.py create mode 100644 tests/test_v4801_admin_menu_collapsed.py create mode 100644 tests/test_v480_operational_automation_audit_recovery.py create mode 100644 tests/test_v482_production_stabilization.py create mode 100644 tests/test_v483_operations_guardrails.py create mode 100644 tests/test_v484_fiscal_link_consistency.py create mode 100644 tests/test_v485_bounce_ignore_card_ux.py create mode 100644 tests/test_v486_opportunity_board_layout_cleanup.py create mode 100644 tests/test_v488_operations_classic_cards.py create mode 100644 tests/test_v489_operations_card_polish.py create mode 100644 tests/test_v490_operations_noise_cleanup.py create mode 100644 tests/test_v4911_odoo_name_match_reconciliation.py create mode 100644 tests/test_v4912_odoo_fulfilment_reconciliation.py create mode 100644 tests/test_v4913_apply_reconstructed_process.py create mode 100644 tests/test_v4914_reconciliation_link_hotfix.py create mode 100644 tests/test_v4915_reconciliation_amount_cast.py create mode 100644 tests/test_v491_operational_reconciliation.py create mode 100644 tests/test_v4920_reconciliation_multiple_orders.py create mode 100644 tests/test_v4920_reconciliation_multiple_purchases.py create mode 100644 tests/test_v4922_reconciliation_rebuild_button.py create mode 100644 tests/test_v4923_reconciliation_customer_first_pipeline.py create mode 100644 tests/test_v4924_reconciliation_process_review.py create mode 100644 tests/test_v4925_2_odoo_customer_field_compat.py create mode 100644 tests/test_v4925_3_odoo_customer_identity_guard.py create mode 100644 tests/test_v4925_fiscal_enrichment_worker.py create mode 100644 tests/test_v4926_1_email_identity_quality.py create mode 100644 tests/test_v4926_5_manual_opportunity_static.py create mode 100644 tests/test_v4926_6_2_jasmin_money_extraction_static.py create mode 100644 tests/test_v4926_6_jasmin_reconciliation_static.py create mode 100644 tests/test_v4926_7_identity_and_customer_guards.py create mode 100644 tests/test_v4928_decisional_ui_static.py create mode 100644 tests/test_v492_external_api_sync.py create mode 100644 tests/test_v493_jasmin_recent_sync.py create mode 100644 tests/test_v494_reconciliation_recent_window.py create mode 100644 tests/test_v495_reconciliation_operation_suggestions.py create mode 100644 tests/test_v496_reconciliation_nif_suggestions.py create mode 100644 tests/test_v498_reconciliation_matching_board_cleanup.py create mode 100644 tests/test_v499_process_timeline_reconstruction.py create mode 100644 tests/test_v499_process_timeline_reconstruction_behavior.py diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..f9ddf01 --- /dev/null +++ b/.env.example @@ -0,0 +1,91 @@ +# ClientFlow +APP_NAME=ClientFlow +ENV=dev +CLIENTFLOW_PERSIST=true +CLIENTFLOW_WEBHOOK_SECRET= + +# Base de dados +DATABASE_URL=postgresql+psycopg://clientflow:password@127.0.0.1:5432/clientflow +PSQL_DATABASE_URL=postgresql://clientflow:password@127.0.0.1:5432/clientflow + +# LLM / OpenRouter +OPENROUTER_API_KEY=coloca_a_tua_chave_aqui +OPENROUTER_MODEL=qwen/qwen3-30b-a3b + +# Chatwoot +CHATWOOT_BASE_URL=http://127.0.0.1:3010 +CHATWOOT_PUBLIC_URL=https://inbox.exemplo.pt +CHATWOOT_ACCOUNT_ID=1 +CHATWOOT_API_TOKEN= +CHATWOOT_WRITE_ENABLED=false + +# Odoo: motor de vendas, stock e produção +ODOO_ENABLED=false +ODOO_BASE_URL=https://odoo.exemplo.pt +ODOO_PUBLIC_URL=https://odoo.exemplo.pt +ODOO_DB=odoo19_prod +ODOO_USERNAME=clientflow@example.pt +ODOO_API_KEY= + +# Jasmin: faturação oficial +JASMIN_ENABLED=false +JASMIN_BASE_URL=https://my.jasminsoftware.com +JASMIN_PUBLIC_URL=https://my.jasminsoftware.com +JASMIN_ACCOUNT= +JASMIN_SUBSCRIPTION= +JASMIN_CLIENT_ID= +JASMIN_CLIENT_SECRET= + +# Packlink PRO: envios/recolhas/tracking +PACKLINK_ENABLED=false +PACKLINK_OUTBOX_ENABLED=false +PACKLINK_BASE_URL=https://api.packlink.com/v1 +PACKLINK_PUBLIC_URL=https://pro.packlink.pt +PACKLINK_API_KEY= +PACKLINK_DEFAULT_SERVICE_ID=20571 +PACKLINK_DEFAULT_SERVICE=Paq 24 +PACKLINK_DEFAULT_CARRIER=Correos Express +PACKLINK_SOURCE=PRO +PACKLINK_PLATFORM=PRO +PACKLINK_PLATFORM_COUNTRY=UN +PACKLINK_COLLECTION_TIME=09:00-14:00 +PACKLINK_COLLECTION_DAYS_AHEAD=1 +PACKLINK_DEFAULT_PACKAGE_HEIGHT=10 +PACKLINK_DEFAULT_PACKAGE_WIDTH=20 +PACKLINK_DEFAULT_PACKAGE_LENGTH=30 +PACKLINK_DEFAULT_PACKAGE_WEIGHT=2 +PACKLINK_SENDER_NAME= +PACKLINK_SENDER_SURNAME=. +PACKLINK_SENDER_COMPANY= +PACKLINK_SENDER_STREET1= +PACKLINK_SENDER_STREET2= +PACKLINK_SENDER_ZIP= +PACKLINK_SENDER_CITY= +PACKLINK_SENDER_COUNTRY=PT +PACKLINK_SENDER_PHONE= +PACKLINK_SENDER_EMAIL= +PACKLINK_FALLBACK_PHONE= +PACKLINK_FALLBACK_EMAIL= + +# Admin interno: configure atrás de proxy/auth se exposto fora da rede local +CLIENTFLOW_ADMIN_TOKEN= + +# Jasmin API validada em testes reais +JASMIN_OUTBOX_ENABLED=false +JASMIN_TOKEN_URL=https://identity.primaverabss.com/connect/token +JASMIN_SCOPE=application +JASMIN_COMPANY_KEY=CTULDA +JASMIN_QUOTATION_TYPE=ORC +JASMIN_QUOTATION_SERIE=ORC2026 +JASMIN_DEFAULT_PRICE_LIST=03 +JASMIN_DEFAULT_PAYMENT_METHOD=TRA +JASMIN_DEFAULT_PAYMENT_TERM=00 +JASMIN_DEFAULT_DELIVERY_TERM=TRANSP +JASMIN_DEFAULT_CURRENCY=EUR +JASMIN_DEFAULT_COUNTRY=PT +JASMIN_DEFAULT_CUSTOMER_GROUP=02 +JASMIN_DEFAULT_PARTY_TAX_SCHEMA=CONTINENTE +JASMIN_DEFAULT_UNIT=UN +JASMIN_DEFAULT_ITEM_TAX_SCHEMA=NORMAL +# Opcional: fallback quando a linha da oportunidade não tem SKU/Jasmin item. +JASMIN_DEFAULT_SALES_ITEM=CARREGADOR_MONO_7KW diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..059c57d --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +__pycache__/ +*.py[cod] +*.broken_* +.env +clientflow.db +.DS_Store +backups/ diff --git a/CLEANUP_SUMMARY.md b/CLEANUP_SUMMARY.md new file mode 100644 index 0000000..6aa0120 --- /dev/null +++ b/CLEANUP_SUMMARY.md @@ -0,0 +1,28 @@ +# ClientFlow cleanup aplicado + +Alterações principais: + +- `.env.example` alinhado com `OPENROUTER_MODEL=qwen/qwen3-30b-a3b`. +- `DATABASE_URL` passou a ser obrigatório e SQLite é rejeitado explicitamente. +- Schema core corrigido com criação de `task_preparations`. +- `init_db()` deixa de esconder falhas de schema. +- Triagem ficou LLM-only, com validação por catálogo fechado. +- Action catalog reduzido aos action codes atuais. +- Action codes legados removidos do fluxo principal, reclassificação, autocomplete e oportunidade. +- Webhook Chatwoot ganhou idempotência real: eventos já processados são ignorados sem duplicar análise/tarefa/nota. +- Preparação operacional deixou de chamar scripts por caminho absoluto e passou a usar `app.preparation_service`. +- `task_detail` atualizado com foco em “Próxima ação”, “Dados em falta”, “Dados confirmados”, “Mensagem sugerida” e detalhes técnicos recolhidos. +- Ficheiros `.broken_*` e caches removidos; `.gitignore` adicionado. + +Limitação consciente: + +- `admin_dashboard.py` ainda não foi totalmente partido em rotas/templates. Foi atualizado de forma incremental para reduzir risco. O próximo passo seguro é extrair `layout/nav`, CSS e task detail para templates. + + +## Iteração UI — tarefas e oportunidades + +- `/tasks` redesenhada para cartões operacionais com próxima ação, prioridade, SLA e tabela compacta recolhida. +- `/opportunities` redesenhada para lista comercial limpa com KPIs, fase, próxima ação e prioridade. +- `/opportunities/{id}` redesenhada com foco em próxima ação, resumo essencial, pipeline, produtos, tarefas e histórico. +- Rotas antigas `/tasks-bs/{id}`, `/tasks-old/{id}`, `/tasks-v3` e `/tasks/table` removidas do pacote atualizado. +- Autenticação não foi adicionada porque o acesso já é protegido pelo Nginx. diff --git a/README.md b/README.md new file mode 100644 index 0000000..6f8947c --- /dev/null +++ b/README.md @@ -0,0 +1,144 @@ +# ClientFlow MVP Backend + +ClientFlow é o novo nome do antigo CCE / Customer Context Engine. + +Função: + +```text +Receber mensagens de clientes +→ interpretar intenção com Qwen3 30B via OpenRouter +→ aplicar regras ClientFlow +→ devolver estado comercial normalizado +``` + +## Decisão técnica atual + +```text +Modelo base: qwen/qwen3-30b-a3b +Provider: OpenRouter +Qwen3 Max: não usar por defeito nesta fase +Hardware local: não comprar ainda +``` + +A validação anterior mostrou que o modelo base com regras ClientFlow é suficiente para o MVP. + +## Instalação local + +```bash +cd clientflow_mvp_backend +python3 -m venv .venv +source .venv/bin/activate +pip install -r requirements.txt +cp .env.example .env +nano .env +``` + +Define: + +```env +OPENROUTER_API_KEY=... +OPENROUTER_MODEL=qwen/qwen3-30b-a3b +``` + +## Correr API + +```bash +./scripts/run_dev.sh +``` + +Health check: + +```bash +curl http://127.0.0.1:8000/health | jq . +``` + +Testar análise: + +```bash +./scripts/test_sample.sh +``` + +## Endpoint + +```text +POST /analyze +``` + +Input: + +```json +{ + "last_customer_message": "", + "previous_context": "", + "current_state": { + "conversation_state": "a_aguardar_cliente", + "commercial_state": "proposta_enviada", + "relationship_state": "lead_existente", + "last_action_by": "empresa" + }, + "source": "email", + "conversation_id": "", + "contact_id": "" +} +``` + +Output: + +```json +{ + "app": "ClientFlow", + "model": "qwen/qwen3-30b-a3b", + "raw_llm_result": {}, + "normalized_result": {}, + "usage": {}, + "needs_review": false +} +``` + +## Próximos passos + +1. Testar `/analyze` com os casos reais. +2. Migrar os testes `caso_001...caso_050` para chamadas HTTP. +3. Guardar resultados em PostgreSQL. +4. Criar webhook receiver para Chatwoot. +5. Adicionar nota privada no Chatwoot com resumo ClientFlow. + + +## ClientFlow v3.2 + +Versão incremental com HTMX progressivo na página de oportunidade, feedback visual para ações Jasmin/outbox, validação antes de criar orçamento, reprocessamento de outbox pela UI e filtro de produtos ativos sem Artigo Jasmin. Ver `docs/CLIENTFLOW_V32_NOTES.md`. + +## ClientFlow v4 operational commands + +Apply migrations: + +```bash +python scripts/apply_migrations.py --dry-run +python scripts/apply_migrations.py +``` + +Check health: + +```bash +python scripts/check_clientflow_health.py +``` + +Install Jasmin outbox timer: + +```bash +sudo ./scripts/install_systemd_timers.sh /mnt/ssd/home/plx/clientflow_backend +``` + +## ClientFlow v4.7 — UI operacional + +A navegação da UI admin foi reorganizada para separar operação diária de diagnóstico técnico: + +- Chatwoot é a inbox/conversa. +- Operations é o centro de trabalho do operador. +- Opportunities mantém o contexto comercial. +- Admin agrupa Tasks, Communications, Outbox, Events, Runs, Queues e System health. + + +## v4.8.4 — Fiscal Link Consistency Hotfix + +Corrige a leitura do cliente fiscal em Operations: a fila passa a usar o cliente ligado à oportunidade antes de qualquer fallback da task. Isto evita cards que dizem "Cliente fiscal por associar" quando a ficha da task já mostra um cliente fiscal válido. diff --git a/RELEASE_NOTES_v4926_4.md b/RELEASE_NOTES_v4926_4.md new file mode 100644 index 0000000..16a1d3c --- /dev/null +++ b/RELEASE_NOTES_v4926_4.md @@ -0,0 +1,46 @@ +# ClientFlow v4926.4 — Email identity assisted enrichment + +## Objetivo +Integrar a extração de identidade de email no fluxo operacional de enriquecimento fiscal assistido, sem auto-aplicação agressiva. + +## Principais alterações +- Configuração separada para o modelo de email identity: + - `EMAIL_IDENTITY_LLM_MODEL` + - `EMAIL_IDENTITY_LLM_FALLBACK_MODEL` + - `EMAIL_IDENTITY_LLM_TIMEOUT_SECONDS` + - `EMAIL_IDENTITY_LLM_MAX_BODY_CHARS` +- Fallback automático no extractor: + - tenta o modelo principal; + - se o resultado ficar fraco/sem empresa em domínio empresarial, tenta o fallback. +- Nova revisão assistida de identidade: + - `email_identity_review_for_opportunity()` + - corre mesmo quando a oportunidade já tem cliente fiscal, para detetar conflitos. +- Novo enriquecimento assistido por identidade: + - `assist_email_identity_enrichment()` + - cria sugestões pendentes a partir da empresa extraída; + - deteta conflito entre empresa mencionada no email e cliente fiscal já associado; + - por defeito não auto-aplica. +- UI do detalhe da oportunidade: + - bloco “Identidade extraída do email”; + - mostra pessoa, empresas mencionadas, domínio, morada, telefones, evidências e modelo; + - botão “Reextrair identidade”; + - botão “Procurar cliente fiscal por identidade”. +- Batch validation: + - aceita `--fallback-model`; + - grava `llm_model` e `fallback_used` no CSV/JSONL. + +## Configuração recomendada + +```env +EMAIL_IDENTITY_EXTRACTION_ENABLED=true +EMAIL_IDENTITY_EXTRACTION_USE_LLM=true +EMAIL_IDENTITY_LLM_MODEL=qwen/qwen3-235b-a22b-2507 +EMAIL_IDENTITY_LLM_FALLBACK_MODEL=qwen/qwen3-max +EMAIL_IDENTITY_LLM_TIMEOUT_SECONDS=20 +EMAIL_IDENTITY_LLM_MAX_BODY_CHARS=3500 +``` + +## Testes +- `python -m compileall -q app scripts` +- `pytest -q` +- Resultado local: `188 passed` diff --git a/RELEASE_NOTES_v4926_5.md b/RELEASE_NOTES_v4926_5.md new file mode 100644 index 0000000..ac44234 --- /dev/null +++ b/RELEASE_NOTES_v4926_5.md @@ -0,0 +1,55 @@ +# ClientFlow v4.9.26.5 — Manual opportunity from fiscal customer + +Inclui também a hotfix `v4926.4.1_skip_duplicate_identity_suggestion`. + +## Alterações principais + +- Página do cliente fiscal passa a ter formulário **Nova oportunidade**. +- Permite criar oportunidade manual por telefone, WhatsApp, email, presencial ou manual. +- A oportunidade nasce já ligada ao `customers.id` fiscal. +- Opcionalmente cria uma task inicial para a próxima ação. +- Guarda snapshot fiscal do cliente na metadata da oportunidade. +- Regista evento `manual_opportunity_created` na timeline da oportunidade. +- O assistente de identidade de email deixa de criar sugestão pendente duplicada quando o cliente sugerido já é o cliente fiscal atual. +- O detalhe da oportunidade mostra mensagem específica quando a identidade extraída confirma o cliente fiscal atual. + +## Configuração recomendada + +```env +EMAIL_IDENTITY_EXTRACTION_ENABLED=true +EMAIL_IDENTITY_EXTRACTION_USE_LLM=true +EMAIL_IDENTITY_LLM_MODEL=qwen/qwen3-235b-a22b-2507 +EMAIL_IDENTITY_LLM_FALLBACK_MODEL=qwen/qwen3-max +EMAIL_IDENTITY_LLM_TIMEOUT_SECONDS=20 +EMAIL_IDENTITY_LLM_MAX_BODY_CHARS=3500 +``` + +## Testes manuais recomendados + +### Criar oportunidade a partir de cliente fiscal + +1. Abrir `/customers`. +2. Abrir uma ficha de cliente. +3. Usar o formulário **Nova oportunidade**. +4. Confirmar que a nova oportunidade abre em `/opportunities/{id}`. +5. Confirmar que o cliente fiscal já está associado. +6. Confirmar que a task inicial aparece quando a checkbox está marcada. + +### Validar skip duplicate identity suggestion + +```bash +PYTHONPATH=. python - <<'PY' +from app.fiscal_enrichment_service import assist_email_identity_enrichment +print(assist_email_identity_enrichment("OPPORTUNITY_ID", refresh=True, apply_safe=False)) +PY +``` + +Se a identidade extraída apontar para o mesmo cliente fiscal já associado, o esperado é: + +```json +{ + "suggested": 0, + "skipped": 1, + "status": "email_identity_matches_current_fiscal_customer" +} +``` diff --git a/RELEASE_v4926_3.md b/RELEASE_v4926_3.md new file mode 100644 index 0000000..79f1609 --- /dev/null +++ b/RELEASE_v4926_3.md @@ -0,0 +1,35 @@ +# v4926.3 — Email identity model config and cleanup + +## Objetivo +Melhorar a extração de identidade de emails antes do enriquecimento fiscal, com modelo LLM dedicado e limpeza determinística final. + +## Alterações +- Novo `EMAIL_IDENTITY_LLM_MODEL` separado de `OPENROUTER_MODEL`. +- `scripts/batch_validate_email_identity.py` aceita `--model` para comparar modelos sem alterar `.env`. +- O batch mostra o modelo usado no cabeçalho. +- Sanitização de `person_name`: + - remove artefactos Unicode/CJK como `Luis Roch游戏副本a`; + - rejeita rodapés tipo `Enviado do meu Galaxy`. +- Sanitização de `company_mentions`: + - remove frases longas de avisos legais/GDPR; + - extrai nomes curtos como `SOPAC, S.A` e `Imoprates, Lda`; + - rejeita mentions com palavras de ruído legal. +- `confidence` fica limitada quando não há empresa extraída, para não parecer evidência fiscal forte. +- O resultado guarda `llm_model` no payload quando usa LLM. + +## Uso recomendado +```bash +EMAIL_IDENTITY_LLM_MODEL="anthropic/claude-sonnet-4" +``` + +Ou teste isolado: +```bash +PYTHONPATH=. python scripts/batch_validate_email_identity.py \ + --limit 20 \ + --use-llm \ + --timeout-seconds 30 \ + --model "anthropic/claude-sonnet-4" +``` + +## Nota operacional +A extração continua a ser evidência para revisão/enriquecimento. Não deve auto-aplicar cliente fiscal sozinha. diff --git a/VERSION_NOTES_v4926_6_3.md b/VERSION_NOTES_v4926_6_3.md new file mode 100644 index 0000000..6e42e70 --- /dev/null +++ b/VERSION_NOTES_v4926_6_3.md @@ -0,0 +1,24 @@ +# v4926.6.3 — Jasmin opportunity item price upsert hotfix + +Fixes re-import/backfill of Jasmin quotation details when an opportunity item already exists from a previous import with zero values. + +## Changes + +- Updates existing `opportunity_items` matched by: + - `metadata->>'source_system' = 'jasmin'` + - `metadata->>'source_document'` + - `metadata->>'source_line_id'` +- Rewrites `quantity`, `unit_price`, and `total_price` using current Jasmin line values. +- Merges metadata with `price_source = jasmin_document_line`. +- Keeps rejected/cancelled items rejected/cancelled. +- Makes `jasmin_lines_imported` count updated lines as well as newly inserted lines. + +## Expected for ORC.ORC2026.156 after rerun + +- `opportunities.value_amount = 244.77` +- `commercial_documents.amount = 199.00` +- `commercial_documents.total_amount = 244.77` +- `commercial_document_lines.unit_price = 199.00` +- `commercial_document_lines.total_amount = 244.77` +- `opportunity_items.unit_price = 199.00` +- `opportunity_items.total_price = 244.77` diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/action_catalog.py b/app/action_catalog.py new file mode 100644 index 0000000..eef69e1 --- /dev/null +++ b/app/action_catalog.py @@ -0,0 +1,115 @@ +"""Catálogo único de ações atuais do ClientFlow. + +O LLM só pode produzir `TRIAGE_ACTION_CODES`. Códigos operacionais +antigos foram removidos da triagem e substituídos por eventos/operações +internas (`business_events` e `operation_links`). +""" + +TRIAGE_ACTION_CODES = { + "SEND_INFO", + "SEND_QUOTE", + "SEND_PROFORMA", + "SEND_INVOICE", + "CONFIRM_PAYMENT", + "SUPPORT", + "REMOVE_FROM_LIST", + "IGNORE_SPAM", + "REVIEW_MANUALLY", + "NO_ACTION", + "MARK_NO_INTEREST", + "IGNORE_BOUNCE", +} + +ACTION_CODES = TRIAGE_ACTION_CODES + +ACTION_MAP = { + "SEND_QUOTE": { + "route": "vendas", + "action_required": True, + "action": "Enviar proposta/cotação", + "safe_to_post": True, + "business_event_on_done": "quote_sent", + }, + "SEND_INFO": { + "route": "vendas", + "action_required": True, + "action": "Enviar informação ao cliente", + "safe_to_post": True, + "business_event_on_done": "info_sent", + }, + "SEND_PROFORMA": { + "route": "financeiro", + "action_required": True, + "action": "Emitir fatura pró-forma", + "safe_to_post": True, + "business_event_on_done": "proforma_sent", + }, + "SEND_INVOICE": { + "route": "financeiro", + "action_required": True, + "action": "Enviar fatura ao cliente", + "safe_to_post": True, + "business_event_on_done": "invoice_sent", + }, + "CONFIRM_PAYMENT": { + "route": "financeiro", + "action_required": True, + "action": "Confirmar pagamento", + "safe_to_post": True, + "business_event_on_done": "payment_confirmed", + }, + "SUPPORT": { + "route": "suporte", + "action_required": True, + "action": "Responder ao pedido de suporte", + "safe_to_post": True, + "business_event_on_done": "support_handled", + }, + "REMOVE_FROM_LIST": { + "route": "marketing", + "action_required": True, + "action": "Remover contacto da lista", + "safe_to_post": True, + "business_event_on_done": "contact_removed_from_list", + }, + "IGNORE_SPAM": { + "route": "spam", + "action_required": False, + "action": "Ignorar spam", + "safe_to_post": False, + "business_event_on_done": None, + }, + "IGNORE_BOUNCE": { + "route": "sistema", + "action_required": False, + "action": "Ignorar email devolvido/bounce", + "safe_to_post": False, + "business_event_on_done": None, + }, + "REVIEW_MANUALLY": { + "route": "rever", + "action_required": True, + "action": "Rever manualmente", + "safe_to_post": False, + "business_event_on_done": None, + }, + "NO_ACTION": { + "route": "rever", + "action_required": False, + "action": "Sem ação necessária", + "safe_to_post": False, + "business_event_on_done": None, + }, + "MARK_NO_INTEREST": { + "route": "vendas", + "action_required": True, + "action": "Marcar sem interesse", + "safe_to_post": False, + "business_event_on_done": "opportunity_no_interest", + }, +} + + +def get_action_config(action_code: str) -> dict: + code = str(action_code or "").strip().upper() + return ACTION_MAP.get(code, ACTION_MAP["REVIEW_MANUALLY"]) diff --git a/app/action_decider.py b/app/action_decider.py new file mode 100644 index 0000000..4afb476 --- /dev/null +++ b/app/action_decider.py @@ -0,0 +1,94 @@ +import re +from typing import Optional, Tuple + +from app.operation_noise import BOUNCE_NDR_PATTERNS, SYSTEM_SENDER_PATTERNS +from app.action_llm_client import decide_action_with_llm +from app.action_mapper import map_action_decision +from app.schemas import ActionDecision, ActionResult, AnalyzeRequest, UsageInfo + + +# v4.9.0 keeps NDR/Bounce patterns centralized in app.operation_noise. +# Regression terms: Your message couldn't be delivered, Recipient wasn't found, Office 365. +_BOUNCE_PATTERNS = SYSTEM_SENDER_PATTERNS + BOUNCE_NDR_PATTERNS + +_NO_INTEREST_PATTERNS = [ + r"\bn[aã]o\s+temos\s+(?:na\s+nossa\s+)?frota\s+(?:de\s+)?ve[ií]culos\s+el[eé]tricos\b", + r"\bn[aã]o\s+temos\s+(?:ve[ií]culos|viaturas|carros)\s+el[eé]tricos\b", + r"\bn[aã]o\s+possu[ií]mos\s+(?:ve[ií]culos|viaturas|carros)\s+el[eé]tricos\b", + r"\bn[aã]o\s+(?:estamos|temos)\s+interessad[oa]s?\b", + r"\bn[aã]o\s+(?:necessitamos|precisamos)\b", + r"\bn[aã]o\s+se\s+aplica\b", + r"\bsem\s+interesse\b", + r"\bsem\s+necessidade\b", +] + + +def _normalize_text(value: str) -> str: + text = str(value or "").casefold() + text = re.sub(r"<[^>]+>", " ", text) + text = re.sub(r"https?://\S+", " ", text) + text = re.sub(r"\s+", " ", text).strip() + return text + + +def detect_deterministic_action(request: AnalyzeRequest) -> Optional[ActionDecision]: + """Regras de alta confiança antes do LLM. + + Usadas só para respostas inequívocas que devem gerar uma ação operacional + própria. A regra evita classificar recusas explícitas como SUPPORT ou + REVIEW_MANUALLY. + """ + text = _normalize_text("\n".join([request.previous_context or "", request.last_customer_message or ""])) + if not text: + return None + + for pattern in _BOUNCE_PATTERNS: + if re.search(pattern, text, flags=re.I): + return ActionDecision( + action_code="IGNORE_BOUNCE", + note="Mensagem automática de devolução/erro de entrega. Ignorar no fluxo operacional.", + confidence=0.99, + ) + + for pattern in _NO_INTEREST_PATTERNS: + if re.search(pattern, text, flags=re.I): + return ActionDecision( + action_code="MARK_NO_INTEREST", + note="Cliente indicou que não tem interesse/necessidade atual.", + confidence=0.95, + ) + + return None + + +async def decide_action(request: AnalyzeRequest) -> Tuple[ActionDecision, ActionResult, UsageInfo, str]: + """Triagem de mensagens. + + Mantém LLM para a maioria dos casos, mas aplica regras determinísticas de + alta confiança para intenções críticas/inequívocas que devem ser estáveis. + """ + deterministic = detect_deterministic_action(request) + if deterministic: + result = map_action_decision(deterministic) + usage = UsageInfo( + id=None, + model="deterministic-rule", + provider="rule", + prompt_tokens=0, + completion_tokens=0, + total_tokens=0, + cost=0.0, + ) + return deterministic, result, usage, "rule" + + decision, usage, _raw = await decide_action_with_llm(request) + result = map_action_decision(decision) + + # Garante que a decisão persistida reflete o código normalizado/permitido. + decision = ActionDecision( + action_code=result.action_code, + note=decision.note or result.note, + confidence=decision.confidence, + ) + + return decision, result, usage, "llm" diff --git a/app/action_llm_client.py b/app/action_llm_client.py new file mode 100644 index 0000000..1355b4d --- /dev/null +++ b/app/action_llm_client.py @@ -0,0 +1,243 @@ +import json +import re +from typing import Any, Dict, Tuple + +import httpx + +from app.action_prompt import build_action_system_prompt, build_action_user_prompt +from app.action_catalog import TRIAGE_ACTION_CODES +from app.action_mapper import ACTION_CODE_ALIASES +from app.config import settings +from app.schemas import ActionDecision, AnalyzeRequest, UsageInfo + + +def extract_first_json_object(raw: str) -> str: + """ + Extrai o primeiro objeto JSON de uma resposta LLM. + Suporta markdown, texto antes/depois e quebras de linha. + """ + s = str(raw or "").strip() + + if s.startswith("```"): + lines = s.splitlines() + if lines and lines[0].strip().startswith("```"): + lines = lines[1:] + if lines and lines[-1].strip().startswith("```"): + lines = lines[:-1] + s = "\n".join(lines).strip() + + start = s.find("{") + if start == -1: + raise ValueError(f"no JSON object found: {s[:300]}") + + in_string = False + escaped = False + depth = 0 + + for i in range(start, len(s)): + ch = s[i] + + if escaped: + escaped = False + continue + + if ch == "\\": + escaped = True + continue + + if ch == '"': + in_string = not in_string + continue + + if in_string: + continue + + if ch == "{": + depth += 1 + elif ch == "}": + depth -= 1 + if depth == 0: + return s[start:i + 1] + + raise ValueError(f"incomplete JSON object: {s[:500]}") + + +def fallback_parse_decision_text(raw: str) -> Dict[str, Any]: + """ + Fallback para respostas quase-JSON. + Ex.: note com aspas internas não escapadas. + Só extrai campos explícitos; não inventa decisão. + """ + s = str(raw or "") + + action_match = re.search( + r'["\']action_code["\']\s*:\s*["\']([A-Z0-9_]+)["\']', + s, + flags=re.I, + ) + + confidence_match = re.search( + r'["\']confidence["\']\s*:\s*([0-9]+(?:\.[0-9]+)?)', + s, + flags=re.I, + ) + + note_match = re.search( + r'["\']note["\']\s*:\s*["\'](.+?)["\']\s*(?:,|\n\s*["\']confidence|})', + s, + flags=re.I | re.S, + ) + + if not action_match: + raise ValueError(f"could not fallback-parse action_code: {s[:500]}") + + confidence = 0.75 + if confidence_match: + try: + confidence = float(confidence_match.group(1)) + except Exception: + confidence = 0.75 + + note = "" + if note_match: + note = note_match.group(1).strip() + note = note.replace('\\"', '"') + note = re.sub(r"\s+", " ", note) + + return { + "action_code": action_match.group(1).upper(), + "note": note, + "confidence": confidence, + } + + +def extract_json(text: str) -> Dict[str, Any]: + raw = str(text or "").strip() + + try: + return json.loads(raw) + except Exception: + pass + + try: + candidate = extract_first_json_object(raw) + return json.loads(candidate) + except Exception: + return fallback_parse_decision_text(raw) + + +def clean_llm_text(raw: str) -> str: + s = str(raw or "").strip() + if s.startswith("```"): + lines = s.splitlines() + if lines and lines[0].strip().startswith("```"): + lines = lines[1:] + if lines and lines[-1].strip().startswith("```"): + lines = lines[:-1] + s = "\n".join(lines).strip() + return s.strip().strip('"').strip("'").strip() + + +def parse_action_code_response(raw: str) -> ActionDecision: + """Parse da resposta LLM-only. + + O formato esperado é apenas o action_code, mas aceitamos JSON antigo + {"action_code": "..."} para compatibilidade durante transição. + """ + text = clean_llm_text(raw) + + # Caminho principal: resposta é só o código. + candidate = re.sub(r"[^A-Za-z0-9_].*$", "", text).strip().upper() + candidate = ACTION_CODE_ALIASES.get(candidate, candidate) + if candidate in TRIAGE_ACTION_CODES: + return ActionDecision(action_code=candidate, note="", confidence=0.85) + + # Compatibilidade com respostas JSON antigas. + try: + data = extract_json(text) + raw_code = str(data.get("action_code") or "").strip().upper() + code = ACTION_CODE_ALIASES.get(raw_code, raw_code) + if code in TRIAGE_ACTION_CODES: + return ActionDecision( + action_code=code, + note=str(data.get("note") or "").strip(), + confidence=float(data.get("confidence") or 0.85), + ) + except Exception: + pass + + # Tenta encontrar um código permitido algures no texto, mas sem inventar. + upper_text = text.upper() + for code in sorted(TRIAGE_ACTION_CODES, key=len, reverse=True): + if re.search(rf"\b{re.escape(code)}\b", upper_text): + return ActionDecision(action_code=code, note="", confidence=0.75) + + return ActionDecision( + action_code="REVIEW_MANUALLY", + note=f"Resposta LLM inválida para action_code: {text[:120]}", + confidence=0.0, + ) + + +def parse_decision(data: Dict[str, Any]) -> ActionDecision: + # Mantido para compatibilidade com imports/testes antigos. + return ActionDecision( + action_code=str(data.get("action_code") or "REVIEW_MANUALLY"), + note=str(data.get("note") or "").strip(), + confidence=float(data.get("confidence") or 0.0), + ) + + +async def decide_action_with_llm(request: AnalyzeRequest) -> Tuple[ActionDecision, UsageInfo, Dict[str, Any]]: + payload = { + "model": settings.openrouter_model, + "messages": [ + {"role": "system", "content": build_action_system_prompt()}, + { + "role": "user", + "content": build_action_user_prompt( + request.last_customer_message, + request.previous_context, + request.current_state, + ), + }, + ], + "temperature": 0, + "max_tokens": 20, + "reasoning": { + "effort": "none", + "exclude": True, + }, + } + + headers = { + "Authorization": f"Bearer {settings.openrouter_api_key}", + "Content-Type": "application/json", + } + + async with httpx.AsyncClient(timeout=60) as client: + response = await client.post( + settings.openrouter_url, + headers=headers, + json=payload, + ) + + response.raise_for_status() + raw_response = response.json() + + content = raw_response["choices"][0]["message"].get("content") or "" + decision = parse_action_code_response(content) + + usage_data = raw_response.get("usage") or {} + + usage = UsageInfo( + id=raw_response.get("id"), + model=raw_response.get("model") or settings.openrouter_model, + provider=raw_response.get("provider"), + prompt_tokens=usage_data.get("prompt_tokens") or 0, + completion_tokens=usage_data.get("completion_tokens") or 0, + total_tokens=usage_data.get("total_tokens") or 0, + cost=usage_data.get("cost") or 0.0, + ) + + return decision, usage, raw_response diff --git a/app/action_mapper.py b/app/action_mapper.py new file mode 100644 index 0000000..5b4d505 --- /dev/null +++ b/app/action_mapper.py @@ -0,0 +1,36 @@ +from app.action_catalog import ACTION_CODES, get_action_config +from app.schemas import ActionDecision, ActionResult + + +# Apenas normalizações benignas de formato/typo. Códigos legados operacionais +# não são convertidos: se aparecerem, a tarefa vai para revisão. +ACTION_CODE_ALIASES = { + "SENDO_INVOICE": "SEND_INVOICE", + "SENDING_INVOICE": "SEND_INVOICE", +} + + +def normalize_action_code(value: str) -> str: + code = str(value or "").strip().upper() + code = ACTION_CODE_ALIASES.get(code, code) + if code not in ACTION_CODES: + return "REVIEW_MANUALLY" + return code + + +def map_action_decision(decision: ActionDecision) -> ActionResult: + code = normalize_action_code(decision.action_code) + config = get_action_config(code) + + note = str(decision.note or "").strip() + if not note: + note = config["action"] + + return ActionResult( + action_code=code, + route=config["route"], + action_required=config["action_required"], + action=config["action"], + note=note, + safe_to_post=config["safe_to_post"], + ) diff --git a/app/action_prompt.py b/app/action_prompt.py new file mode 100644 index 0000000..57262ea --- /dev/null +++ b/app/action_prompt.py @@ -0,0 +1,136 @@ +from app.action_catalog import TRIAGE_ACTION_CODES + + +def build_action_system_prompt() -> str: + actions = "\n".join(f"- {code}" for code in sorted(TRIAGE_ACTION_CODES)) + + return f"""És o ClientFlow, um classificador LLM-only de emails B2B para a BLIF. + +A BLIF vende carregadores para veículos elétricos. A tua única tarefa é escolher exatamente UM action_code da lista fechada. + +Ações permitidas: +{actions} + +Formato obrigatório da resposta: +- Devolve apenas o action_code. +- Não devolvas JSON. +- Não expliques. +- Não escrevas texto antes ou depois. + +Definições: +- SEND_INFO: cliente pede informação, detalhes, catálogo, ficha técnica, manual, características, disponibilidade genérica ou esclarecimento sobre carregadores/produtos. +- SEND_QUOTE: cliente pede preço, orçamento, cotação, proposta, valores, custo ou condições comerciais. +- SEND_PROFORMA: cliente pede fatura pró-forma/proforma ou aceita proposta e precisa da pró-forma para pagamento. +- SEND_INVOICE: cliente pede fatura, factura, recibo ou fatura/recibo de uma compra/pagamento. +- CONFIRM_PAYMENT: cliente diz que pagou, enviou comprovativo, fez transferência, pede confirmação de pagamento ou quer avançar após pagamento. +- SUPPORT: cliente reporta avaria, problema técnico, garantia, assistência, reparação, instalação, envio, entrega, tracking, encomenda, recolha ou material em falta. +- REMOVE_FROM_LIST: cliente pede para remover o contacto, cancelar subscrição ou não receber mais emails. +- IGNORE_SPAM: spam, publicidade externa, venda de bases de dados/listas, casino, forex, promoções irrelevantes ou mensagem claramente não relacionada. +- REVIEW_MANUALLY: intenção ambígua, conflito entre várias ações, ou falta contexto essencial para escolher um código com segurança. +- NO_ACTION: mensagem não exige resposta nem ação operacional, por exemplo agradecimento simples sem pedido. +- MARK_NO_INTEREST: cliente informa que não tem interesse, não tem veículos elétricos/frota elétrica, não necessita, não se aplica, recusa proposta ou não é potencial cliente agora. + +Prioridade quando houver várias intenções: +1. REMOVE_FROM_LIST +2. IGNORE_SPAM +3. CONFIRM_PAYMENT +4. SEND_PROFORMA +5. SEND_INVOICE +6. SEND_QUOTE +7. SEND_INFO +8. SUPPORT +9. MARK_NO_INTEREST +10. NO_ACTION +11. REVIEW_MANUALLY + +Regras importantes: +- Emails curtos mas claros devem ser classificados automaticamente. Ex.: assunto "Carregador" + mensagem "Enviar informação detalhada" = SEND_INFO. +- Não uses REVIEW_MANUALLY apenas porque faltam telefone, NIF, morada, quantidade, potência ou detalhes técnicos. +- Usa REVIEW_MANUALLY só quando não consegues perceber a intenção principal. +- Classifica a intenção de negócio, não apenas palavras exatas. +- Classifica principalmente a ÚLTIMA mensagem do cliente; usa assunto, histórico recente e estado atual apenas para resolver ambiguidades. +- Não escolhas uma ação antiga só porque aparece no histórico anterior. + +Exemplos: +Assunto: Carregador +Mensagem: Enviar informação detalhada. +Resposta: +SEND_INFO + +Mensagem: Envie ficha técnica do carregador. +Resposta: +SEND_INFO + +Mensagem: Pretendo orçamento para 2 carregadores monofásicos. +Resposta: +SEND_QUOTE + +Mensagem: Qual o preço da wallbox de 22 kW? +Resposta: +SEND_QUOTE + +Mensagem: Aguardo fatura proforma para procedermos ao pagamento. +Resposta: +SEND_PROFORMA + +Mensagem: Pode enviar a fatura/recibo? +Resposta: +SEND_INVOICE + +Mensagem: Segue comprovativo da transferência. +Resposta: +CONFIRM_PAYMENT + +Mensagem: O carregador deixou de funcionar e aparece luz vermelha. +Resposta: +SUPPORT + +Mensagem: Quando vai ser entregue a encomenda? +Resposta: +SUPPORT + +Mensagem: Removam-me da lista. +Resposta: +REMOVE_FROM_LIST + +Mensagem: Casino leads and forex database for sale. +Resposta: +IGNORE_SPAM + +Mensagem: Obrigado. +Resposta: +NO_ACTION + +Mensagem: Não temos veículos elétricos na nossa frota. +Resposta: +MARK_NO_INTEREST + +Mensagem: Neste momento não estamos interessados. +Resposta: +MARK_NO_INTEREST +""" + + +def build_action_user_prompt(last_message: str, previous_context: str = "", current_state=None) -> str: + state_block = "" + if current_state: + try: + state_block = f"""\nEstado interno conhecido: +- última ação: {getattr(current_state, 'last_action_code', 'desconhecido')} +- última fila: {getattr(current_state, 'last_route', 'desconhecido')} +- último estado de tarefa: {getattr(current_state, 'last_task_status', 'desconhecido')}\n""" + except Exception: + state_block = "" + + return f"""Classifica a próxima ação operacional com base principalmente na ÚLTIMA mensagem do cliente. +Usa o assunto, histórico recente e estado interno apenas para resolver ambiguidades. +Não escolhas uma ação antiga só porque aparece no histórico. + +Contexto compacto: +{previous_context or "Sem contexto anterior."} +{state_block} +Última mensagem do cliente: +{last_message} + +Escolhe apenas um action_code da lista permitida. +""" diff --git a/app/admin_dashboard.py b/app/admin_dashboard.py new file mode 100644 index 0000000..81ad9c5 --- /dev/null +++ b/app/admin_dashboard.py @@ -0,0 +1,1784 @@ +from pathlib import Path +import os +from datetime import datetime, timezone +import html +import json +from uuid import UUID +from hmac import compare_digest +from typing import Optional + +from fastapi import APIRouter, Request, Depends, HTTPException +from fastapi.responses import HTMLResponse, RedirectResponse, PlainTextResponse, Response +from starlette.concurrency import run_in_threadpool + +from app.admin_queries import list_action_runs, list_business_events +from app.integration_outbox_service import get_outbox_item, list_outbox, set_outbox_status +from app.config import settings +from app.preparation_service import prepare_task as run_task_preparation +from app.preparation_view_model import build_preparation_view_model +from app.workflow_guard import OperationActionBlocked, get_workflow_action_plan +from app.odoo_service import ( + test_odoo_connection, + sync_odoo_products, + get_odoo_product_snapshot, + sync_opportunity_odoo_status, +) +from app.operation_service import get_operation_snapshot, operation_next_steps, register_operation_action +from app.operations_service import get_operations_summary, get_system_health_summary, list_unified_opportunity_timeline +from app.communication_service import ( + classification_action, + create_timeline_event, + get_communication, + get_communications_summary, + link_communication_to_customer, + link_communication_to_opportunity, + list_communications, + list_communications_for_opportunity, + set_communication_status, +) +from app.task_service import complete_task, complete_task_with_note, get_admin_dashboard_metrics, get_customer_profile, get_system_health_metrics, get_task_detail, get_latest_task_preparation, list_admin_recent_raw_events, list_admin_recent_tasks, list_customer_messages, list_customer_opportunity_mappings, list_customer_task_history, list_customer_tasks, list_tasks, skip_task, reclassify_task +from app.opportunity_service import ( + OPPORTUNITY_BOARD_COLUMNS, + OPPORTUNITY_STAGE_LABELS, + get_opportunity, + list_opportunities, + list_opportunity_events, + list_opportunity_tasks, + set_opportunity_stage, + stage_label, +) +from app.product_service import ( + add_opportunity_item, + create_product, + delete_opportunity_item, + get_product, + list_opportunity_items, + list_product_categories, + list_products, + set_product_active, + update_opportunity_item, + update_product, +) +from app.admin_ui.components import kpi_card +from app.admin_ui.layout import layout +from app.admin_ui.styles import ADMIN_UI_V451_CSS + + +# ADMIN_UI_CSS moved to app.admin_ui.styles in v4.7. +# Route handlers moved to app.admin_ui.pages.* in v4.7.2. + + + + +def require_admin_access(request: Request) -> None: + """Proteção opcional da UI admin. + + Se CLIENTFLOW_ADMIN_TOKEN estiver vazio, mantém compatibilidade local. + Em produção deve ser definido e enviado em X-ClientFlow-Admin-Token, + cookie clientflow_admin_token, ou query param admin_token atrás de HTTPS/proxy. + """ + expected = (settings.clientflow_admin_token or "").strip() + if not expected: + return + received = ( + request.headers.get("X-ClientFlow-Admin-Token") + or request.cookies.get("clientflow_admin_token") + or request.query_params.get("admin_token") + or "" + ).strip() + if not received or not compare_digest(received, expected): + raise HTTPException(status_code=401, detail="admin auth required") + + +router = APIRouter(prefix="", tags=["admin"], dependencies=[Depends(require_admin_access)]) + + +def esc(value) -> str: + return html.escape(str(value or "")) + + +def chatwoot_conversation_url(conversation_id: object) -> str: + conversation_id = str(conversation_id or "").strip() + if not conversation_id: + return "" + public_url = ( + getattr(settings, "chatwoot_public_url", "") + or getattr(settings, "chatwoot_base_url", "") + or "" + ).rstrip("/") + account_id = str(getattr(settings, "chatwoot_account_id", "") or "").strip() + if not public_url or not account_id: + return "" + return f"{public_url}/app/accounts/{account_id}/conversations/{conversation_id}" + + +def chatwoot_button(conversation_id: object, label: str = "Abrir Chatwoot") -> str: + href = chatwoot_conversation_url(conversation_id) + if not href: + return "" + return f' {esc(label)}' + + + +def shell_output(cmd: list[str], *, timeout: int = 8) -> str: + try: + import subprocess + result = subprocess.run( + cmd, + capture_output=True, + text=True, + timeout=timeout, + check=False, + ) + output = (result.stdout or "") + (result.stderr or "") + return output.strip() + except Exception as e: + return f"erro ao executar {' '.join(cmd)}: {e!r}" + + +def status_pill(ok: bool, label: str) -> str: + cls = "pill-ok" if ok else "pill-bad" + return f'{esc(label)}' + + +def latest_backup_info() -> dict: + backup_dir = Path(os.getenv("CLIENTFLOW_BACKUP_DIR", "./backups/clientflow")) + files = sorted( + backup_dir.glob("clientflow-*.sql.gz"), + key=lambda x: x.stat().st_mtime if x.exists() else 0, + reverse=True, + ) + + if not files: + return { + "exists": False, + "file": "", + "size": "", + "mtime": "", + } + + f = files[0] + stat = f.stat() + + return { + "exists": True, + "file": str(f), + "size": f"{stat.st_size / 1024:.1f} KB", + "mtime": datetime.fromtimestamp(stat.st_mtime).isoformat(timespec="seconds"), + } + + +def done_note_options_html_for(action_code: str) -> str: + done_note_templates = { + "SEND_INFO": [ + "Informação enviada ao cliente no Chatwoot.", + "Cliente informado por email.", + "Informação comercial enviada; aguardar resposta.", + ], + "SEND_QUOTE": [ + "Proposta/cotação enviada ao cliente.", + "Cotação enviada no Chatwoot.", + "Proposta enviada; aguardar confirmação do cliente.", + ], + "SEND_PROFORMA": [ + "Fatura pró-forma emitida/enviada ao cliente.", + "Pró-forma enviada; aguardar pagamento/confirmação.", + ], + "SEND_INVOICE": [ + "Fatura enviada ao cliente.", + "Cliente informado do envio da fatura.", + ], + "CONFIRM_PAYMENT": [ + "Pagamento confirmado.", + "Comprovativo validado; processo segue para operações se aplicável.", + ], + "SUPPORT": [ + "Pedido de suporte respondido ou encaminhado.", + "Cliente informado; suporte vai acompanhar o caso.", + "Pedido encaminhado para análise.", + ], + "REMOVE_FROM_LIST": [ + "Contacto removido da lista.", + "Pedido de remoção tratado.", + ], + "REVIEW_MANUALLY": [ + "Caso revisto manualmente.", + "Sem ação automática; tratado manualmente.", + ], + "MARK_NO_INTEREST": [ + "Marcado sem interesse atual.", + "Cliente informou que não tem necessidade atual.", + "Oportunidade encerrada/sem seguimento comercial por agora.", + ], + "NO_ACTION": [ + "Sem ação necessária.", + ], + "IGNORE_SPAM": [ + "Mensagem ignorada como spam.", + ], + } + + default_done_notes = [ + "Tarefa concluída.", + "Cliente informado no Chatwoot.", + "Pedido tratado manualmente.", + ] + + templates = done_note_templates.get(action_code or "", default_done_notes) + + return "".join( + f'' + for option in templates + ) + + +def suggested_reply_for_task(task: dict) -> str: + action_code = task.get("action_code") or "" + customer_name = task.get("customer_name") or "" + first_name = str(customer_name).strip().split(" ")[0] if customer_name else "" + + greeting = f"Olá {first_name}," if first_name and first_name.lower() not in ["cliente", "desconhecido"] else "Olá," + closing = "Obrigado,\nEquipa BLIF" + + templates = { + "SEND_INFO": f"""{greeting} + +Obrigado pelo seu contacto. + +Segue informação sobre os nossos carregadores para veículos elétricos. Podemos ajudar com a escolha do modelo mais adequado, disponibilidade, condições de entrega e instalação. + +Caso pretenda, envie-nos por favor: +- tipo de viatura; +- local de instalação; +- potência disponível; +- se pretende carregador monofásico ou trifásico. + +{closing}""", + + "SEND_QUOTE": f"""{greeting} + +Obrigado pelo seu pedido. + +Vamos preparar/enviar a proposta para o carregador solicitado, incluindo preço, disponibilidade e condições de entrega. + +Se ainda não tiver indicado, confirme por favor: +- modelo pretendido; +- quantidade; +- morada/localidade para entrega; +- dados para faturação, se desejar avançar. + +{closing}""", + + "SEND_PROFORMA": f"""{greeting} + +Podemos emitir/enviar a fatura pró-forma. + +Para isso, envie por favor os dados de faturação: +- nome/empresa; +- NIF; +- morada; +- email para envio; +- produto/quantidade pretendida. + +{closing}""", + + "SEND_INVOICE": f"""{greeting} + +Obrigado pela confirmação. + +Vamos enviar a fatura conforme solicitado. Caso ainda não tenha enviado os dados de faturação, envie por favor: +- nome/empresa; +- NIF; +- morada; +- email. + +{closing}""", + + "CONFIRM_PAYMENT": f"""{greeting} + +Obrigado pelo envio da informação/comprovativo. + +Vamos confirmar o pagamento e dar seguimento ao processo. Se for aplicável, encaminhamos também a encomenda para preparação/envio. + +Assim que tivermos atualização, informamos. + +{closing}""", + + "SUPPORT": f"""{greeting} + +Obrigado pelo contacto. + +Vamos encaminhar o seu pedido para suporte. Para ajudar na análise, envie por favor, se aplicável: +- modelo do carregador/equipamento; +- descrição do pedido/problema; +- fotos/vídeos, se possível; +- morada/local de instalação, entrega ou recolha; +- contacto telefónico. + +{closing}""", + + "REMOVE_FROM_LIST": f"""{greeting} + +Confirmamos que vamos tratar o pedido de remoção da lista de contactos. + +{closing}""", + + "MARK_NO_INTEREST": f"""{greeting} + +Obrigado pela informação. + +Ficamos ao dispor caso no futuro venham a integrar veículos elétricos na frota ou necessitem de soluções de carregamento. + +{closing}""", + + "REVIEW_MANUALLY": f"""{greeting} + +Obrigado pela sua mensagem. + +Vamos analisar o pedido internamente e responder assim que possível. + +{closing}""", + } + + return templates.get(action_code, f"""{greeting} + +Obrigado pela sua mensagem. + +Vamos analisar o pedido e responder assim que possível. + +{closing}""") + + +ACTION_UI_LABELS = { + "SEND_INFO": "Enviar informação", + "SEND_QUOTE": "Enviar orçamento", + "SEND_PROFORMA": "Enviar pró-forma", + "SEND_INVOICE": "Enviar fatura", + "CONFIRM_PAYMENT": "Confirmar pagamento", + "SUPPORT": "Tratar suporte", + "REMOVE_FROM_LIST": "Remover da lista", + "MARK_NO_INTEREST": "Marcar sem interesse", + "IGNORE_SPAM": "Ignorar spam", + "REVIEW_MANUALLY": "Rever manualmente", + "NO_ACTION": "Sem ação", +} + +ACTION_HINTS = { + "SEND_INFO": "Enviar informação geral e pedir os dados mínimos para recomendar o carregador certo.", + "SEND_QUOTE": "Preparar/enviar orçamento com preço, disponibilidade, condições de entrega e dados necessários para avançar.", + "SEND_PROFORMA": "Recolher dados de faturação e emitir/enviar a pró-forma.", + "SEND_INVOICE": "Confirmar dados de faturação e enviar a fatura solicitada.", + "CONFIRM_PAYMENT": "Validar pagamento/comprovativo e encaminhar para preparação/envio se aplicável.", + "SUPPORT": "Responder ao pedido e recolher informação técnica mínima para análise.", + "REMOVE_FROM_LIST": "Confirmar remoção do contacto de comunicações futuras.", + "MARK_NO_INTEREST": "Cliente indicou ausência de interesse atual/necessidade após divulgação; não é o mesmo que oportunidade perdida por preço ou funcionalidades.", + "IGNORE_SPAM": "Ignorar a mensagem e não criar seguimento comercial.", + "REVIEW_MANUALLY": "Analisar manualmente porque a intenção não ficou suficientemente clara.", + "NO_ACTION": "Não é necessária ação operacional.", +} + +ACTION_MISSING_HINTS = { + "SEND_INFO": ["potência pretendida", "tipo de instalação", "localidade", "contacto telefónico"], + "SEND_QUOTE": ["modelo/produto", "quantidade", "morada/localidade", "dados de faturação se avançar"], + "SEND_PROFORMA": ["nome/empresa", "NIF", "morada fiscal", "email de faturação", "produto/quantidade"], + "SEND_INVOICE": ["nome/empresa", "NIF", "morada fiscal", "email de faturação"], + "CONFIRM_PAYMENT": ["valor recebido", "referência/comprovativo", "morada de entrega", "contacto para entrega"], + "SUPPORT": ["modelo", "descrição do problema", "fotos/vídeos", "local de instalação", "telefone"], + "MARK_NO_INTEREST": ["motivo", "se é apenas falta de interesse atual", "se deve manter contacto para futuro"], +} + +PIPELINE_STEPS = [ + ("NEW_LEAD", "Novo pedido"), + ("INFO_SENT", "Info enviada"), + ("QUOTE_SENT", "Proposta"), + ("PROFORMA_SENT", "Pró-forma"), + ("PAYMENT_CONFIRMED", "Pagamento"), + ("ODOO_ORDER_CREATED", "Odoo"), + ("IN_PRODUCTION", "Produção"), + ("READY_TO_SHIP", "Pronto"), + ("SHIPMENT_CREATED", "Envio"), + ("WON", "Concluído"), + ("NO_INTEREST", "Sem interesse"), +] + + +def action_label(code: str) -> str: + code = str(code or "").strip().upper() + return ACTION_UI_LABELS.get(code, code or "Tarefa") + + +def compact_text(value, limit: int = 120) -> str: + text = " ".join(str(value or "").split()) + if len(text) > limit: + return text[: max(0, limit - 1)].rstrip() + "…" + return text + + + +def fmt_dt(value) -> str: + """Formata datas/timestamps para leitura rápida na UI.""" + if not value: + return "—" + try: + if isinstance(value, str): + dt = datetime.fromisoformat(value.replace("Z", "+00:00")) + else: + dt = value + if getattr(dt, "tzinfo", None) is None: + dt = dt.replace(tzinfo=timezone.utc) + return dt.strftime("%Y-%m-%d %H:%M") + except Exception: + return compact_text(value, 32) + + +def humanize_task_detail(value) -> str: + detail = str(value or "").strip() + lower = detail.casefold() + if "resposta llm inválida" in lower or ("invalid" in lower and "action_code" in lower): + return "Classificação da mensagem falhou. Rever no Chatwoot e escolher a ação correta." + if "limpo manualmente" in lower or "resolvido manualmente" in lower: + return "Item já limpo manualmente. Deve ficar no histórico, não na fila diária." + return detail + + +def task_next_action_text(task: dict) -> str: + note = compact_text(humanize_task_detail(task.get("note") or task.get("action") or ""), 120) + if note: + return note + code = str(task.get("action_code") or "") + return action_label(code) + + +def opportunity_contact_name(opportunity: dict) -> str: + return str( + opportunity.get("customer_name") + or opportunity.get("customer_email") + or opportunity.get("contact_id") + or "Cliente" + ).strip() + + +def opportunity_customer_name(opportunity: dict) -> str: + # Preferir a ficha fiscal ligada, porque é ela que será usada para Jasmin, + # faturas e envios. O contacto original continua visível como origem. + return str( + opportunity.get("linked_customer_name") + or opportunity.get("customer_name") + or opportunity.get("customer_email") + or opportunity.get("contact_id") + or "Cliente" + ).strip() + + +def _norm_customer_text(value: object) -> str: + return " ".join(str(value or "").strip().casefold().split()) + + +def _customer_name_tokens(value: object) -> set[str]: + text = _norm_customer_text(value) + for ch in "-_,.;:/()[]{}+&|\n\t": + text = text.replace(ch, " ") + legal_suffixes = { + "lda", "ltd", "sa", "s", "a", "unipessoal", "sociedade", "limitada", + "empresa", "companhia", "pt", "portugal", "the", "and", "e", "de", "da", "do", + "das", "dos", "para", "com", "ao", "aos", "as", "os", + } + tokens: set[str] = set() + for token in text.split(): + token = token.strip() + if len(token) < 3 or token in legal_suffixes or "@" in token: + continue + tokens.add(token) + # Prefixes reduce false positives with short/truncated labels such as + # "Riotec elec" vs "Riotec - Electricidade, ...". + if len(token) >= 4: + tokens.add(token[:4]) + return tokens + + +def opportunity_customer_mismatch(opportunity: dict) -> bool: + """Nome de contacto ≠ cliente fiscal não é um erro fiável. + + Ex.: contacto "Bruno Oliveira" pode representar a empresa fiscal + "Nortuflex"; "Riotec elec" pode ser abreviação da entidade fiscal. + A validação crítica deve focar NIF/morada/documentos, não semelhança de nomes. + """ + return False + + +def opportunity_customer_context_html(opportunity: dict) -> str: + linked = opportunity.get("linked_customer_name") + original = opportunity.get("customer_name") or opportunity.get("customer_email") + tax_id = opportunity.get("linked_customer_tax_id") + linked_email = opportunity.get("linked_customer_email") + if linked: + html = f'
Cliente fiscal: {esc(linked)}' + if tax_id: + html += f' · NIF {esc(tax_id)}' + html += '
' + if opportunity_customer_mismatch(opportunity): + html += f'
Contacto original: {esc(original or "—")}
' + elif linked_email: + html += f'
{esc(linked_email)}
' + return html + return f'
{esc(original or "Sem cliente fiscal associado")}
' + + +def opportunity_next_action_text(opportunity: dict) -> str: + pending_count = int(opportunity.get("pending_task_count") or 0) + stage = str(opportunity.get("stage") or "NEW_LEAD") + last_action = str(opportunity.get("last_action_code") or "") + if pending_count: + return "Concluir tarefa pendente" + by_stage = { + "NEW_LEAD": "Qualificar pedido", + "INFO_REQUESTED": "Aguardar dados do cliente", + "INFO_SENT": "Confirmar interesse", + "QUOTE_REQUESTED": "Preparar proposta", + "QUOTE_SENT": "Acompanhar decisão", + "PROFORMA_REQUESTED": "Preparar pró-forma", + "PROFORMA_SENT": "Aguardar pagamento", + "INVOICE_REQUESTED": "Emitir fatura", + "INVOICE_SENT": "Aguardar pagamento", + "WAITING_PAYMENT": "Confirmar pagamento", + "PAYMENT_CONFIRMED": "Preparar encomenda", + "ORDER_PREPARATION": "Preparar material/envio", + "ODOO_ORDER_CREATED": "Validar estado Odoo", + "IN_PRODUCTION": "Acompanhar produção", + "READY_TO_SHIP": "Criar envio", + "INVOICED": "Criar/validar envio", + "SHIPMENT_CREATED": "Enviar tracking", + "SHIPPED": "Acompanhar entrega", + "TRACKING_SENT": "Acompanhar entrega", + "DELIVERED": "Fechar oportunidade", + "WON": "Concluída", + "LOST": "Perdida", + "NO_INTEREST": "Sem interesse", + "REVIEW": "Rever manualmente", + } + if last_action: + return by_stage.get(stage, action_label(last_action)) + return by_stage.get(stage, "Acompanhar oportunidade") + + +def opportunity_priority_chip(opportunity: dict) -> str: + pending = int(opportunity.get("pending_task_count") or 0) + stage = str(opportunity.get("stage") or "") + if pending: + return 'Requer ação' + if stage in {"WAITING_PAYMENT", "PAYMENT_CONFIRMED", "ORDER_PREPARATION", "READY_TO_SHIP"}: + return 'Prioritária' + if stage in {"WON", "LOST", "NO_INTEREST"}: + return 'Fechada' + return 'Normal' + + + + +def is_uuid_text(value: object) -> bool: + try: + UUID(str(value or "")) + return True + except Exception: + return False + + +def metadata_dict(value) -> dict: + if isinstance(value, dict): + return value + if isinstance(value, str) and value.strip(): + try: + parsed = json.loads(value) + return parsed if isinstance(parsed, dict) else {} + except Exception: + return {} + return {} + + +def opportunity_id_from_task(task: dict) -> str: + meta = metadata_dict(task.get("metadata")) + return str(task.get("opportunity_id") or meta.get("opportunity_id") or "").strip() + + +def customer_display(task: dict) -> str: + return str(task.get("customer_name") or task.get("customer_email") or task.get("contact_id") or task.get("customer_id") or "Cliente").strip() + + +def action_recommendation_html(action_code: str) -> str: + action_code = str(action_code or "").upper() + hint = ACTION_HINTS.get(action_code, "Executar a ação indicada e atualizar o estado da tarefa.") + missing = ACTION_MISSING_HINTS.get(action_code, []) + missing_html = "" + if missing: + missing_html = "
Pedir se faltar: " + ", ".join(esc(x) for x in missing) + ".
" + return f""" +
+
{esc(hint)}
+ {missing_html} +
+ """ + + +def stage_progress_html(current_stage: str) -> str: + rank = OPPORTUNITY_STAGE_LABELS + current = str(current_stage or "NEW_LEAD") + stage_order = [x[0] for x in PIPELINE_STEPS] + current_index = 0 + for idx, stage in enumerate(stage_order): + if stage == current: + current_index = idx + break + if stage in {"NEW_LEAD", "INFO_SENT", "QUOTE_SENT", "WAITING_PAYMENT", "ORDER_PREPARATION", "SHIPPED", "WON"}: + pass + # aproxima estados intermédios para o passo visual mais próximo + stage_to_step = { + "NEW_LEAD": 0, + "INFO_REQUESTED": 0, + "INFO_SENT": 1, + "QUOTE_REQUESTED": 1, + "QUOTE_SENT": 2, + "PROFORMA_REQUESTED": 2, + "PROFORMA_SENT": 3, + "INVOICE_REQUESTED": 2, + "INVOICE_SENT": 3, + "WAITING_PAYMENT": 3, + "PAYMENT_CONFIRMED": 4, + "ODOO_ORDER_CREATED": 5, + "ORDER_PREPARATION": 5, + "IN_PRODUCTION": 6, + "READY_TO_SHIP": 7, + "INVOICED": 7, + "SHIPMENT_CREATED": 8, + "SHIPPED": 8, + "TRACKING_SENT": 8, + "DELIVERED": 9, + "WON": 9, + "LOST": 9, + "NO_INTEREST": 9, + "REVIEW": 0, + } + current_index = stage_to_step.get(current, 0) + items = "" + for idx, (stage, label) in enumerate(PIPELINE_STEPS): + css = "done" if idx < current_index else ("active" if idx == current_index else "") + items += f"
{esc(label)}
" + return f"
{items}
" + + +def opportunity_quick_actions_html(opportunity_id: str) -> str: + return "" + +def operation_status_badge(status: str) -> str: + value = str(status or "not_created").strip() + normalized = { + "0": "draft", + "1": "open", + "2": "completed", + "3": "closed", + "open": "open", + "completed": "completed", + "complete": "completed", + "closed": "closed", + "converted": "converted", + }.get(value.casefold(), value.casefold()) + cls = { + "not_created": "cf-chip-gray", "pending": "cf-chip-orange", "processing": "cf-chip-purple", "created": "cf-chip-blue", + "open": "cf-chip-blue", "draft": "cf-chip-gray", "completed": "cf-chip-green", "closed": "cf-chip-gray", + "converted": "cf-chip-green", "issued": "cf-chip-green", "confirmed": "cf-chip-green", "validated": "cf-chip-green", + "in_progress": "cf-chip-orange", "sent": "cf-chip-green", "delivered": "cf-chip-green", "failed": "cf-chip-red", + "blocked": "cf-chip-red", "dry_run": "cf-chip-gray", "ignored": "cf-chip-gray", "cancelled": "cf-chip-gray", + }.get(normalized, "cf-chip-gray") + label = { + "not_created": "Não criado", "pending": "Pendente", "processing": "A processar", "created": "Criado", "issued": "Emitida", + "open": "Aberto", "draft": "Rascunho", "completed": "Concluído", "closed": "Fechado", "converted": "Convertido", + "confirmed": "Confirmado", "validated": "Validado", "in_progress": "Em curso", "sent": "Processado", + "delivered": "Entregue", "failed": "Falhou", "blocked": "Bloqueado", "dry_run": "Dry-run", "ignored": "Ignorado", "cancelled": "Cancelado", + }.get(normalized, value) + return f'{esc(label)}' + + +def commercial_document_display_number(doc: dict | None, *, fallback: str = "sem número") -> str: + """Return a human commercial document number without exposing UUIDs. + + Imported Jasmin documents can temporarily have only a UUID/internal id. That + is useful for diagnostics, but confusing and unsafe as a commercial number. + """ + doc = doc or {} + parts = " ".join([ + str(doc.get("document_type") or "").strip(), + str(doc.get("serie") or "").strip(), + str(doc.get("series_number") or "").strip(), + ]).strip() + for value in (doc.get("document_number"), parts, doc.get("external_name"), doc.get("external_ref")): + text_value = str(value or "").strip() + if text_value and not is_uuid_text(text_value): + return text_value + return fallback + + +def should_hide_regressive_quotation_hint(opportunity: dict | None, snapshot: dict | None, next_action: dict | None) -> bool: + """Avoid suggesting quote creation in later commercial/fulfilment phases.""" + opportunity = opportunity or {} + snapshot = snapshot or {} + next_action = next_action or {} + action_key = str(next_action.get("action_key") or "").strip() + label = str(next_action.get("label") or "").strip().casefold() + if action_key != "jasmin_quotation" and "criar orçamento" not in label: + return False + + stage = str(opportunity.get("stage") or "").strip().upper() + late_stages = { + "QUOTE_SENT", "PROFORMA_SENT", "INVOICE_SENT", "WAITING_PAYMENT", "PAYMENT_CONFIRMED", + "ORDER_CONFIRMED", "IN_PRODUCTION", "READY_TO_SHIP", "SHIPMENT_CREATED", "SHIPPED", + "TRACKING_SENT", "DELIVERED", "WON", "LOST", "NO_INTEREST", + } + if stage in late_stages: + return True + + document_keys = {"jasmin_quotation", "jasmin_proforma", "jasmin_invoice"} + for card in snapshot.get("cards") or []: + key = str(card.get("key") or "").strip() + status = str(card.get("status") or "").strip().lower() + if key in document_keys and status not in {"", "not_created", "failed", "blocked", "cancelled", "ignored"}: + return True + return False + + +def operation_cockpit_html(opportunity_id: str, opportunity: dict, snapshot: dict) -> str: + plan = get_workflow_action_plan(opportunity_id) + + pending_tasks = int(opportunity.get("pending_task_count") or 0) + has_invoice_card = any( + str(card.get("key") or "") == "jasmin_invoice" + and str(card.get("status") or "").lower() not in {"", "not_created", "failed", "blocked", "cancelled", "ignored"} + for card in (snapshot.get("cards") or []) + ) + + next_action = plan.get("next_action") or {} + next_kind = str(next_action.get("kind") or "") + workflow_label = next_action.get("label") or "Sem ação" + workflow_reason = next_action.get("reason") or "" + physical_reason = plan.get("physical_reason") or "" + physical_next = plan.get("physical_next_action") or "" + + if pending_tasks > 0: + main_label = "Concluir tarefa pendente" + main_reason = "Existe uma tarefa ativa nesta oportunidade." + if should_hide_regressive_quotation_hint(opportunity, snapshot, next_action): + main_extra = "Depois: continuar a partir do documento/fase atual." + else: + main_extra = f"Depois: {workflow_label}" + action_html = 'Ver tarefas' + else: + main_label = workflow_label + main_reason = physical_reason or workflow_reason + main_extra = physical_next if physical_next and physical_next != main_reason else "" + + if should_hide_regressive_quotation_hint(opportunity, snapshot, next_action): + main_label = "Rever fluxo atual" + main_reason = "A oportunidade já tem documento/fase posterior; não criar novo orçamento neste processo." + main_extra = "Continua pela fatura, pagamento, envio ou histórico conforme o caso." + next_kind = "review" + + if next_kind == "operation" and next_action.get("action_key"): + action_key = str(next_action.get("action_key") or "") + action_html = ( + f'
' + f'
' + f'' + f'' + f'
' + f'
' + ) + elif next_kind == "sync_odoo": + action_html = ( + f'
' + f'' + f'
' + ) + elif next_kind == "wait": + action_html = 'Aguardar' + else: + action_html = 'Sem ação' + + def step_visual(status): + s = str(status or "").lower() + if s in {"confirmed", "issued", "created", "validated", "sent", "delivered", "done", "ready_to_ship"}: + return "bg-success text-white", "✓" + if s in {"in_progress", "pending", "running", "open", "in_production"}: + return "bg-warning text-dark", "…" + if s in {"failed", "blocked", "cancelled", "not_found"}: + return "bg-danger text-white", "!" + return "bg-light text-secondary border", "○" + + short_labels = { + "payment": "Pagamento", + "proforma": "Pró-forma", + "odoo_sale_order": "Venda", + "odoo_production": "Produção", + "physical_status": "Odoo", + "physical_validation": "Validado", + "jasmin_quotation": "Orçamento", + "jasmin_invoice": "Fatura", + "packlink_shipment": "Envio", + "tracking": "Tracking", + "delivery": "Entregue", + } + + steps_html = "" + for card in snapshot.get("cards", []): + badge_class, mark = step_visual(card.get("status")) + key = str(card.get("key") or "") + label = short_labels.get(key, card.get("label") or "") + url = str(card.get("external_url") or "").strip() + title = card.get("external_name") or card.get("status_label") or label + + link_open = "" + if url: + link_open = ( + '' + ) + + steps_html += ( + '
' + '
' + '
' + f'{mark}' + f'
{esc(label)}
' + f'{link_open}' + '
' + '
' + '
' + ) + + if not steps_html: + steps_html = '
Sem integrações registadas.
' + + if has_invoice_card and str(main_label).strip().casefold() == "criar orçamento jasmin": + main_label = "Acompanhar fatura" + main_reason = "Já existe fatura Jasmin associada; não criar novo orçamento neste processo." + main_extra = "Confirma pagamento, envio ou marca como histórico/concluído." + action_html = 'Rever processo' + + reason_html = "" + if main_reason: + reason_html += f'
{esc(main_reason)}
' + if main_extra: + reason_html += f'
{esc(main_extra)}
' + + return ( + '
' + '
' + '
' + '
' + '
Fluxo operacional
' + f'

{esc(main_label)}

' + f'{reason_html}' + '
' + f'
{action_html}
' + '
' + '
' + f'{steps_html}' + '
' + '
' + '
' + ) + +def task_priority_chip(task: dict) -> str: + priority = str(task.get("priority") or "").strip().lower() + if is_task_overdue(task) or priority == "alta": + return 'Alta' + if priority == "normal": + return 'Normal' + if priority == "baixa": + return 'Baixa' + route = str(task.get("route") or "") + if route in {"financeiro", "operacoes"}: + return 'Normal' + return 'Baixa' + +def pretty_json(value) -> str: + return json.dumps(value or {}, ensure_ascii=False, indent=2, default=str) + + +def status_badge(status: str) -> str: + value = str(status or "").strip() or "unknown" + label = { + "pending": "Pendente", + "done": "Concluída", + "skipped": "Ignorada", + "failed": "Falha", + "sent": "Enviada", + "ignored": "Ignorada", + }.get(value, value) + cls = { + "pending": "status-pending", + "done": "status-done", + "skipped": "status-skipped", + "failed": "status-failed", + "sent": "status-done", + "ignored": "status-skipped", + }.get(value, "status-skipped") + return f'{esc(label)}' + + +def route_badge(route: str) -> str: + value = str(route or "").strip() or "rever" + label = { + "vendas": "COMERCIAL", + "financeiro": "FINANCEIRO", + "suporte": "SUPORTE", + "operacoes": "LOGÍSTICA", + "spam": "SPAM", + "rever": "REVER", + }.get(value, value.upper()) + cls = { + "vendas": "route-vendas", + "financeiro": "route-financeiro", + "suporte": "route-suporte", + "operacoes": "route-operacoes", + "spam": "route-rever", + "rever": "route-rever", + }.get(value, "route-rever") + return f'{esc(label)}' + + +def task_sla_minutes(route: str) -> int: + return { + "suporte": 120, + "vendas": 240, + "financeiro": 480, + "operacoes": 1440, + "rever": 1440, + }.get(route or "", 1440) + + +def task_age_minutes(task: dict) -> int: + created_at = task.get("created_at") + + if not created_at: + return 0 + + if isinstance(created_at, str): + try: + created_at = datetime.fromisoformat(created_at.replace("Z", "+00:00")) + except Exception: + return 0 + + if created_at.tzinfo is None: + created_at = created_at.replace(tzinfo=timezone.utc) + + return max(0, int((datetime.now(timezone.utc) - created_at).total_seconds() // 60)) + + +def is_task_overdue(task: dict) -> bool: + if task.get("status") != "pending": + return False + + return task_age_minutes(task) > task_sla_minutes(task.get("route")) + + +def is_task_today(task: dict) -> bool: + created_at = task.get("created_at") + + if not created_at: + return False + + if isinstance(created_at, str): + try: + created_at = datetime.fromisoformat(created_at.replace("Z", "+00:00")) + except Exception: + return False + + if created_at.tzinfo is None: + created_at = created_at.replace(tzinfo=timezone.utc) + + now = datetime.now(timezone.utc) + return created_at.date() == now.date() + + +def sla_badge_html(task: dict) -> str: + if task.get("status") != "pending": + return "" + + age = task_age_minutes(task) + sla = task_sla_minutes(task.get("route")) + + if age > sla: + overdue = age - sla + if overdue >= 60: + label = f"Atrasada {overdue // 60}h" + else: + label = f"Atrasada {overdue}m" + return f'{esc(label)}' + + remaining = sla - age + if remaining >= 60: + label = f"SLA {remaining // 60}h" + else: + label = f"SLA {remaining}m" + + return f'{esc(label)}' + + + +# ADMIN_UI_V451_CSS moved to app.admin_ui.styles in v4.7. + + +# Layout, navigation and KPI card helpers moved to app.admin_ui in v4.7. + +def is_htmx(request: Request) -> bool: + return str(request.headers.get("HX-Request") or "").lower() == "true" + + + + +@router.get("/ui.css") +async def admin_ui_css(): + return Response(ADMIN_UI_V451_CSS, media_type="text/css") + + + + + + + + + + + + + + + + + + + + + + + + +def opportunity_stage_badge(stage: str) -> str: + classes = { + "NEW_LEAD": "cf-chip-blue", + "INFO_REQUESTED": "cf-chip-blue", + "INFO_SENT": "cf-chip-green", + "QUOTE_REQUESTED": "cf-chip-orange", + "QUOTE_SENT": "cf-chip-green", + "PROFORMA_REQUESTED": "cf-chip-orange", + "PROFORMA_SENT": "cf-chip-green", + "INVOICE_REQUESTED": "cf-chip-orange", + "INVOICE_SENT": "cf-chip-green", + "WAITING_PAYMENT": "cf-chip-orange", + "PAYMENT_CONFIRMED": "cf-chip-green", + "ORDER_PREPARATION": "cf-chip-purple", + "SHIPPED": "cf-chip-purple", + "WON": "cf-chip-green", + "LOST": "cf-chip-red", + "NO_INTEREST": "cf-chip-gray", + "REVIEW": "cf-chip-gray", + } + return f'{esc(stage_label(stage))}' + + +def _opportunity_board_column_for_stage(stage: str) -> str: + stage = str(stage or "") + for key, _label, stages in OPPORTUNITY_BOARD_COLUMNS: + if stage in stages: + return key + return "requests" + +def money_html(value, currency: str = "€") -> str: + try: + number = float(value or 0) + except Exception: + number = 0.0 + formatted = f"{number:,.2f}".replace(",", "X").replace(".", ",").replace("X", ".") + return f"{formatted} {esc(currency)}" + + +def product_status_badge(active) -> str: + if active: + return 'Ativo' + return 'Inativo' + + +def item_status_label(status: str) -> str: + labels = { + "INTERESTED": "Em análise", + "QUOTED": "Orçamentado", + "ACCEPTED": "Aceite", + "REJECTED": "Rejeitado", + "CANCELLED": "Cancelado", + "UNAVAILABLE": "Indisponível", + } + return labels.get(str(status or "").upper(), str(status or "—")) + + +def item_status_badge(status: str) -> str: + status = str(status or "").upper() + cls = { + "INTERESTED": "cf-chip-orange", + "QUOTED": "cf-chip-blue", + "ACCEPTED": "cf-chip-green", + "REJECTED": "cf-chip-red", + "CANCELLED": "cf-chip-gray", + "UNAVAILABLE": "cf-chip-gray", + }.get(status, "cf-chip-gray") + return f'{esc(item_status_label(status))}' + + +def product_form_html(product: Optional[dict] = None, *, action: str = "/products", submit_label: str = "Guardar produto") -> str: + product = product or {} + checked = "checked" if product.get("active", True) else "" + return f''' +
+
+ + +
+
+ + +
Usado para criar orçamentos/faturas Jasmin. Pode ser diferente do SKU Odoo.
+
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ +
+
+ + Cancelar +
+
+ ''' + + + + +def _outbox_items_for_opportunity(opportunity_id: str, *, target_system: str | None = "jasmin", limit: int = 30) -> list[dict]: + try: + items = list_outbox(target_system=target_system, limit=300) + except Exception: + return [] + filtered = [] + for item in items: + payload = item.get("payload") or {} + if isinstance(payload, str): + try: + payload = json.loads(payload) + except Exception: + payload = {} + if str(payload.get("opportunity_id") or "") == str(opportunity_id): + item = dict(item) + item["payload"] = payload + filtered.append(item) + if len(filtered) >= limit: + break + return filtered + + +def opportunity_integrations_panel_html(opportunity_id: str) -> str: + items = _outbox_items_for_opportunity(opportunity_id, target_system=None, limit=16) + counts = {"pending": 0, "failed": 0, "blocked": 0, "dry_run": 0} + for item in items: + status = str(item.get("status") or "pending") + if status in counts: + counts[status] += 1 + + rows = "" + for item in items[:8]: + status = str(item.get("status") or "pending") + row_cls = f"cf-outbox-row-{status}" + err = compact_text(item.get("last_error") or "", 160) + actions = "" + if status in {"failed", "blocked", "dry_run", "ignored", "cancelled"}: + actions = f''' +
+ + +
+ ''' + elif status == "pending": + actions = 'A aguardar timer' + rows += f''' + + {esc(item.get('target_system'))}
{esc(item.get('action_type'))}
+ {operation_status_badge(status)} + {esc(fmt_dt(item.get('updated_at') or item.get('created_at')))} + {esc(err)} + {actions} + + ''' + + if not rows: + rows = 'Sem ações de integração para esta oportunidade.' + + return f''' +
+
+
+
+

Integrações da oportunidade

+
Estado operacional das ações Jasmin, Packlink, Chatwoot e Mautic ligadas a esta oportunidade.
+
+ Ver outbox +
+
+
+
Pendentes{counts['pending']}
+
Falhadas{counts['failed']}
+
Bloqueadas{counts['blocked']}
+
Dry-run{counts['dry_run']}
+
+
+
+ + + {rows} +
Sistema / AçãoEstadoAtualizadoÚltimo erroAção
+
+
+
+ ''' + + +def opportunity_outbox_panel_html(opportunity_id: str, *, target_system: str = "jasmin") -> str: + items = _outbox_items_for_opportunity(opportunity_id, target_system=target_system, limit=12) + if not items: + return "" + rows = "" + for item in items: + retry = "" + if item.get("status") in {"failed", "pending"}: + retry = f''' +
+ + +
+ ''' + err = esc(item.get("last_error") or "") + if err and len(err) > 180: + err = err[:180] + "…" + rows += f''' + + {esc(item.get('action_type'))}
{esc(fmt_dt(item.get('created_at')))}
+ {operation_status_badge(str(item.get('status') or 'pending'))} + {err} + {retry} + + ''' + return f''' +
+
+
Ações Jasmin / outbox
Processamento automático ou manual das ações pedidas pelos botões.
+ Ver outbox +
+
+ {rows}
AçãoEstadoErro
+
+
+ ''' + + +def jasmin_documents_html(opportunity_id: str, *, notice: str = "", error_notice: str = "") -> str: + try: + from app.commercial_service import list_commercial_documents + docs = list_commercial_documents(opportunity_id=opportunity_id, limit=20) + except Exception as exc: + docs = [] + error = str(exc) + else: + error = "" + + try: + from app.jasmin_backfill_service import find_jasmin_document_candidates_for_opportunity + jasmin_candidates = find_jasmin_document_candidates_for_opportunity(opportunity_id, limit=8) + except Exception as exc: + jasmin_candidates = [] + if not error: + error = f"Erro ao procurar documentos Jasmin existentes: {exc}" + + linked_tax_id = "" + try: + from app.commercial_service import get_customer_for_opportunity, normalize_tax_id + linked_customer = get_customer_for_opportunity(opportunity_id) + linked_tax_id = normalize_tax_id((linked_customer or {}).get("tax_id")) + except Exception: + linked_tax_id = "" + + rows = "" + kind_labels = {"quotation": "Orçamento", "proforma": "Pró-forma", "invoice": "Fatura"} + for doc in docs: + number = commercial_document_display_number(doc, fallback="número por atualizar") + amount = doc.get("total_amount") if doc.get("total_amount") is not None else doc.get("amount") + kind = kind_labels.get(str(doc.get("document_kind") or ""), doc.get("document_kind") or "Documento") + doc_id = str(doc.get("id") or "") + actions = f""" +
+
+ + +
+ PDF +
+ """ + role_label = { + "current": "Atual", + "accepted": "Aceite", + "historical": "Histórico", + "cancelled": "Cancelado", + "related": "Relacionado", + }.get(str(doc.get("role") or "current"), str(doc.get("role") or "current")) + role_class = "text-bg-primary" if str(doc.get("role") or "current") in {"current", "accepted"} and doc.get("is_primary") else "text-bg-light" + external_id_html = "" + if not doc.get("document_number") and doc.get("external_id") and not is_uuid_text(doc.get("external_id")): + external_id_html = f"
ref. externa {esc(doc.get('external_id') or '')}
" + elif not doc.get("document_number") and doc.get("external_id"): + external_id_html = "
ID técnico oculto; usar Atualizar nº.
" + rows += ( + "" + f"{esc(kind)}
v{esc(doc.get('version_number') or '—')}
{esc(role_label)}" + f"{esc(number)}{external_id_html}" + f"{operation_status_badge(str(doc.get('status') or 'created'))}" + f"{money_html(amount or 0)}" + f"{esc(fmt_dt(doc.get('created_at')))}" + f"{actions}" + "" + ) + if not rows: + rows = 'Ainda sem documentos Jasmin nesta oportunidade.' + + current_jasmin_docs_exist = bool(docs) + invoice_source_exists = any( + str(doc.get("document_kind") or "") in {"quotation", "proforma"} + and str(doc.get("status") or "").lower() not in {"cancelled", "failed"} + and str(doc.get("role") or "current") in {"current", "accepted", "related"} + for doc in docs + ) + candidate_rows = "" + ignored_rows = "" + valid_candidate_count = 0 + ignored_candidate_count = 0 + hidden_other_customer_count = 0 + for item in jasmin_candidates: + totals = item.get("totals") if isinstance(item.get("totals"), dict) else {} + total_amount = totals.get("total_amount") or item.get("amount") or 0 + doc_number = item.get("document_number") or (item.get("external_id") if not is_uuid_text(item.get("external_id")) else None) or "número por atualizar" + match_reason = item.get("match_reason") or "match" + match_score = item.get("match_score") or "" + item_id = str(item.get("id") or "") + is_valid = bool(item.get("is_valid_candidate")) + candidate_tax = "" + try: + from app.commercial_service import normalize_tax_id + candidate_tax = normalize_tax_id(item.get("customer_tax_id")) + except Exception: + candidate_tax = str(item.get("customer_tax_id") or "").strip() + other_customer = bool(linked_tax_id and candidate_tax and candidate_tax != linked_tax_id) + tax_conflict = bool(other_customer) + invalid_reason = item.get("invalid_reason") or "" + if tax_conflict: + # Segurança operacional: um documento Jasmin de NIF diferente nunca deve + # aparecer como candidato acionável. Fica apenas em auditoria/revisão. + is_valid = False + invalid_reason = invalid_reason or "NIF divergente do cliente fiscal validado; rever manualmente." + if is_valid: + valid_candidate_count += 1 + else: + ignored_candidate_count += 1 + status_label = item.get("jasmin_status_label") or "—" + status_badge_html = ( + 'NIF divergente' + if tax_conflict + else ( + 'Aberto/válido' + if is_valid + else 'Ignorado' + ) + ) + if is_valid: + if current_jasmin_docs_exist: + action_html = f''' +
+
+ +
+ Já existe documento atual. A associação direta fica bloqueada para evitar duplicados. +
+ ''' + else: + action_html = f''' +
+
+ +
+
+ ''' + else: + action_html = ( + '' + if tax_conflict + else '' + ) + candidate_customer_meta = ( + f'
NIF {esc(item.get("customer_tax_id") or "—")}
' + if is_valid + else ( + '
NIF divergente oculto em auditoria
' + if tax_conflict + else '
NIF oculto em auditoria
' + ) + ) + row_html = f''' + + {esc(doc_number)}
{esc(item.get('external_type') or 'jasmin')}
{esc(fmt_dt(item.get('document_date') or item.get('updated_at')))}
+ {money_html(total_amount or 0)}
{esc(item.get('currency') or 'EUR')}
+ {esc(item.get('customer_name') or '—')}{candidate_customer_meta} + {status_badge_html}
Jasmin: {esc(status_label)} {esc(item.get('jasmin_status_code') or '')}
{esc(invalid_reason)}
+ {esc(match_reason)} {esc(match_score)}
{esc(item.get('line_count') or 0)} linha(s)
+ {action_html} + + ''' + if is_valid: + candidate_rows += row_html + else: + ignored_rows += row_html + + candidates_html = "" + if candidate_rows or ignored_rows or hidden_other_customer_count: + if candidate_rows: + candidates_html += f''' +
+ Candidatos Jasmin acionáveis encontrados.
+ Valida antes de substituir ou associar, especialmente em processos antigos/reconstruídos. +
+
+ + + {candidate_rows} +
Documento encontradoValorCliente JasminValidaçãoMatchAção
+
+ ''' + else: + candidates_html += ''' +
+ Nenhum orçamento/pró-forma aberto elegível encontrado.
+ Documentos antigos, fechados ou de outro cliente não são apresentados como ação principal. +
+ ''' + if ignored_rows: + candidates_html += f''' +
+ Ver documentos ignorados / auditoria +
+ + + {ignored_rows} +
DocumentoValorCliente JasminValidaçãoMatchAção
+
+
+ ''' + if hidden_other_customer_count: + candidates_html += f'
{hidden_other_customer_count} documento(s) ignorado(s) de outro NIF ocultados da lista principal.
' + + notice_html = f'
{esc(notice)}
' if notice else '' + error_notice_html = f'
Não foi possível pedir a ação.
{esc(error_notice).replace(chr(10), "
")}
' if error_notice else '' + error_html = f'
{esc(error)}
' if error else '' + outbox_html = opportunity_outbox_panel_html(opportunity_id, target_system="jasmin") + refreshed_at = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC") + if invoice_source_exists: + convert_invoice_button_html = ( + f'
' + '' + '
' + ) + else: + convert_invoice_button_html = '' + + if current_jasmin_docs_exist: + create_quotation_button_html = f''' +
+ +
+ ''' + else: + create_quotation_button_html = f''' +
+ +
+ ''' + return f''' +
+
+
+
+

Documentos Jasmin

+
Antes de criar novo orçamento, valida candidatos Jasmin abertos/mais recentes para evitar duplicados.
+
Última atualização: {esc(refreshed_at)}. Atualização manual para evitar reconstrução automática da janela.
+
+
+ {create_quotation_button_html} + {convert_invoice_button_html} +
+ +
+
+ +
+ +
+
+
{notice_html}{error_notice_html}{error_html}
+ {candidates_html} +
+ + + {rows} +
TipoNúmero/IDEstadoValorCriadoAções
+
+ {outbox_html} +
+
+ ''' + +def opportunity_items_table_html(opportunity_id: str, items: list[dict]) -> str: + rows = "" + historical_rows = "" + for item in items: + row_html = f''' + + {esc(item.get('product_name') or 'Produto')}
SKU/Odoo {esc(item.get('sku') or '—')}
Jasmin {esc(item.get('jasmin_sales_item') or '—')}
+ {esc(item.get('quantity') or '1')} + {money_html(item.get('unit_price'))} + {money_html(item.get('discount_amount'))} + {money_html(item.get('total_price'))} + {item_status_badge(item.get('status'))} + +
+ +
+ + + ''' + if str(item.get('status') or '').upper() in {"DELIVERED", "HISTORICAL"}: + historical_rows += row_html + else: + rows += row_html + if not rows: + rows = 'Sem produtos atuais nesta oportunidade.' + historical_html = "" + if historical_rows: + historical_html = f''' +
+ Ver linhas históricas / entregues +
+ + + {historical_rows} +
ProdutoQtd.PreçoDesc.TotalEstado
+
+
+ ''' + return f''' +
+ + + {rows} +
ProdutoQtd.PreçoDesc.TotalEstado
+
+ {historical_html} + ''' + + +def opportunity_add_item_form_html(opportunity_id: str, products: list[dict]) -> str: + options = '' + for product in products: + product_id = esc(product.get("id")) + sku = esc(product.get("sku") or "") + jasmin = esc(product.get("jasmin_sales_item") or "") + name = esc(product.get("name") or "") + options += f'' + return f''' +
+
+ + +
+
+ + +
+
+ + +
+
+ +
+
+ ''' + + +def opportunity_products_panel_html(opportunity_id: str, *, notice: str = "", error_notice: str = "") -> str: + try: + items = list_opportunity_items(opportunity_id) + active_products = list_products(active="true", limit=300) + except Exception as exc: + return f'
Erro ao carregar produtos: {esc(exc)}
' + total = sum(float(item.get("total_price") or 0) for item in items if str(item.get("status") or "").upper() not in {"REJECTED", "CANCELLED", "DELIVERED", "HISTORICAL"}) + notice_html = f'
{esc(notice)}
' if notice else '' + error_html = f'
{esc(error_notice)}
' if error_notice else '' + missing = [item for item in items if str(item.get("status") or "").upper() not in {"REJECTED", "CANCELLED"} and not item.get("jasmin_sales_item")] + validation_html = "" + if missing: + lis = "".join(f"
  • {esc(i.get('product_name') or i.get('sku') or 'Produto')} sem Artigo Jasmin.
    • " for i in missing) + validation_html = f'
    Atenção: estes produtos bloqueiam o orçamento Jasmin:
    ' + return f''' +
    +
    +
    +

    Produtos

    Linhas comerciais da oportunidade.
    + Total linhas atuais {money_html(total)} +
    +

    Adicionar produto

    {notice_html}{error_html}{opportunity_add_item_form_html(opportunity_id, active_products)}
    + {opportunity_items_table_html(opportunity_id, items)} +
    {validation_html}
    +
    +
    + ''' + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/app/admin_queries.py b/app/admin_queries.py new file mode 100644 index 0000000..fde33c8 --- /dev/null +++ b/app/admin_queries.py @@ -0,0 +1,55 @@ +from typing import Any, Dict, List + +from sqlalchemy import text + +from app.db import engine + + +def list_business_events(limit: int = 100) -> List[Dict[str, Any]]: + sql = text(""" + SELECT + id::text, + event_type, + task_id::text, + action_run_id::text, + conversation_id, + contact_id, + payload, + created_by, + created_at + FROM business_events + ORDER BY created_at DESC + LIMIT :limit + """) + + with engine.begin() as conn: + rows = conn.execute(sql, {"limit": limit}).mappings().all() + + return [dict(row) for row in rows] + + +def list_action_runs(limit: int = 100) -> List[Dict[str, Any]]: + sql = text(""" + SELECT + id::text, + conversation_id, + contact_id, + source_system, + model, + provider, + decision_source, + action_decision, + action_result, + total_tokens, + cost, + needs_review, + created_at + FROM action_runs + ORDER BY created_at DESC + LIMIT :limit + """) + + with engine.begin() as conn: + rows = conn.execute(sql, {"limit": limit}).mappings().all() + + return [dict(row) for row in rows] diff --git a/app/admin_ui/__init__.py b/app/admin_ui/__init__.py new file mode 100644 index 0000000..f061d51 --- /dev/null +++ b/app/admin_ui/__init__.py @@ -0,0 +1,5 @@ +"""Admin UI package for the incremental ClientFlow v4 refactor. + +The legacy `app.admin_dashboard` module still owns the routes for now. New UI +partials and pages should be moved here gradually without changing public URLs. +""" diff --git a/app/admin_ui/components.py b/app/admin_ui/components.py new file mode 100644 index 0000000..bbf90f3 --- /dev/null +++ b/app/admin_ui/components.py @@ -0,0 +1,116 @@ +"""Reusable HTML helpers for the admin UI refactor. + +Keep these helpers dependency-light so pages can migrate from +`admin_dashboard.py` gradually. +""" +from __future__ import annotations + +import html +from datetime import datetime +from decimal import Decimal +from typing import Any + + +def esc(value: Any) -> str: + return html.escape("" if value is None else str(value)) + + +def money(value: Any, currency: str = "EUR") -> str: + try: + amount = Decimal(str(value or "0")) + return f"{amount:,.2f} {currency}".replace(",", "X").replace(".", ",").replace("X", ".") + except Exception: + return f"{esc(value)} {esc(currency)}" + + +def fmt_dt(value: Any) -> str: + if not value: + return "—" + if isinstance(value, datetime): + return value.strftime("%Y-%m-%d %H:%M") + return esc(value) + + +def status_chip(status: str | None) -> str: + status = (status or "unknown").lower() + css = { + "pending": "cf-chip-orange", + "processing": "cf-chip-purple", + "sent": "cf-chip-green", + "failed": "cf-chip-red", + "blocked": "cf-chip-red", + "dry_run": "cf-chip-gray", + "ignored": "cf-chip-gray", + "cancelled": "cf-chip-gray", + "created": "cf-chip-blue", + "issued": "cf-chip-green", + "converted": "cf-chip-green", + }.get(status, "cf-chip-gray") + return f'{esc(status)}' + + +def kpi_icon_for(label: str) -> str: + text = (label or "").lower() + if "oportun" in text or "pipeline" in text: + return "bi-funnel" + if "valor" in text or "pagamento" in text or "finance" in text or "fatura" in text: + return "bi-currency-euro" + if "document" in text or "orçamento" in text or "orcamento" in text: + return "bi-file-earmark-text" + if "task" in text or "tarefa" in text or "pendente" in text: + return "bi-list-check" + if "email" in text or "comunica" in text or ("cliente" in text and "sem" in text): + return "bi-envelope" + if "erro" in text or "invál" in text or "inval" in text or "bloque" in text: + return "bi-exclamation-triangle" + if "envio" in text or "packlink" in text or "opera" in text or "prepar" in text: + return "bi-truck" + if "cliente" in text: + return "bi-person-vcard" + if "produto" in text or "artigo" in text or "categoria" in text or "jasmin" in text: + return "bi-box-seam" + if "integra" in text or "sistema" in text or "ambiente" in text or "estado" in text: + return "bi-puzzle" + if "retry" in text or "retries" in text: + return "bi-arrow-clockwise" + return "bi-grid" + + +def kpi_tone_for(label: str, hint: str = "") -> str: + text = f"{label or ''} {hint or ''}".lower() + if "erro" in text or "falh" in text or "invál" in text or "inval" in text: + return "cf-kpi-tone-red" + if "pagamento" in text or "valor" in text or "finance" in text or "€" in text: + return "cf-kpi-tone-green" + if "envio" in text or "packlink" in text or "pendente" in text or "aguarda" in text or "incompleto" in text or "bloque" in text: + return "cf-kpi-tone-orange" + if "cliente" in text or "comunica" in text or "email" in text: + return "cf-kpi-tone-purple" + return "" + + +def kpi_card(label: str, value: object, href: str | None = None, hint: str = "", icon: str | None = None, tone: str | None = None) -> str: + tag = "a" if href else "div" + href_attr = f' href="{esc(href)}"' if href else "" + reset_class = " text-reset text-decoration-none" if href else "" + icon_class = icon or kpi_icon_for(label) + tone_class = tone or kpi_tone_for(label, hint) + return f""" + <{tag} class="cf-card cf-kpi{reset_class} {esc(tone_class)}"{href_attr}> + + {esc(label)} + {esc(value)} + {esc(hint)} + + """ + + +def alert_box(title: str, detail: str, tone: str = "warning") -> str: + css = { + "danger": "alert-danger", + "warning": "alert-warning", + "info": "alert-info", + "success": "alert-success", + }.get(tone, "alert-warning") + return f'
    {esc(title)}
    {esc(detail)}
    ' + diff --git a/app/admin_ui/guidance.py b/app/admin_ui/guidance.py new file mode 100644 index 0000000..1791d76 --- /dev/null +++ b/app/admin_ui/guidance.py @@ -0,0 +1,353 @@ +"""Small operator-facing UI helpers for guided operations. + +v4.8.2 intentionally keeps business rules unchanged. Prontidão is shown as UI guidance only. These helpers only turn +existing opportunity/task/outbox data into clearer labels, blockers and +checklists for the operator. +""" +from __future__ import annotations + +import html +from typing import Any, Iterable + + +def esc(value: Any) -> str: + return html.escape("" if value is None else str(value)) + + +def _present(value: Any) -> bool: + return bool(str(value or "").strip()) + + +def _field(data: dict[str, Any] | None, *keys: str) -> str: + data = data or {} + for key in keys: + value = data.get(key) + if _present(value): + return str(value).strip() + return "" + + +DOCUMENT_ACTION_CODES = { + "SEND_PROFORMA", + "SEND_INVOICE", +} + +# Pagamento e envio têm checklists próprias. Não devem bloquear uma oportunidade +# antiga/reconstruída só porque faltam campos fiscais no ClientFlow depois de o +# documento oficial já ter sido emitido. +FULFILMENT_ACTION_CODES = { + "CONFIRM_PAYMENT_AND_PREPARE_SHIPMENT", + "PREPARE_ORDER", + "CREATE_SHIPMENT", +} + +DOCUMENT_STAGES = { + "PROFORMA_REQUESTED", + "PROFORMA_SENT", + "INVOICE_REQUESTED", + "INVOICE_SENT", + "WAITING_PAYMENT", + "PAYMENT_CONFIRMED", + "ORDER_PREPARATION", + "READY_TO_SHIP", + "INVOICED", + "SHIPMENT_CREATED", + "SHIPPED", + "TRACKING_SENT", + "DELIVERED", +} + +SHIPMENT_ACTION_CODES = {"CONFIRM_PAYMENT_AND_PREPARE_SHIPMENT", "PREPARE_ORDER", "CREATE_SHIPMENT"} +SHIPMENT_STAGES = {"PAYMENT_CONFIRMED", "ORDER_PREPARATION", "READY_TO_SHIP", "SHIPMENT_CREATED", "SHIPPED", "TRACKING_SENT", "DELIVERED"} + + +def normalized_action_code(value: Any) -> str: + return str(value or "").strip().upper() + + +def action_requires_fiscal_customer(action_code: Any) -> bool: + return normalized_action_code(action_code) in DOCUMENT_ACTION_CODES + + +def action_requires_shipment_readiness(action_code: Any) -> bool: + return normalized_action_code(action_code) in SHIPMENT_ACTION_CODES + + +def stage_requires_fiscal_customer(stage: Any) -> bool: + return str(stage or "").strip().upper() in DOCUMENT_STAGES + + +def stage_requires_shipment_readiness(stage: Any) -> bool: + return str(stage or "").strip().upper() in SHIPMENT_STAGES + + +def item_requires_fiscal_customer(item: dict[str, Any] | None) -> bool: + item = item or {} + return action_requires_fiscal_customer(item.get("action_code")) + + +def item_requires_shipment_readiness(item: dict[str, Any] | None) -> bool: + item = item or {} + return action_requires_shipment_readiness(item.get("action_code")) + +def fiscal_customer_missing_fields(customer: dict[str, Any] | None) -> list[str]: + """Minimum fields needed before fiscal documents should be issued.""" + if not customer: + return ["cliente fiscal associado"] + missing: list[str] = [] + if not _field(customer, "name", "linked_customer_name"): + missing.append("nome fiscal") + if not _field(customer, "tax_id", "linked_customer_tax_id"): + missing.append("NIF") + if not _field(customer, "email", "linked_customer_email", "customer_email"): + missing.append("email de faturação") + if not _field(customer, "street_name"): + missing.append("morada fiscal") + if not _field(customer, "postal_zone"): + missing.append("código postal") + if not _field(customer, "city_name"): + missing.append("localidade") + return missing + + +def shipment_missing_fields(customer: dict[str, Any] | None, opportunity: dict[str, Any] | None = None) -> list[str]: + missing: list[str] = [] + if not customer: + return ["cliente fiscal associado", "morada de entrega", "telefone"] + if not _field(customer, "street_name"): + missing.append("morada de entrega") + if not _field(customer, "postal_zone"): + missing.append("código postal de entrega") + if not _field(customer, "city_name"): + missing.append("localidade de entrega") + if not (_field(customer, "phone") or _field(opportunity or {}, "customer_phone")): + missing.append("telefone") + return missing + + +def opportunity_context_customer(opportunity: dict[str, Any], linked_customer: dict[str, Any] | None = None) -> dict[str, Any] | None: + if linked_customer: + return linked_customer + if opportunity.get("linked_customer_id") or opportunity.get("linked_customer_name"): + return { + "id": opportunity.get("linked_customer_id"), + "name": opportunity.get("linked_customer_name"), + "email": opportunity.get("linked_customer_email"), + "tax_id": opportunity.get("linked_customer_tax_id"), + "street_name": opportunity.get("linked_customer_street_name"), + "postal_zone": opportunity.get("linked_customer_postal_zone"), + "city_name": opportunity.get("linked_customer_city_name"), + "phone": opportunity.get("linked_customer_phone"), + } + return None + + +def opportunity_blockers( + opportunity: dict[str, Any], + linked_customer: dict[str, Any] | None = None, + *, + action_code: Any = None, +) -> list[str]: + """Return only blockers that matter for the current journey stage. + + Missing fiscal data is not automatically a blocker at the first contact. It + becomes a current blocker only when the next action/stage needs fiscal + documents, payment or fulfilment. + """ + customer = opportunity_context_customer(opportunity, linked_customer) + blockers: list[str] = [] + linking_status = str(opportunity.get("opportunity_linking_status") or opportunity.get("linking_status") or "").lower() + if linking_status == "ambiguous": + blockers.append("Associação de oportunidade por confirmar") + + needs_fiscal = action_requires_fiscal_customer(action_code) if action_code else stage_requires_fiscal_customer(opportunity.get("stage")) + if needs_fiscal: + if not customer: + blockers.append("Cliente fiscal por associar") + else: + for item in fiscal_customer_missing_fields(customer): + blockers.append(f"Cliente fiscal sem {item}") + + needs_product = needs_fiscal or str(opportunity.get("stage") or "").upper() in {"QUOTE_REQUESTED", "QUOTE_SENT"} + if needs_product and not _field(opportunity, "product_interest", "title"): + blockers.append("Produto/interesse por definir") + return blockers + + +def work_item_fiscal_customer(item: dict[str, Any]) -> dict[str, Any] | None: + """Return the fiscal customer carried by an Operations item, if any. + + The work queue must not infer a fiscal customer from Chatwoot contact_id. It + only uses the opportunity/customer links already resolved by the backend. + """ + item = item or {} + if not _field(item, "fiscal_customer_name"): + return None + return { + "name": item.get("fiscal_customer_name"), + "email": item.get("fiscal_customer_email"), + "tax_id": item.get("fiscal_customer_tax_id"), + "street_name": item.get("fiscal_customer_street_name"), + "postal_zone": item.get("fiscal_customer_postal_zone"), + "city_name": item.get("fiscal_customer_city_name"), + } + + +def work_item_blockers(item: dict[str, Any]) -> list[str]: + """Compact blockers for Operations cards, based on the next action.""" + blockers: list[str] = [] + linking_status = str(item.get("opportunity_linking_status") or "").lower() + if linking_status == "ambiguous": + blockers.append("Associação de oportunidade por confirmar") + + if item_requires_fiscal_customer(item): + customer = work_item_fiscal_customer(item) + if not customer: + blockers.append("Cliente fiscal por associar") + else: + for missing in fiscal_customer_missing_fields(customer): + blockers.append(f"Cliente fiscal sem {missing}") + return blockers + +def blocker_alert_html(blockers: Iterable[str], *, empty_text: str = "Sem bloqueios críticos visíveis.") -> str: + blockers = [str(item) for item in blockers if str(item or "").strip()] + if not blockers: + return f'
    {esc(empty_text)}
    ' + items = "".join(f"
  • {esc(item)}
    • " for item in blockers) + return f'
    Bloqueios atuais
    ' + + +def readiness_checklist_html( + *, + title: str, + missing: Iterable[str], + ok_text: str = "Pronto para avançar.", + blocked_text: str = "Ação bloqueada até corrigir os dados em falta.", +) -> str: + missing_items = [str(item) for item in missing if str(item or "").strip()] + if missing_items: + pills = "".join(f'⚠ {esc(item)}' for item in missing_items) + state = f'
    {esc(blocked_text)}
    ' + else: + pills = '✓ Dados mínimos completos' + state = f'
    {esc(ok_text)}
    ' + return f''' +
    +
    +

    {esc(title)}

    +
    {pills}
    + {state} +
    +
    + ''' + + +def fiscal_contact_panel_html( + *, + fiscal_customer: dict[str, Any] | None, + contact_name: Any = "", + contact_email: Any = "", + contact_phone: Any = "", + conversation_id: Any = "", + contact_id: Any = "", + customer_href: str = "", +) -> str: + customer = fiscal_customer or {} + customer_name = _field(customer, "name", "linked_customer_name") or "Cliente fiscal por associar" + tax_id = _field(customer, "tax_id", "linked_customer_tax_id") or "—" + email = _field(customer, "email", "linked_customer_email") or "—" + address_bits = [ + _field(customer, "street_name"), + " ".join(bit for bit in [_field(customer, "postal_zone"), _field(customer, "city_name")] if bit), + ] + address = " · ".join(bit for bit in address_bits if bit) or "—" + contact_name = str(contact_name or "Contacto por confirmar") + contact_email = str(contact_email or "—") + contact_phone = str(contact_phone or "") + conversation = str(conversation_id or "") + contact_ref = str(contact_id or "") + customer_button = f'Ver cliente fiscal' if customer_href else "" + return f''' +
    +
    +

    Cliente fiscal e contacto Chatwoot

    +
    +
    +
    Cliente fiscal
    + {esc(customer_name)} +
    NIF {esc(tax_id)}
    +
    {esc(email)}
    +
    {esc(address)}
    + {customer_button} +
    +
    +
    Contacto Chatwoot
    + {esc(contact_name)} +
    {esc(contact_email)}
    +
    {esc(contact_phone)}
    +
    {esc('Conversa #' + conversation if conversation else 'Sem conversa ligada')}
    +
    {esc('Contacto #' + contact_ref if contact_ref else '')}
    +
    +
    +
    +
    + ''' + + +def fiscal_contact_inline_html( + *, + fiscal_name: Any = "", + contact_ref: Any = "", + conversation_id: Any = "", + show_fiscal: bool = True, + fiscal_required: bool = False, +) -> str: + contact = str(contact_ref or "").strip() + conversation = str(conversation_id or "").strip() + contact_bits = [] + if contact: + contact_bits.append(f"contacto {contact}") + if conversation: + contact_bits.append(f"conversa #{conversation}") + contact_line = " · ".join(contact_bits) or "contacto por confirmar" + fiscal_value = str(fiscal_name or "").strip() + + if not show_fiscal and not fiscal_required: + return f''' +
    +
    Contacto Chatwoot{esc(contact_line)}
    +
    + ''' + + fiscal_label = fiscal_value or "Por associar" + return f''' +
    +
    Cliente fiscal{esc(fiscal_label)}
    +
    Contacto Chatwoot{esc(contact_line)}
    +
    + ''' + +def outbox_operator_message(item: dict[str, Any]) -> dict[str, str]: + target = str(item.get("target_system") or "integração").strip() + action = str(item.get("action_type") or "ação").strip() + error = str(item.get("last_error") or "").strip() + lower = error.casefold() + title = f"Falha em {target}.{action}" if error else f"Ação {target}.{action}" + probable = "Ver o detalhe técnico do erro antes de reprocessar." + fix_label = "Ver detalhe" + if "tax" in lower or "nif" in lower or "vat" in lower or "fiscal" in lower: + probable = "Cliente fiscal sem NIF válido ou dados fiscais incompletos." + fix_label = "Corrigir cliente" + elif "address" in lower or "morada" in lower or "postal" in lower or "city" in lower: + probable = "Morada fiscal ou morada de entrega incompleta." + fix_label = "Corrigir morada" + elif "product" in lower or "item" in lower or "sales_item" in lower or "artigo" in lower: + probable = "Produto sem código externo/Jasmin ou artigo inválido." + fix_label = "Corrigir produto" + elif "payment" in lower or "pagamento" in lower: + probable = "Pagamento ainda não validado ou estado financeiro inconsistente." + fix_label = "Ver pagamento" + elif "timeout" in lower or "connection" in lower or "tempor" in lower: + probable = "Falha temporária de ligação à integração externa." + fix_label = "Reprocessar depois" + return {"title": title, "probable": probable, "fix_label": fix_label, "technical": error} diff --git a/app/admin_ui/htmx.py b/app/admin_ui/htmx.py new file mode 100644 index 0000000..6dc32ac --- /dev/null +++ b/app/admin_ui/htmx.py @@ -0,0 +1,35 @@ +"""Small HTMX helpers shared by admin pages.""" +from __future__ import annotations + +from typing import Any + +from fastapi import Request + +from app.admin_ui.components import esc + + +def is_htmx_request(request: Request | None) -> bool: + if request is None: + return False + return str(request.headers.get("HX-Request") or "").lower() == "true" + + +def htmx_attrs( + url: str, + *, + target: str, + swap: str = "innerHTML", + push_url: str | bool | None = None, + indicator: str | None = None, +) -> str: + attrs = [ + f'hx-get="{esc(url)}"', + f'hx-target="{esc(target)}"', + f'hx-swap="{esc(swap)}"', + ] + if push_url is not None: + value = "true" if push_url is True else "false" if push_url is False else str(push_url) + attrs.append(f'hx-push-url="{esc(value)}"') + if indicator: + attrs.append(f'hx-indicator="{esc(indicator)}"') + return " ".join(attrs) diff --git a/app/admin_ui/labels.py b/app/admin_ui/labels.py new file mode 100644 index 0000000..71d6d2e --- /dev/null +++ b/app/admin_ui/labels.py @@ -0,0 +1,109 @@ +"""Centralized Portuguese labels for the admin UI. + +v4.7.3 keeps business rules unchanged, but moves display labels out of the +route handlers so Operations, Tasks and Opportunities use the same wording. +""" +from __future__ import annotations + +from typing import Any + +ACTION_LABELS = { + "SEND_INFO": "Enviar informação", + "SEND_QUOTE": "Preparar orçamento", + "SEND_PROFORMA": "Emitir pró-forma", + "SEND_INVOICE": "Emitir fatura", + "CONFIRM_PAYMENT": "Confirmar pagamento", + "CONFIRM_PAYMENT_AND_PREPARE_SHIPMENT": "Confirmar pagamento", + "PREPARE_ORDER": "Preparar encomenda", + "CREATE_SHIPMENT": "Criar envio", + "REVIEW_MANUALLY": "Rever manualmente", + "ASSOCIATE_CUSTOMER": "Associar cliente", + "ASSOCIATE_OPPORTUNITY": "Associar oportunidade", + "MARK_NO_INTEREST": "Marcar sem interesse", + "REMOVE_FROM_LIST": "Remover da lista", + "SUPPORT": "Responder suporte", + "NO_ACTION": "Sem ação", + "IGNORE_SPAM": "Ignorar spam", +} + +PRIMARY_ACTION_LABELS = { + "SEND_INFO": "Preparar resposta", + "SEND_QUOTE": "Preparar orçamento", + "SEND_PROFORMA": "Preparar pró-forma", + "SEND_INVOICE": "Emitir fatura", + "CONFIRM_PAYMENT": "Confirmar pagamento", + "CONFIRM_PAYMENT_AND_PREPARE_SHIPMENT": "Confirmar pagamento", + "PREPARE_ORDER": "Preparar encomenda", + "CREATE_SHIPMENT": "Criar envio", + "REVIEW_MANUALLY": "Rever mensagem", + "ASSOCIATE_CUSTOMER": "Associar cliente", + "ASSOCIATE_OPPORTUNITY": "Associar oportunidade", + "MARK_NO_INTEREST": "Marcar sem interesse", + "REMOVE_FROM_LIST": "Remover da lista", + "SUPPORT": "Responder suporte", +} + +QUEUE_LABELS = { + "vendas": "Vendas", + "financeiro": "Financeiro", + "operacoes": "Operações", + "operações": "Operações", + "logistica": "Logística", + "logística": "Logística", + "rever": "Revisão", + "revisao": "Revisão", + "revisão": "Revisão", + "suporte": "Suporte", +} + +STATUS_LABELS = { + "pending": "Pendente", + "done": "Concluída", + "skipped": "Ignorada", + "failed": "Falhada", + "blocked": "Bloqueada", + "processing": "Em processamento", + "sent": "Enviada", + "dry_run": "Dry-run", + "ignored": "Ignorada", + "cancelled": "Cancelada", + "created": "Criada", + "issued": "Emitida", + "converted": "Convertida", +} + +BLOCK_REASON_LABELS = { + "ambiguous_opportunity": "Associação por confirmar", + "missing_customer": "Cliente fiscal em falta", + "missing_product_code": "Produto sem código externo", + "outbox_failed": "Integração falhada", + "payment_confirmation": "Confirmação financeira obrigatória", +} + + +def normalized_code(value: Any) -> str: + return str(value or "").strip().upper() + + +def action_label(code: Any, fallback: str = "Ação") -> str: + return ACTION_LABELS.get(normalized_code(code), str(code or fallback)) + + +def primary_action_label(code: Any, fallback: str = "Abrir") -> str: + normalized = normalized_code(code) + return PRIMARY_ACTION_LABELS.get(normalized) or ACTION_LABELS.get(normalized) or str(code or fallback) + + +def queue_label(queue: Any) -> str: + key = str(queue or "").strip().lower() + return QUEUE_LABELS.get(key, str(queue or "—")) + + +def status_label(status: Any) -> str: + key = str(status or "").strip().lower() + return STATUS_LABELS.get(key, str(status or "—")) + + +def block_reason_label(reason: Any) -> str: + key = str(reason or "").strip().lower() + return BLOCK_REASON_LABELS.get(key, str(reason or "Bloqueio operacional")) diff --git a/app/admin_ui/layout.py b/app/admin_ui/layout.py new file mode 100644 index 0000000..e6554fd --- /dev/null +++ b/app/admin_ui/layout.py @@ -0,0 +1,59 @@ +"""Shared page shell for the ClientFlow admin UI.""" +from __future__ import annotations + +from fastapi.responses import HTMLResponse + +from app.admin_ui.components import esc +from app.admin_ui.navigation import nav +from app.admin_ui.styles import ADMIN_UI_V451_CSS + + +def layout(title: str, subtitle: str, body: str, active: str = "") -> HTMLResponse: + html_doc = f""" + + + + + + {esc(title)} · ClientFlow + + + + + +
    + {nav(active)} +
    +
    +
    + + +
    +
    + Hoje · Centro de trabalho + ? + ↗ + CF +
    +
    + +
    +
    +
    +

    {esc(title)}

    +

    {esc(subtitle)}

    +
    +
    + {body} +
    +
    +
    + + + + + """ + return HTMLResponse(html_doc) diff --git a/app/admin_ui/navigation.py b/app/admin_ui/navigation.py new file mode 100644 index 0000000..524d2bb --- /dev/null +++ b/app/admin_ui/navigation.py @@ -0,0 +1,103 @@ +"""Navigation model and renderer for the ClientFlow admin UI.""" +from __future__ import annotations + +from dataclasses import dataclass + +from app.admin_ui.components import esc + + +@dataclass(frozen=True) +class NavItem: + key: str + url: str + icon: str + label: str + area: str = "primary" + + +PRIMARY_NAV_ITEMS: tuple[NavItem, ...] = ( + NavItem("overview", "/", "bi-speedometer2", "Dashboard"), + NavItem("operations", "/operations", "bi-check2-square", "Centro de trabalho"), + NavItem("reconciliation", "/reconciliation", "bi-diagram-3", "Reconciliação"), + NavItem("opportunities", "/opportunities", "bi-funnel", "Oportunidades"), + NavItem("customers", "/customers", "bi-people", "Clientes"), + NavItem("products", "/products", "bi-box-seam", "Produtos"), + NavItem("orders", "/orders", "bi-truck", "Encomendas"), + NavItem("finance", "/finance", "bi-file-earmark-text", "Financeiro"), + NavItem("integrations", "/integrations", "bi-puzzle", "Integrações"), +) + +ADMIN_NAV_ITEMS: tuple[NavItem, ...] = ( + NavItem("tasks", "/tasks", "bi-list-check", "Tasks", "admin"), + NavItem("communications", "/communications", "bi-envelope", "Comunicações", "admin"), + NavItem("outbox", "/outbox", "bi-send", "Outbox", "admin"), + NavItem("events", "/events", "bi-activity", "Eventos", "admin"), + NavItem("runs", "/runs", "bi-cpu", "Runs", "admin"), + NavItem("queues", "/queues", "bi-diagram-3", "Filas", "admin"), + NavItem("system", "/system/health", "bi-heart-pulse", "System health", "admin"), + NavItem("settings", "/settings", "bi-gear", "Configuração", "admin"), +) + +ADMIN_ACTIVE_KEYS = frozenset(item.key for item in ADMIN_NAV_ITEMS) + + +def _render_items(items: tuple[NavItem, ...], active: str) -> str: + links = "" + for item in items: + css = "cf-nav-link active" if item.key == active else "cf-nav-link" + links += f""" + + + {esc(item.label)} + + """ + return links + + +def nav(active: str = "") -> str: + """Render the operator-first navigation introduced in v4.7. + + Primary navigation stays focused on daily work. Technical pages remain + available under Admin, without removing their public URLs. + """ + active = active or "overview" + admin_active = " active" if active in ADMIN_ACTIVE_KEYS else "" + admin_open = " open" if active in ADMIN_ACTIVE_KEYS else "" + primary_links = _render_items(PRIMARY_NAV_ITEMS, active) + admin_links = _render_items(ADMIN_NAV_ITEMS, active) + + return f""" + + """ diff --git a/app/admin_ui/pages/README.md b/app/admin_ui/pages/README.md new file mode 100644 index 0000000..50a4229 --- /dev/null +++ b/app/admin_ui/pages/README.md @@ -0,0 +1,15 @@ +# Admin UI pages + +Incremental extraction target for `app/admin_dashboard.py`. + +Planned page modules: + +- `customers.py` +- `opportunities.py` +- `products.py` +- `documents.py` +- `shipments.py` +- `integrations.py` +- `outbox.py` + +Public routes should stay unchanged while code is migrated page-by-page. diff --git a/app/admin_ui/pages/__init__.py b/app/admin_ui/pages/__init__.py new file mode 100644 index 0000000..062e66c --- /dev/null +++ b/app/admin_ui/pages/__init__.py @@ -0,0 +1 @@ +"""Domain route modules for the ClientFlow admin UI.""" diff --git a/app/admin_ui/pages/communications.py b/app/admin_ui/pages/communications.py new file mode 100644 index 0000000..7459440 --- /dev/null +++ b/app/admin_ui/pages/communications.py @@ -0,0 +1,201 @@ +"""Communication diagnostic routes. + +Moved from app.admin_dashboard in v4.7.2. The handlers still reuse +legacy helpers to keep this refactor behavior-preserving. +""" +from fastapi import APIRouter +import app.admin_dashboard as legacy +from app.admin_dashboard import * # noqa: F401,F403 + +router = APIRouter() + + +@router.get("/communications", response_class=HTMLResponse) +@router.get("/comunicacoes", response_class=HTMLResponse) +async def communications_page(status: Optional[str] = None, classification: Optional[str] = None, q: Optional[str] = None, scope: Optional[str] = None): + items = list_communications(status=status, classification=classification, q=q, limit=120) + if scope == "without_customer": + items = [i for i in items if not i.get("customer_id") and str(i.get("direction") or "inbound") == "inbound"] + elif scope == "without_opportunity": + items = [i for i in items if not i.get("opportunity_id") and str(i.get("direction") or "inbound") == "inbound"] + elif scope == "needs_review": + items = [i for i in items if str(i.get("status") or "") == "needs_review"] + + summary = get_communications_summary() + + rows = "" + for item in items: + action = classification_action(item.get("classification")) + confidence = item.get("confidence") + confidence_label = "—" if confidence is None else f"{float(confidence):.2f}" + linked = [] + if item.get("customer_name"): + linked.append(str(item.get("customer_name"))) + elif item.get("customer_id"): + linked.append("cliente associado") + else: + linked.append("sem cliente") + if item.get("opportunity_title"): + linked.append(str(item.get("opportunity_title"))) + elif item.get("opportunity_id"): + linked.append("oportunidade associada") + else: + linked.append("sem oportunidade") + rows += f""" + + {esc(item.get('subject') or 'Sem assunto')}
    {esc(item.get('sender_name') or item.get('sender_email') or '—')} · {esc(fmt_dt(item.get('created_at')))}
    + {esc(item.get('classification') or 'por classificar')}
    confiança {esc(confidence_label)}
    + {status_badge(item.get('status'))}
    {esc(action.get('queue'))} · {esc(action.get('action'))}
    + {esc(' · '.join(linked))} + Abrir + + """ + if not rows: + rows = 'Sem comunicações para os filtros selecionados.' + + status_options = ["", "new", "classified", "needs_review", "linked", "task_created", "done", "ignored"] + status_select = "".join(f'' for v in status_options) + + body = f""" +
    + Comunicações = emails/mensagens classificados. + Quando uma comunicação exige ação humana, ela também aparece no Centro de trabalho e pode gerar task. +
    + +
    + {kpi_card('Total', summary.get('total', 0), '/communications', 'comunicações registadas', 'bi-inbox')} + {kpi_card('Por tratar', summary.get('open', 0), '/communications', 'new/classified/review', 'bi-envelope-exclamation')} + {kpi_card('Baixa confiança', summary.get('needs_review', 0), '/communications?status=needs_review', 'precisa revisão', 'bi-shield-exclamation')} + {kpi_card('Sem cliente', summary.get('without_customer', 0), '/communications?scope=without_customer', 'associar contacto', 'bi-person-plus')} +
    + +
    +
    +
    +
    +
    +
    + +

    Inbox classificada

    Emails e mensagens recebidas, com classificação e contexto operacional.
    {rows}
    MensagemClassificaçãoEstado / ação sugeridaCliente / oportunidade
    + """ + return layout("Comunicações", "Inbox classificada de emails e mensagens", body, "communications") + + +@router.get("/communications/{communication_id}", response_class=HTMLResponse) +async def communication_detail_page(communication_id: str): + item = get_communication(communication_id) + if not item: + return layout("Comunicação não encontrada", "Inbox classificada", '
    Comunicação não encontrada.
    ', "communications") + + action = classification_action(item.get("classification")) + confidence = item.get("confidence") + confidence_label = "—" if confidence is None else f"{float(confidence):.2f}" + + try: + from app.commercial_service import list_customers + customers = list_customers(limit=200) + except Exception: + customers = [] + try: + opportunities = list_opportunities(limit=200, status="open") + except Exception: + opportunities = [] + + customer_options = '' + for c in customers: + selected = "selected" if str(c.get("id")) == str(item.get("customer_id")) else "" + label = f"{c.get('name') or 'Cliente'} · {c.get('tax_id') or c.get('email') or 'sem NIF'}" + customer_options += f'' + + opportunity_options = '' + for o in opportunities: + selected = "selected" if str(o.get("id")) == str(item.get("opportunity_id")) else "" + label = f"{o.get('title') or 'Oportunidade'} · {o.get('linked_customer_name') or o.get('customer_name') or 'sem cliente'}" + opportunity_options += f'' + + body = f""" + ← Voltar a Comunicações +
    +
    +
    +
    Comunicação

    {esc(item.get('subject') or 'Sem assunto')}

    {esc(item.get('sender_name') or '')} · {esc(item.get('sender_email') or '—')} · {esc(fmt_dt(item.get('created_at')))}
    {status_badge(item.get('status'))}
    +
    {esc(item.get('classification') or 'por classificar')}confiança {esc(confidence_label)}{esc(action.get('queue'))}
    +
    {esc(item.get('body') or 'Sem corpo guardado.')}
    +
    +

    Ação sugerida

    {esc(action.get('action'))}
    Esta ação deve aparecer no Centro de trabalho se a comunicação estiver por tratar.
    +
    + +
    + """ + return layout(str(item.get("subject") or "Comunicação"), "Email/mensagem classificado com contexto", body, "communications") + + +@router.post("/communications/{communication_id}/status") +async def communication_set_status_action(communication_id: str, request: Request): + form = await request.form() + status = str(form.get("status") or "").strip() + try: + set_communication_status(communication_id, status) + item = get_communication(communication_id) + if item and item.get("opportunity_id"): + create_timeline_event( + opportunity_id=str(item.get("opportunity_id")), + customer_id=str(item.get("customer_id") or "") or None, + event_type=f"communication_{status}", + title=f"Comunicação marcada como {status}", + description=str(item.get("subject") or item.get("sender_email") or ""), + source="communications", + related_type="communication", + related_id=communication_id, + created_by="operator", + ) + except Exception as exc: + return PlainTextResponse(f"Erro ao atualizar comunicação: {exc}", status_code=500) + return RedirectResponse(f"/communications/{communication_id}", status_code=303) + + +@router.post("/communications/{communication_id}/link-customer") +async def communication_link_customer_action(communication_id: str, request: Request): + form = await request.form() + customer_id = str(form.get("customer_id") or "").strip() + try: + link_communication_to_customer(communication_id, customer_id or None) + except Exception as exc: + return PlainTextResponse(f"Erro ao associar cliente: {exc}", status_code=500) + return RedirectResponse(f"/communications/{communication_id}", status_code=303) + + +@router.post("/communications/{communication_id}/link-opportunity") +async def communication_link_opportunity_action(communication_id: str, request: Request): + form = await request.form() + opportunity_id = str(form.get("opportunity_id") or "").strip() + try: + link_communication_to_opportunity(communication_id, opportunity_id or None) + if opportunity_id: + item = get_communication(communication_id) + create_timeline_event( + opportunity_id=opportunity_id, + customer_id=str(item.get("customer_id") or "") if item else None, + event_type="communication_linked", + title="Comunicação associada à oportunidade", + description=str((item or {}).get("subject") or (item or {}).get("sender_email") or ""), + source="communications", + related_type="communication", + related_id=communication_id, + created_by="operator", + ) + except Exception as exc: + return PlainTextResponse(f"Erro ao associar oportunidade: {exc}", status_code=500) + return RedirectResponse(f"/communications/{communication_id}", status_code=303) + + diff --git a/app/admin_ui/pages/conversations.py b/app/admin_ui/pages/conversations.py new file mode 100644 index 0000000..dfe4a9d --- /dev/null +++ b/app/admin_ui/pages/conversations.py @@ -0,0 +1,24 @@ +"""Conversation placeholder routes. + +Moved from app.admin_dashboard in v4.7.2. The handlers still reuse +legacy helpers to keep this refactor behavior-preserving. +""" +from fastapi import APIRouter +import app.admin_dashboard as legacy +from app.admin_dashboard import * # noqa: F401,F403 + +router = APIRouter() + + +@router.get("/conversations", response_class=HTMLResponse) +@router.get("/conversas", response_class=HTMLResponse) +async def conversations_page(): + body = ''' +
    +

    Conversas

    +

    A inbox continua no Chatwoot. Esta página fica reservada para a vista consolidada de conversas dentro do ClientFlow.

    +
    + ''' + return layout("Conversas", "Vista futura de conversas sincronizadas", body, "conversations") + + diff --git a/app/admin_ui/pages/customers.py b/app/admin_ui/pages/customers.py new file mode 100644 index 0000000..ad8d5b1 --- /dev/null +++ b/app/admin_ui/pages/customers.py @@ -0,0 +1,199 @@ +"""Customer routes. + +Moved from app.admin_dashboard in v4.7.2. The handlers still reuse +legacy helpers to keep this refactor behavior-preserving. +""" +from fastapi import APIRouter +import app.admin_dashboard as legacy +from app.admin_dashboard import * # noqa: F401,F403 + +router = APIRouter() + + +@router.get("/customers", response_class=HTMLResponse) +@router.get("/clientes", response_class=HTMLResponse) +async def customers_page(q: Optional[str] = None): + try: + from app.commercial_service import list_customers + customer_rows = list_customers(q=q, limit=300) + error = "" + except Exception as exc: + customer_rows = [] + error = str(exc) + + rows = "" + for c in customer_rows: + jasmin_state = "Ligado" if c.get("jasmin_customer_party_key") else "Por validar" + jasmin_cls = "cf-chip-green" if c.get("jasmin_customer_party_key") else "cf-chip-orange" + rows += f""" + + {esc(c.get('name') or 'Cliente')}
    NIF {esc(c.get('tax_id') or '—')}
    +
    {esc(c.get('email') or '—')}
    {esc(c.get('phone') or '')}
    + {esc(c.get('city_name') or '—')}
    {esc(c.get('postal_zone') or '')}
    + {esc(jasmin_state)}
    {esc(c.get('jasmin_customer_party_key') or '')}
    + {int(c.get('opportunity_count') or 0)} oportunidades + {esc(fmt_dt(c.get('updated_at')))} + + """ + if not rows: + rows = 'Sem clientes locais. Cria uma ficha ou associa a partir de uma oportunidade.' + + error_html = f'
    {esc(error)}
    ' if error else '' + body = f""" +
    +
    +
    +
    +
    +
    Limpar
    +
    +
    +
    +
    +

    Novo cliente

    +
    +
    + {error_html} +

    Clientes

    Dados fiscais e moradas vivem aqui. A oportunidade mostra só o estado da compra.
    {rows}
    ClienteContactosLocalidadeJasminPipelineAtualizado
    + """ + return layout("Clientes", "Ficha fiscal, contactos e documentos por cliente", body, "customers") + + +@router.post("/customers/create") +async def create_customer_action(request: Request): + form = await request.form() + try: + from app.commercial_service import upsert_customer + customer = upsert_customer({ + "name": str(form.get("name") or "").strip(), + "tax_id": str(form.get("tax_id") or "").strip(), + "email": str(form.get("email") or "").strip(), + "phone": str(form.get("phone") or "").strip(), + }) + except Exception as exc: + return PlainTextResponse(f"Erro ao criar cliente: {exc}", status_code=500) + return RedirectResponse(f"/customers/{customer.get('id')}", status_code=303) + + +@router.get("/customers/{customer_id}", response_class=HTMLResponse) +async def customer_detail_page(customer_id: str): + try: + from app.commercial_service import get_customer, list_commercial_documents, list_opportunities_for_customer, list_shipments + customer = get_customer(customer_id) + if not customer: + return layout("Cliente não encontrado", "Clientes", '
    Cliente não encontrado.
    ', "customers") + docs = list_commercial_documents(customer_id=customer_id, limit=100) + opps = list_opportunities_for_customer(customer_id, limit=50) + shipments = list_shipments(customer_id=customer_id, limit=50) + except Exception as exc: + return PlainTextResponse(f"Erro ao abrir cliente: {exc}", status_code=500) + + doc_rows = "" + for d in docs: + number = d.get("document_number") or " ".join([str(d.get("document_type") or ""), str(d.get("serie") or ""), str(d.get("series_number") or "")]).strip() or d.get("external_id") or "—" + kind = {"quotation": "Orçamento", "invoice": "Fatura"}.get(str(d.get("document_kind") or ""), d.get("document_kind") or "Documento") + doc_rows += f"{esc(kind)}{esc(number)}{operation_status_badge(str(d.get('status') or 'created'))}{money_html(d.get('total_amount') or d.get('amount') or 0)}{esc(fmt_dt(d.get('created_at')))}" + if not doc_rows: + doc_rows = 'Sem documentos Jasmin locais.' + + opp_rows = "" + for o in opps: + opp_rows += f"{esc(o.get('title') or 'Oportunidade')}
    {esc(o.get('product_interest') or '')}
    {opportunity_stage_badge(o.get('stage'))}{money_html(o.get('value_amount') or 0)}{esc(fmt_dt(o.get('updated_at')))}" + if not opp_rows: + opp_rows = 'Sem oportunidades associadas.' + + shipment_rows = "" + for sh in shipments: + shipment_rows += f"{esc(sh.get('carrier') or '—')}{esc(sh.get('service_name') or '—')}{operation_status_badge(str(sh.get('status') or 'created'))}{esc(sh.get('external_reference') or '')}{esc(sh.get('tracking_code') or '—')}" + if not shipment_rows: + shipment_rows = 'Sem envios Packlink locais.' + + body = f""" + ← Voltar a clientes +
    +
    +

    {esc(customer.get('name'))}

    NIF {esc(customer.get('tax_id') or '—')}
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +

    Nova oportunidade

    Cria um processo comercial manual já ligado a este cliente fiscal.
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +

    Oportunidades

    {opp_rows}
    OportunidadeEstadoValorAtualizada
    +

    Documentos Jasmin

    {doc_rows}
    TipoNúmero/IDEstadoValorCriado
    +

    Envios Packlink

    {shipment_rows}
    TransportadoraServiçoEstadoReferênciaTracking
    +
    +
    + """ + return layout(str(customer.get("name") or "Cliente"), "Ficha fiscal, oportunidades e documentos", body, "customers") + + +@router.post("/customers/{customer_id}/opportunities/create") +async def create_customer_opportunity_action(customer_id: str, request: Request): + form = await request.form() + try: + from app.opportunity_service import create_manual_opportunity_from_customer + result = create_manual_opportunity_from_customer( + customer_id, + origin=str(form.get("origin") or "phone").strip(), + request_type=str(form.get("request_type") or "quote").strip(), + contact_name=str(form.get("contact_name") or "").strip(), + contact_email=str(form.get("contact_email") or "").strip(), + contact_phone=str(form.get("contact_phone") or "").strip(), + product_interest=str(form.get("product_interest") or "").strip(), + notes=str(form.get("notes") or "").strip(), + create_task=bool(form.get("create_task")), + created_by="operator", + ) + except Exception as exc: + return PlainTextResponse(f"Erro ao criar oportunidade: {exc}", status_code=500) + return RedirectResponse(result.get("next_url") or f"/customers/{customer_id}", status_code=303) + + +@router.post("/customers/{customer_id}/update") +async def update_customer_action(customer_id: str, request: Request): + form = await request.form() + try: + from app.commercial_service import update_customer + update_customer(customer_id, { + "name": str(form.get("name") or "").strip(), + "tax_id": str(form.get("tax_id") or "").strip(), + "email": str(form.get("email") or "").strip(), + "phone": str(form.get("phone") or "").strip(), + "street_name": str(form.get("street_name") or "").strip(), + "postal_zone": str(form.get("postal_zone") or "").strip(), + "city_name": str(form.get("city_name") or "").strip(), + "country": str(form.get("country") or "PT").strip(), + "jasmin_customer_party_key": str(form.get("jasmin_customer_party_key") or "").strip(), + "jasmin_customer_id": str(form.get("jasmin_customer_id") or "").strip(), + }) + except Exception as exc: + # Keep database details out of the operator UI. Duplicate NIFs are a + # business conflict, not a technical 500. + status = 409 if exc.__class__.__name__ == "DuplicateCustomerTaxIdError" else 500 + return PlainTextResponse(f"Erro ao guardar cliente: {exc}", status_code=status) + return RedirectResponse(f"/customers/{customer_id}", status_code=303) + + diff --git a/app/admin_ui/pages/dashboard.py b/app/admin_ui/pages/dashboard.py new file mode 100644 index 0000000..f2de6c3 --- /dev/null +++ b/app/admin_ui/pages/dashboard.py @@ -0,0 +1,102 @@ +"""Dashboard and landing routes. + +Moved from app.admin_dashboard in v4.7.2. The handlers still reuse +legacy helpers to keep this refactor behavior-preserving. +""" +from fastapi import APIRouter +import app.admin_dashboard as legacy +from app.admin_dashboard import * # noqa: F401,F403 + +router = APIRouter() + + +@router.get("/", response_class=HTMLResponse) +async def admin_home(): + """v4.5 clean Dashboard: visibility, not daily execution.""" + metrics = get_admin_dashboard_metrics() + ops = get_operations_summary(limit=6) + counts = ops.get("counts") or {} + comms = get_communications_summary() + + def n(key: str) -> int: + return int(metrics.get(key) or counts.get(key) or 0) + + dashboard_cards = [ + ("Oportunidades abertas", counts.get("open_opportunities", 0), "/opportunities?status=open", "Negócio em acompanhamento"), + ("Valor / documentos", counts.get("open_quotations", 0), "/finance", "Orçamentos abertos"), + ("Tasks pendentes", n("pending_total"), "/operations", "Trabalho humano por resolver"), + ("Mensagens a rever", comms.get("open", 0), "/operations", "Ações vindas do Chatwoot"), + ("Erros de integração", counts.get("outbox_failed", 0), "/outbox?status=failed", "Jasmin/Packlink/outbox"), + ("Pagamentos por confirmar", n("pending_financeiro"), "/operations", "Fila financeira"), + ("Envios pendentes", counts.get("shipments_pending", 0), "/orders", "Logística/Packlink"), + ("Clientes incompletos", counts.get("customers_incomplete", 0), "/customers", "Dados fiscais/morada"), + ] + + cards_html = "" + for label, value, href, hint in dashboard_cards: + cards_html += kpi_card(label, value, href, hint) + + alert_items = [] + if int(counts.get("outbox_failed") or 0): + alert_items.append(("Erro de integração", f"{counts.get('outbox_failed')} ação(ões) falhadas na outbox", "/outbox?status=failed", "cf-chip-red")) + if int(comms.get("needs_review") or 0): + alert_items.append(("Rever comunicação", f"{comms.get('needs_review')} mensagem(ns) com baixa confiança", "/operations", "cf-chip-orange")) + if int(counts.get("customers_incomplete") or 0): + alert_items.append(("Dados incompletos", f"{counts.get('customers_incomplete')} cliente(s) sem dados fiscais/morada completos", "/customers", "cf-chip-orange")) + if int(counts.get("products_missing_jasmin") or 0): + alert_items.append(("Produto bloqueante", f"{counts.get('products_missing_jasmin')} produto(s) ativos sem Artigo Jasmin", "/products?active=missing_jasmin", "cf-chip-red")) + + alert_html = "" + for title, detail, href, chip in alert_items[:5]: + alert_html += f""" + +
    {esc(title)}
    {esc(detail)}
    + Abrir → +     + """ + if not alert_html: + alert_html = '
    Sem alertas críticos neste momento.
    ' + + body = f""" +
    +
    + Dashboard = visibilidade. + O Chatwoot é a inbox. O ClientFlow mostra o trabalho, bloqueios e próximas ações. +
    + Abrir Centro de trabalho +
    + +
    + {cards_html} +
    + +
    +
    +
    +
    +
    +

    Alertas

    Sinais globais que merecem atenção.
    + Resolver no Centro de trabalho +
    + {alert_html} +
    +
    +
    +
    +
    +
    +

    Modelo operacional v4.5

    +
    +
    Dashboard
    Mostra o estado e gargalos.
    +
    Centro de trabalho
    Organiza o que precisa de ação agora.
    +
    Chatwoot → ClientFlow
    O Chatwoot continua a ser a inbox; o ClientFlow transforma mensagens em ações, tasks e timeline.
    +
    Oportunidade
    Mantém contexto, documentos, tasks, outbox e timeline.
    +
    +
    +
    +
    +
    + """ + return layout("Dashboard", "Visão geral do negócio e do sistema", body, "overview") + + diff --git a/app/admin_ui/pages/events.py b/app/admin_ui/pages/events.py new file mode 100644 index 0000000..f1e4ee2 --- /dev/null +++ b/app/admin_ui/pages/events.py @@ -0,0 +1,46 @@ +"""Business event routes. + +Moved from app.admin_dashboard in v4.7.2. The handlers still reuse +legacy helpers to keep this refactor behavior-preserving. +""" +from fastapi import APIRouter +import app.admin_dashboard as legacy +from app.admin_dashboard import * # noqa: F401,F403 + +router = APIRouter() + + +@router.get("/events", response_class=HTMLResponse) +async def events_page(limit: int = 100): + events = list_business_events(limit=limit) + + rows = "" + for event in events: + rows += f""" + + {esc(event["created_at"])} + {esc(event.get("event_type"))} + #{esc(event.get("conversation_id"))} + {esc(event.get("task_id"))} + {esc(event.get("created_by"))} + + """ + + body = f""" + + + + + + + + + + + {rows} +
    CriadoEventoConversaTaskCriado por
    + """ + + return layout("Business Events", "Eventos criados após conclusão de tarefas.", body, "events") + + diff --git a/app/admin_ui/pages/finance.py b/app/admin_ui/pages/finance.py new file mode 100644 index 0000000..ee662e9 --- /dev/null +++ b/app/admin_ui/pages/finance.py @@ -0,0 +1,51 @@ +"""Finance routes. + +Moved from app.admin_dashboard in v4.7.2. The handlers still reuse +legacy helpers to keep this refactor behavior-preserving. +""" +from fastapi import APIRouter +import app.admin_dashboard as legacy +from app.admin_dashboard import * # noqa: F401,F403 + +router = APIRouter() + + +@router.get("/finance", response_class=HTMLResponse) +@router.get("/financeiro", response_class=HTMLResponse) +async def finance_page(q: Optional[str] = None): + finance_tasks = list_tasks(status=None, route="financeiro", q=q, limit=200) + finance_actions = {"SEND_PROFORMA", "SEND_INVOICE", "CONFIRM_PAYMENT"} + finance_tasks = [t for t in finance_tasks if str(t.get("action_code") or "") in finance_actions or str(t.get("route") or "") == "financeiro"] + opportunities = list_opportunities(status="all", q=q, limit=300) + payment_opps = [o for o in opportunities if str(o.get("stage") or "") in {"PROFORMA_REQUESTED", "PROFORMA_SENT", "INVOICE_REQUESTED", "INVOICE_SENT", "WAITING_PAYMENT", "PAYMENT_CONFIRMED"}] + + rows = "" + for task in finance_tasks[:80]: + tid = str(task.get("id") or "") + customer = customer_display(task) + code = str(task.get("action_code") or "") + rows += f""" + + {esc(action_label(code))}
    {esc(code)}
    + {esc(customer)}
    {esc(task.get('customer_email') or '')}
    + {status_badge(task.get('status'))} + {esc(task.get('created_at') or '—')} + Abrir + + """ + if not rows: + rows = 'Sem tarefas financeiras.' + + body = f""" +
    + {kpi_card('Tarefas financeiras', len(finance_tasks), '/finance', 'ativas/histórico', 'bi-list-check')} + {kpi_card('A aguardar pagamento', sum(1 for o in payment_opps if str(o.get('stage')) == 'WAITING_PAYMENT'), '/opportunities', 'oportunidades', 'bi-hourglass-split', 'cf-kpi-tone-orange')} + {kpi_card('Pendentes', sum(1 for t in finance_tasks if str(t.get('status')) == 'pending'), '/tasks?status=pending&route=financeiro', 'abrir tarefas', 'bi-list-check')} + {kpi_card('Pagamentos confirmados', sum(1 for o in payment_opps if str(o.get('stage')) == 'PAYMENT_CONFIRMED'), '/opportunities', 'seguir para envio', 'bi-check2-circle', 'cf-kpi-tone-green')} +
    +
    Limpar
    +

    Financeiro operacional

    Pró-formas, faturas e pagamentos a tratar.
    {rows}
    AçãoClienteEstadoCriada
    + """ + return layout("Financeiro", "O que falta faturar ou confirmar?", body, "finance") + + diff --git a/app/admin_ui/pages/integrations.py b/app/admin_ui/pages/integrations.py new file mode 100644 index 0000000..2f8d7d6 --- /dev/null +++ b/app/admin_ui/pages/integrations.py @@ -0,0 +1,198 @@ +"""Integration configuration and action routes. + +Moved from app.admin_dashboard in v4.7.2. The handlers still reuse +legacy helpers to keep this refactor behavior-preserving. +""" +from fastapi import APIRouter +import app.admin_dashboard as legacy +from app.admin_dashboard import * # noqa: F401,F403 + +router = APIRouter() + + +@router.get("/integrations", response_class=HTMLResponse) +@router.get("/integracoes", response_class=HTMLResponse) +async def integrations_page(): + odoo_stats = {} + odoo_last_sync = "—" + odoo_error = "" + + try: + snapshot = get_odoo_product_snapshot(limit=1) + odoo_stats = snapshot.get("stats") or {} + odoo_last_sync = odoo_stats.get("last_synced_at") or "—" + except Exception as exc: + odoo_error = str(exc) + + def badge(enabled: bool) -> str: + if enabled: + return 'Ativo' + return 'Inativo' + + odoo_enabled = bool(getattr(settings, "odoo_enabled", False)) + jasmin_enabled = bool(getattr(settings, "jasmin_enabled", False)) + packlink_enabled = bool(getattr(settings, "packlink_enabled", False)) + chatwoot_enabled = bool(getattr(settings, "chatwoot_base_url", "")) + + odoo_error_html = "" + if odoo_error: + odoo_error_html = f""" +
    + Erro Odoo: {esc(odoo_error)} +
    + """ + + body = f""" +
    +
    +
    Configuração
    +

    Integrações

    +

    Ligações externas usadas pelo cockpit operacional.

    +
    +
    + +
    +
    +
    +
    +
    +
    +

    Odoo

    +
    Produtos vendáveis, stock e estado operacional.
    +
    + {badge(odoo_enabled)} +
    + +
    +
    Base URL
    +
    {esc(getattr(settings, "odoo_base_url", "") or "—")}
    + +
    Base de dados
    +
    {esc(getattr(settings, "odoo_db", "") or "—")}
    + +
    Produtos sincronizados
    +
    {esc(odoo_stats.get("synced_products") or 0)}
    + +
    Com BOM
    +
    {esc(odoo_stats.get("products_with_bom") or 0)}
    + +
    Com stock disponível
    +
    {esc(odoo_stats.get("products_with_available_stock") or 0)}
    + +
    Último sync
    +
    {esc(odoo_last_sync)}
    +
    + +
    +
    + +
    +
    + +
    +
    + + {odoo_error_html} +
    +
    +
    + +
    +
    +
    +
    +
    +

    Jasmin

    +
    Pró-formas, faturas e documentos fiscais.
    +
    + {badge(jasmin_enabled)} +
    + +
    +
    Base URL
    +
    {esc(getattr(settings, "jasmin_base_url", "") or "—")}
    + +
    URL público
    +
    {esc(getattr(settings, "jasmin_public_url", "") or "—")}
    +
    +
    +
    +
    + +
    +
    +
    +
    +
    +

    Packlink PRO

    +
    Recolhas, envios, etiquetas e tracking.
    +
    + {badge(packlink_enabled)} +
    + +
    +
    Base URL
    +
    {esc(getattr(settings, "packlink_base_url", "") or "—")}
    + +
    URL público
    +
    {esc(getattr(settings, "packlink_public_url", "") or "—")}
    +
    +
    +
    +
    + +
    +
    +
    +
    +
    +

    Chatwoot

    +
    Comunicação com cliente.
    +
    + {badge(chatwoot_enabled)} +
    + +
    +
    Base URL
    +
    {esc(getattr(settings, "chatwoot_base_url", "") or "—")}
    + +
    URL público
    +
    {esc(getattr(settings, "chatwoot_public_url", "") or "—")}
    +
    +
    +
    +
    +
    + +
    +
    +

    Regra operacional

    +

    + O ClientFlow não replica Odoo, Jasmin ou Packlink. Apenas consulta e guarda o estado necessário + para decidir a próxima ação com o cliente. +

    +
    +
    + """ + + return layout("Integrações", "Configuração das ligações externas", body, "integrations") + + +@router.post("/integrations/odoo/test") +async def integrations_odoo_test_action(request: Request): + try: + test_odoo_connection() + except Exception: + pass + return RedirectResponse(url="/integrations", status_code=303) + + +@router.post("/integrations/odoo/sync-products") +async def integrations_odoo_sync_products_action(request: Request): + try: + sync_odoo_products(limit=1000, include_inactive=True) + except Exception: + pass + return RedirectResponse(url="/integrations", status_code=303) + + diff --git a/app/admin_ui/pages/operations.py b/app/admin_ui/pages/operations.py new file mode 100644 index 0000000..a8a5282 --- /dev/null +++ b/app/admin_ui/pages/operations.py @@ -0,0 +1,185 @@ +"""Operator workbench routes. + +Moved from app.admin_dashboard in v4.7.2. v4.7.3 adds a first HTMX +slice: the work queue can be filtered and refreshed without replacing the full +page. The page still keeps behavior-preserving legacy helpers where needed. +""" +from __future__ import annotations + +from fastapi import APIRouter, Request +from fastapi.responses import HTMLResponse + +import app.admin_dashboard as legacy +from app.admin_dashboard import * # noqa: F401,F403 +from app.admin_ui.components import esc +from app.admin_ui.htmx import is_htmx_request +from app.admin_ui.guidance import work_item_blockers +from app.admin_ui.view_models.operations import ( + build_operations_view_model, + is_blocked, + is_high_priority, + operation_card_detail, + operation_card_subtitle, + operation_card_title, + operation_primary_label, + operation_status_chip, +) + +router = APIRouter() + +# Regression strings kept intentionally: Fila operacional priorizada, +# Mensagens para revisão, Mensagens sem cliente, Associar oportunidade. +# Filtros operacionais: Todas, Vendas, Financeiro, Logística, Revisão, Bloqueadas, Concluídas hoje. +# Botões primários: Preparar orçamento, Preparar pró-forma, Confirmar pagamento, Rever mensagem, Associar cliente. + + +def _operation_card_class(item: dict) -> str: + card_class = "high" if is_high_priority(item) else "normal" + if is_blocked(item): + card_class += " blocked" + return card_class + + +def _render_work_item(item: dict) -> str: + high = is_high_priority(item) + blocked = is_blocked(item) + title = operation_card_title(item) + subtitle = operation_card_subtitle(item) + detail = compact_text(operation_card_detail(item), 160) + if not detail: + detail = "Abrir a ação para ver o contexto e concluir o próximo passo." + blockers = work_item_blockers(item) + blockers_html = "" + if blockers: + blocker_items = "".join(f"
  • {esc(blocker)}
    • " for blocker in blockers) + blockers_html = f'
    Bloqueio atual
    ' + + opportunity_button = "" + if item.get("opportunity_id"): + opportunity_button = f'Ver oportunidade' + chatwoot_button = "" + if item.get("chatwoot_url"): + chatwoot_button = f'Abrir Chatwoot ↗' + + status_text = operation_status_chip(item) + priority_chip = "cf-chip-red" if high else "cf-chip-blue" + if blocked: + priority_chip = "cf-chip-orange" + if status_text == "sem oportunidade comercial": + priority_chip = "cf-chip-gray" + status_chip_html = "" if status_text == "normal" else f'{esc(status_text)}' + primary = operation_primary_label(item) + # v4.8.9: details are intentionally not rendered in Operations cards. + # Technical metadata remains available in task/opportunity/admin pages, while + # the work queue keeps only decision-making information. + return f""" +
    +
    +
    +
    {esc(title)}
    +
    {esc(subtitle)}
    +
    +
    + {status_chip_html} +
    +
    +
    + Próxima ação + {esc(primary)} +
    {esc(detail)}
    +
    + {blockers_html} +
    + {esc(primary)} +
    + {chatwoot_button} + {opportunity_button} +
    +
    +
    + """ + +def _render_work_group(label: str, items: list[dict]) -> str: + if not items: + return "" + cards = "".join(_render_work_item(item) for item in items) + return f'
    {esc(label)}
    {cards}
    ' + + +def render_operations_work_items(model: dict) -> str: + queue_html = _render_work_group("Prioridade alta", model.get("high_items") or []) + _render_work_group("Normal", model.get("normal_items") or []) + if not queue_html: + queue_html = '
    Sem trabalho pendente neste filtro.
    Quando houver ações humanas ou bloqueios concretos, aparecem aqui.
    ' + return f''' +
    +
    + A atualizar… +
    + {queue_html} +
    + ''' + + +def render_operations_filterbar(model: dict) -> str: + scope = model.get("scope") or "all" + links = [] + for key, label, href, partial_href in model.get("filters") or []: + if key == "done": + links.append(f'{esc(label)}') + continue + active = "active" if key == scope else "" + links.append( + f'{esc(label)}' + ) + return "".join(links) + + +@router.get("/operations/partials/work-items", response_class=HTMLResponse) +async def operations_work_items_partial(scope: str = "all"): + model = build_operations_view_model(scope=scope, limit=30) + return HTMLResponse(render_operations_work_items(model)) + + +@router.get("/operations", response_class=HTMLResponse) +@router.get("/operacoes", response_class=HTMLResponse) +async def operations_page(request: Request, scope: str = "all"): + # Daily work queue for the operator. This is not a mini-dashboard. + # Design note: antiga "Fila operacional priorizada" passa a lista única. Mensagens para revisão e Mensagens sem cliente continuam como critérios operacionais internos. + model = build_operations_view_model(scope=scope, limit=30) + counts = model.get("counts") or {} + filter_html = render_operations_filterbar(model) + queue_html = render_operations_work_items(model) + if is_htmx_request(request): + return HTMLResponse(queue_html) + + body = f''' +
    + O Chatwoot é a inbox. O Centro de trabalho mostra só o próximo trabalho humano: responder, corrigir, confirmar, reprocessar ou validar associação. Não é um mini-dashboard técnico. +
    + +
    +
    A fazer agora{esc(counts.get('work_queue_total', 0))}
    +
    Atrasadas{esc(model.get('overdue_total', counts.get('overdue_tasks', 0)))}
    +
    Bloqueadas{esc(model.get('blocked_total', 0))}
    +
    Associações por confirmar{esc(model.get('ambiguous_total', 0))}
    +
    + +
    +
    +
    +
    +

    Centro de trabalho

    +
    Hoje, {esc(model.get('today_label'))} · pergunta principal: o que tenho de fazer agora?
    +
    + Ver lista completa de tarefas +
    +
    {filter_html}
    +
    +
    + + {queue_html} + ''' + return layout("Centro de trabalho", "Lista única de trabalho do operador", body, "operations") diff --git a/app/admin_ui/pages/opportunities.py b/app/admin_ui/pages/opportunities.py new file mode 100644 index 0000000..a202c2c --- /dev/null +++ b/app/admin_ui/pages/opportunities.py @@ -0,0 +1,1282 @@ +"""Commercial opportunity routes and actions. + +Moved from app.admin_dashboard in v4.7.2. The handlers still reuse +legacy helpers to keep this refactor behavior-preserving. +""" +from fastapi import APIRouter, Request +import app.admin_dashboard as legacy +from app.admin_dashboard import * # noqa: F401,F403 +from app.admin_ui.labels import primary_action_label +from app.operation_noise import is_noise_operation_item +from app.opportunity_next_action_service import get_opportunity_next_action +from app.admin_ui.guidance import ( + blocker_alert_html, + fiscal_contact_inline_html, + fiscal_contact_panel_html, + fiscal_customer_missing_fields, + opportunity_blockers, + opportunity_context_customer, + readiness_checklist_html, + shipment_missing_fields, + stage_requires_fiscal_customer, +) + +_opportunity_board_column_for_stage = legacy._opportunity_board_column_for_stage + +router = APIRouter() + + +def _render_email_identity_review(opportunity_id: str, linked_customer: dict | None) -> str: + try: + from app.fiscal_enrichment_service import email_identity_review_for_opportunity + review = email_identity_review_for_opportunity(opportunity_id, refresh=False) + except Exception as exc: + return f""" +
    +
    Identidade do email
    +
    Erro ao ler identidade extraída: {esc(exc)}
    +
    + """ + if not review.get("ok") or not review.get("identity"): + return f""" +
    +
    Identidade do email
    +
    Ainda não existe identidade extraída para esta oportunidade.
    +
    + +
    +
    + """ + + identity = review.get("identity") or {} + companies = review.get("valid_company_mentions") or identity.get("company_mentions") or [] + phones = identity.get("phones") or [] + evidence = identity.get("evidence") or [] + conflict = bool(review.get("conflict")) + suggested = review.get("suggested_internal_customer") or {} + model = (identity.get("raw_payload") or {}).get("llm_model") if isinstance(identity.get("raw_payload"), dict) else identity.get("llm_model") + model = model or identity.get("llm_model") or "—" + confidence = identity.get("confidence") + try: + confidence_value = float(confidence or 0) + confidence_text = f"{confidence_value * 100:.0f}%" if confidence_value <= 1 else f"{confidence_value:.0f}%" + except Exception: + confidence_text = "—" + company_html = "".join(f'{esc(c)}' for c in companies) or 'Sem empresa explícita válida' + phone_html = ", ".join(esc(p) for p in phones) if phones else "—" + evidence_html = "".join(f'
  • {esc(compact_text(e, 90))}
    • ' for e in evidence[:3]) + conflict_html = "" + if conflict: + conflict_html = f""" +
    + Possível conflito fiscal.
    + O email menciona {esc(', '.join(companies) or 'outra empresa')}, mas a oportunidade está ligada a {esc(review.get('linked_customer_name') or 'outro cliente')}. +
    + """ + suggested_html = "" + if suggested and companies: + suggested_html = f""" +
    +
    Cliente interno compatível
    + {esc(suggested.get('nome') or suggested.get('name') or 'Cliente')} +
    NIF {esc(suggested.get('nif') or suggested.get('tax_id') or '—')}
    +
    + """ + return f""" +
    +
    +
    +
    Identidade extraída do email
    +
    {esc(identity.get('extraction_method') or identity.get('method') or '—')} · {esc(model)} · confiança {esc(confidence_text)}
    +
    + {status_badge('conflito') if conflict and 'status_badge' in globals() else ''} +
    + {conflict_html} +
    Pessoa
    +
    {esc(identity.get('person_name') or '—')}
    +
    Empresa mencionada
    +
    {company_html}
    +
    Email / domínio
    +
    {esc(identity.get('email') or '—')} · {esc(identity.get('domain') or '—')}
    +
    Morada
    +
    {esc(identity.get('address') or '—')}
    +
    Telefones
    +
    {phone_html}
    + {suggested_html} + {f'' if evidence_html else ''} +
    +
    + +
    +
    + +
    +
    + +
    +
    +
    + """ + + + +def _local_normalize_fiscal_name(value: object) -> str: + text = " ".join(str(value or "").strip().casefold().replace(",", " ").replace(".", " ").split()) + legal = {"lda", "ltd", "sa", "s", "a", "unipessoal", "limitada", "sociedade", "portugal"} + return " ".join(token for token in text.split() if token not in legal) + +def _render_fiscal_suggestions(opportunity_id: str, linked_customer: dict | None) -> str: + try: + from app.fiscal_enrichment_service import list_fiscal_suggestions_for_opportunity + suggestions = list_fiscal_suggestions_for_opportunity(opportunity_id, limit=3) + except Exception: + suggestions = [] + if linked_customer and not suggestions: + return "" + if not suggestions: + return f""" +
    +
    Sem sugestão fiscal externa registada.
    +
    + +
    +
    + """ + rows = "" + linked_name_norm = _local_normalize_fiscal_name(linked_customer.get("name") if linked_customer else "") + linked_tax_id = str((linked_customer or {}).get("tax_id") or "").strip() + linked_customer_id = str((linked_customer or {}).get("id") or "").strip() + visible_suggestions = [] + for suggestion in suggestions: + status = str(suggestion.get("status") or "pending") + lookup_value = str(suggestion.get("lookup_value") or "").strip().lower() + suggested_nif = str(suggestion.get("suggested_nif") or "").strip() + suggested_name_norm = _local_normalize_fiscal_name(suggestion.get("suggested_name")) + suggested_customer_id = str(suggestion.get("suggested_customer_id") or "").strip() + if lookup_value in {"pt", "com", "net", "org", "www", "http", "https", "mail", "email"}: + continue + # Do not show old accepted suggestions that merely confirm the current fiscal customer. + # The fiscal card already shows the truth; repeating an accepted suggestion with stale + # suggested_nif=NULL is confusing. + same_current_customer = bool( + linked_customer + and status == "accepted" + and ( + (suggested_customer_id and linked_customer_id and suggested_customer_id == linked_customer_id) + or (linked_name_norm and suggested_name_norm and linked_name_norm == suggested_name_norm) + or (linked_tax_id and suggested_nif and linked_tax_id == suggested_nif) + ) + ) + if same_current_customer: + continue + visible_suggestions.append(suggestion) + for suggestion in visible_suggestions: + sid = str(suggestion.get("id") or "") + status = str(suggestion.get("status") or "pending") + badge = status_badge(status) if "status_badge" in globals() else f"{esc(status)}" + confidence = suggestion.get("confidence") + if confidence is not None: + try: + confidence_value = float(confidence) + confidence_text = f"{confidence_value * 100:.0f}%" if confidence_value <= 1 else f"{confidence_value:.0f}%" + except Exception: + confidence_text = "—" + else: + confidence_text = "—" + actions = "" + if status == "pending" and sid: + actions = f""" +
    +
    + +
    +
    + +
    +
    + """ + rows += f""" +
    +
    {esc(suggestion.get('suggested_name') or 'Empresa sugerida')}{badge}
    +
    Sugestão fiscal · NIF {esc(suggestion.get('suggested_nif') or '—')} · confiança {esc(confidence_text)}
    +
    {esc(suggestion.get('match_type') or suggestion.get('lookup_type') or 'match')}
    + {actions} +
    + """ + if not rows.strip(): + return "" + return f""" +
    +
    Sugestões fiscais por validar
    +
    Não é cliente fiscal confirmado. Associar apenas depois de validar nome/NIF.
    + {rows} +
    + +
    +
    + """ + + +def _jasmin_candidate_tax_conflict_message(opportunity_id: str, item_id: str) -> str: + """Return a blocking message when a Jasmin candidate belongs to another NIF.""" + try: + from app.commercial_service import get_customer_for_opportunity, normalize_tax_id + from app.jasmin_backfill_service import find_jasmin_document_candidates_for_opportunity + + linked_customer = get_customer_for_opportunity(opportunity_id) + linked_tax_id = normalize_tax_id((linked_customer or {}).get("tax_id")) + if not linked_tax_id: + return "" + for item in find_jasmin_document_candidates_for_opportunity(opportunity_id, limit=50): + if str(item.get("id") or "") != str(item_id): + continue + candidate_tax = normalize_tax_id(item.get("customer_tax_id")) + if candidate_tax and candidate_tax != linked_tax_id: + return ( + "NIF divergente: o documento Jasmin pertence a outro cliente fiscal. " + "Rever manualmente na reconciliação antes de associar/substituir." + ) + return "" + except Exception: + # Não bloquear quando não conseguimos confirmar conflito; o serviço de importação + # continua responsável por validar a operação. + return "" + return "" + + +def _opportunity_jasmin_state(opportunity_id: str) -> dict: + # Small UI helper: summarize current Jasmin evidence imported in ClientFlow. + try: + from sqlalchemy import text + from app.db import engine + with engine.begin() as conn: + row = conn.execute(text(""" + SELECT + COUNT(*) FILTER (WHERE system = 'jasmin')::int AS jasmin_documents, + COUNT(*) FILTER (WHERE system = 'jasmin' AND document_kind = 'quotation')::int AS quotations, + COUNT(*) FILTER (WHERE system = 'jasmin' AND document_kind = 'proforma')::int AS proformas, + COUNT(*) FILTER (WHERE system = 'jasmin' AND document_kind = 'invoice')::int AS invoices, + (ARRAY_AGG(document_number ORDER BY COALESCE(document_date, created_at::date) DESC, created_at DESC) FILTER (WHERE system = 'jasmin'))[1] AS current_document_number, + (ARRAY_AGG(document_kind ORDER BY COALESCE(document_date, created_at::date) DESC, created_at DESC) FILTER (WHERE system = 'jasmin'))[1] AS current_document_kind, + (ARRAY_AGG(total_amount ORDER BY COALESCE(document_date, created_at::date) DESC, created_at DESC) FILTER (WHERE system = 'jasmin'))[1] AS current_total_amount + FROM commercial_documents + WHERE opportunity_id = CAST(:opportunity_id AS UUID) + """), {"opportunity_id": str(opportunity_id)}).mappings().first() + item_count = conn.execute(text(""" + SELECT COUNT(*)::int + FROM opportunity_items + WHERE opportunity_id = CAST(:opportunity_id AS UUID) + """), {"opportunity_id": str(opportunity_id)}).scalar() or 0 + data = dict(row or {}) + data["item_count"] = int(item_count or 0) + return data + except Exception: + return {"jasmin_documents": 0, "item_count": 0} + + +def _opportunity_consistency_alert_html(opportunity: dict, tasks: list[dict], opportunity_items: list[dict], opportunity_id: str) -> str: + # Surface soft inconsistencies without blocking the operator. + state = _opportunity_jasmin_state(opportunity_id) + stage = str(opportunity.get("stage") or "") + pending_action_codes = {str(t.get("action_code") or "") for t in tasks if str(t.get("status") or "") == "pending"} + has_payment_task = bool({"CONFIRM_PAYMENT", "CONFIRM_PAYMENT_AND_PREPARE_SHIPMENT"} & pending_action_codes) + has_quote = int(state.get("quotations") or 0) > 0 + has_proforma = int(state.get("proformas") or 0) > 0 + has_invoice = int(state.get("invoices") or 0) > 0 + has_items = bool(opportunity_items) or int(state.get("item_count") or 0) > 0 + + alerts = [] + if has_payment_task and has_quote and not (has_proforma or has_invoice): + alerts.append( + "Existe tarefa de confirmar pagamento, mas o documento Jasmin atual ainda é orçamento. " + "Antes de concluir a tarefa, confirma que o cliente recebeu pedido de pagamento/pró-forma ou que o pagamento foi efetivamente indicado." + ) + if stage == "WAITING_PAYMENT" and has_quote and not (has_proforma or has_invoice): + alerts.append( + "A fase está em pagamento com apenas orçamento Jasmin importado. Isto pode estar correto se o cliente já aceitou/pagou, " + "mas a fase documental ainda não mostra pró-forma/fatura." + ) + if has_items and int(state.get("jasmin_documents") or 0) <= 0: + alerts.append( + "A oportunidade tem produtos, mas ainda não tem documento Jasmin importado. Usa Reimportar detalhes ou Criar orçamento." + ) + + if not alerts: + return "" + items = "".join(f"
  • {esc(a)}
    • " for a in alerts[:3]) + return f''' +
    + Verificação de consistência operacional + +
    + ''' + + +def _derived_timeline_html(opportunity_id: str) -> str: + # Fallback timeline based on current documents/items/tasks when no audit events exist. + try: + from sqlalchemy import text + from app.db import engine + with engine.begin() as conn: + docs = conn.execute(text(""" + SELECT document_kind, document_number, total_amount, status, created_at + FROM commercial_documents + WHERE opportunity_id = CAST(:opportunity_id AS UUID) + ORDER BY created_at DESC + LIMIT 3 + """), {"opportunity_id": str(opportunity_id)}).mappings().all() + item_count = conn.execute(text(""" + SELECT COUNT(*)::int + FROM opportunity_items + WHERE opportunity_id = CAST(:opportunity_id AS UUID) + """), {"opportunity_id": str(opportunity_id)}).scalar() or 0 + except Exception: + docs, item_count = [], 0 + + items = "" + for doc in docs: + title = "Documento Jasmin importado" + detail = f"{doc.get('document_number') or 'documento'} · {money_html(doc.get('total_amount') or 0)}" + items += f''' +
    +
    {esc(fmt_dt(doc.get('created_at')))}
    derivado
    +
    {esc(title)}{operation_status_badge(str(doc.get('status') or 'created'))}
    {esc(detail)}
    +
    + ''' + if item_count and not docs: + items += f''' +
    +
    —
    derivado
    +
    Produtos na oportunidade
    {esc(item_count)} linha(s) comerciais associadas.
    +
    + ''' + return items + +def _opportunity_query_string(q: Optional[str] = None, status: Optional[str] = "open", scope: Optional[str] = "all", limit: int = 300) -> str: + parts = [] + if q: + parts.append(f"q={esc(q)}") + if status and status != "open": + parts.append(f"status={esc(status)}") + if scope and scope != "all": + parts.append(f"scope={esc(scope)}") + if limit and int(limit) != 300: + parts.append(f"limit={int(limit)}") + return ("?" + "&".join(parts)) if parts else "" + + +def _opportunity_visible_set(q: Optional[str] = None, status: Optional[str] = "open", scope: Optional[str] = "all", limit: int = 300) -> tuple[list[dict], dict, list[tuple[str, str, object]]]: + if (status or "open") == "closed": + status = "open" + opportunities = list_opportunities(q=q, status=status or "open", limit=limit) + visible_board_columns = [column for column in OPPORTUNITY_BOARD_COLUMNS if column[0] != "closed"] + grouped = {key: [] for key, _label, _stages in visible_board_columns} + visible = [] + for opportunity in opportunities: + if _is_noise_opportunity(opportunity): + continue + key = _opportunity_board_column_for_opportunity(opportunity) + if key == "closed": + continue + if scope and scope not in {"all", "open"}: + if scope == "blocked": + pending = int(opportunity.get("pending_task_count") or 0) + if pending <= 0 and not opportunity_customer_mismatch(opportunity): + continue + elif key != scope: + continue + visible.append(opportunity) + grouped.setdefault(key, []).append(opportunity) + return visible, grouped, visible_board_columns + + +def _compact_identity(value: object) -> str: + value = compact_text(str(value or "").strip(), 42) + if value.casefold() in {"", "geral", "cliente", "contacto"} or value.isdigit(): + return "" + return value + + +def _opportunity_card_identity(opp: dict) -> tuple[str, str]: + fiscal = _compact_identity(opp.get("linked_customer_name")) + contact_name = _compact_identity(opp.get("customer_name")) + contact_email = _compact_identity(opp.get("customer_email")) + if fiscal: + subtitle = contact_email or contact_name + return fiscal, (f"Contacto: {subtitle}" if subtitle and subtitle != fiscal else "") + if contact_name: + return contact_name, contact_email if contact_email and contact_email != contact_name else "" + if contact_email: + return contact_email, "" + conversation = str(opp.get("conversation_id") or "").strip() + return "Contacto sem identificação", (f"Conversa Chatwoot #{conversation}" if conversation else "") + + +# Legacy regression context: cta_label = "Concluir tarefa pendente" if pending else "Ver oportunidade". +# v4.8.5 replaces that generic CTA with a specific action label. +def _opportunity_card_next_action(opp: dict) -> str: + if int(opp.get("pending_task_count") or 0) > 0: + action_code = str(opp.get("last_action_code") or "").strip() + return primary_action_label(action_code, fallback="Ver tarefa pendente") + return opportunity_next_action_text(opp) + + +def _is_noise_opportunity(opp: dict) -> bool: + """Hide old bounce/NDR opportunities from the commercial board. + + Operations already hides technical mailbox noise; the opportunity board must + use the same guard so legacy Mail Delivery/postmaster opportunities do not + keep appearing as commercial work. + """ + return is_noise_operation_item({ + "customer_name": opp.get("customer_name"), + "contact_display_name": opp.get("customer_name"), + "fiscal_customer_name": opp.get("linked_customer_name"), + "message_subject": opp.get("product_interest"), + "title": opp.get("title"), + "detail": opp.get("product_interest"), + "request_text": (opp.get("metadata") or {}).get("request_text") if isinstance(opp.get("metadata"), dict) else "", + "source_system": opp.get("source_system"), + "action_code": opp.get("last_action_code"), + "no_opportunity_reason": (opp.get("metadata") or {}).get("no_opportunity_reason") if isinstance(opp.get("metadata"), dict) else "", + "status": opp.get("status"), + }) + + +def _opportunity_board_column_for_opportunity(opp: dict) -> str: + """Choose a visual board column from stage plus next pending action. + + The stored stage remains unchanged. This only avoids showing opportunities + with a financial/logistics next step under the initial "Pedidos" column. + """ + action_code = str(opp.get("last_action_code") or "").upper().strip() + if int(opp.get("pending_task_count") or 0) > 0: + if action_code in {"SEND_INVOICE", "SEND_PROFORMA", "CONFIRM_PAYMENT"}: + return "payment" + if action_code in {"PREPARE_ORDER", "CREATE_SHIPMENT"}: + return "operations" + return _opportunity_board_column_for_stage(opp.get("stage")) + + +def _render_opportunity_card(opp: dict) -> str: + oid = str(opp.get("id") or "") + title, subtitle = _opportunity_card_identity(opp) + subject = compact_text(opp.get("product_interest") or opp.get("title") or "Pedido comercial", 64) + next_action = compact_text(_opportunity_card_next_action(opp), 72) + pending = int(opp.get("pending_task_count") or 0) + blockers = opportunity_blockers(opp) + cta_label = next_action if pending else "Ver oportunidade" + cta_class = "btn-primary" if pending else "btn-outline-primary" + blocker_html = blocker_alert_html(blockers, empty_text="") if blockers else "" + subtitle_html = f'
    {esc(subtitle)}
    ' if subtitle else "" + blocker_class = " has-blocker" if blockers else "" + return f""" +
    + +
    {esc(title)}
    + {subtitle_html} +
    {esc(subject)}
    + {blocker_html} +
    + Próxima ação + {esc(next_action)} +
    +     + {esc(cta_label)} +
    + """ + + +def render_opportunities_board_partial(q: Optional[str] = None, status: Optional[str] = "open", scope: Optional[str] = "all", limit: int = 300) -> str: + visible_opportunities, grouped, visible_board_columns = _opportunity_visible_set(q=q, status=status, scope=scope, limit=limit) + board_html = "" + for key, label, _stages in visible_board_columns: + cards = "".join(_render_opportunity_card(opp) for opp in grouped.get(key, [])) + if not cards: + cards = '
    Sem oportunidades nesta etapa.
    ' + board_html += f""" +
    +
    + {esc(label)} + {len(grouped.get(key, []))} +
    +
    {cards}
    +
    + """ + return f""" +
    +
    + {len(visible_opportunities)} resultado(s) + A atualizar… +
    +
    +
    {board_html}
    +
    +
    + """ + + +@router.get("/opportunities/partials/board", response_class=HTMLResponse) +async def opportunities_board_partial(q: Optional[str] = None, status: Optional[str] = "open", scope: Optional[str] = "all", limit: int = 300): + return HTMLResponse(render_opportunities_board_partial(q=q, status=status, scope=scope, limit=limit)) + + +@router.get("/opportunities", response_class=HTMLResponse) +@router.get("/oportunidades", response_class=HTMLResponse) +async def opportunities_page( + request: Request, + q: Optional[str] = None, + status: Optional[str] = "open", + scope: Optional[str] = "all", + limit: int = 300, +): + # Quadro operacional em Bootstrap 5. v4.7.4 adds an HTMX board partial + # while preserving the same opportunity query and card semantics. + if (status or "open") == "closed": + status = "open" + + visible_opportunities, grouped, visible_board_columns = _opportunity_visible_set(q=q, status=status, scope=scope, limit=limit) + total_open = sum(1 for opp in visible_opportunities if str(opp.get("status") or "") == "open") + total_pending = sum(int(opp.get("pending_task_count") or 0) for opp in visible_opportunities) + total_value = sum(float(opp.get("value_amount") or 0) for opp in visible_opportunities) + attention = [opp for opp in visible_opportunities if int(opp.get("pending_task_count") or 0) > 0] + + if is_htmx(request): + return HTMLResponse(render_opportunities_board_partial(q=q, status=status, scope=scope, limit=limit)) + + status_options = "" + for value, label in [("open", "Abertas"), ("all", "Todas")]: + selected = "selected" if (status or "open") == value else "" + status_options += f'' + + stage_tabs = "" + filters = [("all", "Todas"), ("new", "Novas"), ("quote", "Orçamento enviado"), ("proforma", "Pró-forma enviada"), ("payment", "Pagamento pendente"), ("shipment", "Enviadas"), ("blocked", "Bloqueadas")] + for key, label in filters: + href = "/opportunities" + _opportunity_query_string(q=q, status=status, scope=key, limit=limit) + partial_href = "/opportunities/partials/board" + _opportunity_query_string(q=q, status=status, scope=key, limit=limit) + active = "btn-primary" if (scope or "all") == key else "btn-outline-secondary" + stage_tabs += f'{esc(label)}' + + body = f""" +
    +
    Abertas
    {total_open}
    em acompanhamento
    +
    Com tarefa
    {len(attention)}
    requerem ação
    +
    Tarefas pendentes
    {total_pending}
    ligadas a vendas
    +
    Valor estimado
    {money_html(total_value)}
    lista atual
    +
    + +
    Limpar
    + +
    Filtros:{stage_tabs}
    + +
    +
    +
    +

    Quadro de oportunidades

    Cards por etapa, com identificação clara, assunto, próxima ação e bloqueios relevantes. Filtros atualizam por HTMX.
    + {len(visible_opportunities)} resultado(s) +
    +
    {render_opportunities_board_partial(q=q, status=status, scope=scope, limit=limit)}
    +
    +
    + """ + return layout("Oportunidades", "Pipeline comercial com foco na próxima ação", body, active="opportunities") + + +@router.get("/opportunities/{opportunity_id}", response_class=HTMLResponse) +async def opportunity_detail_page(opportunity_id: str, notice: Optional[str] = None): + opportunity = get_opportunity(opportunity_id) + if not opportunity: + return layout("Oportunidade não encontrada", "Pipeline comercial", '
    Oportunidade não encontrada.
    ', "opportunities") + + tasks = list_opportunity_tasks(opportunity_id, limit=100) + events = list_opportunity_events(opportunity_id, limit=100) + stage = str(opportunity.get("stage") or "NEW_LEAD") + pending_tasks = [t for t in tasks if str(t.get("status")) == "pending"] + next_task = pending_tasks[0] if pending_tasks else None + opportunity_items = list_opportunity_items(opportunity_id) + active_products = list_products(active="true", limit=200) + try: + from app.commercial_service import list_commercial_documents + linked_documents = list_commercial_documents(opportunity_id=opportunity_id, limit=8) + except Exception: + linked_documents = [] + primary_document = next( + ( + doc for doc in linked_documents + if str(doc.get("document_kind") or "") == "invoice" + and str(doc.get("role") or "current") in {"current", "accepted"} + and bool(doc.get("is_primary", True)) + ), + next( + ( + doc for doc in linked_documents + if str(doc.get("role") or "current") in {"current", "accepted"} + and bool(doc.get("is_primary", True)) + ), + linked_documents[0] if linked_documents else None, + ), + ) + opportunity_items_total = sum( + float(item.get("total_price") or 0) + for item in opportunity_items + if str(item.get("status") or "").upper() not in {"REJECTED", "CANCELLED", "DELIVERED", "HISTORICAL"} + ) + document_value = float(primary_document.get("total_amount") or primary_document.get("amount") or 0) if primary_document else 0 + estimated_value = document_value or opportunity_items_total or float(opportunity.get("value_amount") or 0) + value_source = "documento principal" if document_value else ("linhas atuais" if opportunity_items_total else "oportunidade") + operation_snapshot = get_operation_snapshot(opportunity_id) + try: + opportunity_communications = list_communications_for_opportunity(opportunity_id, limit=12) + except Exception: + opportunity_communications = [] + notice_html = f'
    {esc(notice)}
    ' if notice else '' + metadata = opportunity.get("metadata") if isinstance(opportunity.get("metadata"), dict) else {} + record_mode = str(metadata.get("clientflow_record_mode") or "") + legacy_mode = record_mode in {"reconstructed_invoice_review", "historical_reconstructed", "legacy_review"} + legacy_notice_html = "" + if legacy_mode: + legacy_notice_html = ( + '
    ' + 'Registo antigo/reconstruído.
    ' + 'A oportunidade foi normalizada a partir de documentos já existentes. ' + 'Valida pagamento, valor e linhas antes de executar novas ações.' + '
    ' + ) + + try: + next_action = get_opportunity_next_action(opportunity_id) + except Exception: + next_action = {} + if next_action: + primary_action = next_action.get("label") or action_label(next_action.get("action_code")) + primary_note = next_action.get("description") or "Continuar a próxima ação recomendada." + target_url = next_action.get("target_url") or (f"/tasks/{next_task.get('id')}" if next_task else "/tasks?status=pending") + button_label = "Abrir tarefa" if str(target_url).startswith("/tasks/") else "Continuar" + if next_action.get("action_code") == "VALIDATE_FISCAL_CUSTOMER": + primary_button = f'
    ' + else: + primary_button = f'{esc(button_label)}' + elif next_task: + primary_action = action_label(next_task.get("action_code")) + primary_note = next_task.get("note") or next_task.get("action") or "Abrir tarefa pendente para continuar." + primary_button = f'Abrir tarefa' + else: + primary_action = opportunity_next_action_text(opportunity) + primary_note = "Não existe tarefa pendente ligada. Atualiza o estado ou acompanha a oportunidade." + primary_button = 'Ver tarefas' + + task_rows = "" + for task in tasks[:8]: + task_rows += f''' + + {esc(action_label(task.get('action_code')))}
    {esc(compact_text(task.get('note') or task.get('action') or '', 70))}
    + {route_badge(task.get('route'))} + {status_badge(task.get('status'))} + {esc(fmt_dt(task.get('created_at')))} + + ''' + if not task_rows: + task_rows = 'Sem tarefas associadas.' + + communication_rows = "" + for communication in opportunity_communications: + action = classification_action(communication.get("classification")) + communication_rows += f''' + + {esc(communication.get('subject') or 'Sem assunto')}
    {esc(communication.get('sender_name') or communication.get('sender_email') or '—')}
    + {esc(communication.get('classification') or 'por classificar')} + {status_badge(communication.get('status'))} + {esc(fmt_dt(communication.get('created_at')))} + + ''' + if not communication_rows: + conv = str(opportunity.get("conversation_id") or "").strip() + if conv: + communication_rows = f''' + + Conversa Chatwoot #{esc(conv)}
    Ainda não há mensagens indexadas/ligadas nesta oportunidade.
    + por sincronizar + sem ligação local + — + + ''' + else: + communication_rows = 'Sem comunicações associadas à oportunidade.' + + timeline_items = "" + try: + unified_timeline = list_unified_opportunity_timeline(opportunity_id, limit=14) + except Exception: + unified_timeline = [] + for event in unified_timeline: + status = event.get("status") + status_html = status_badge(status) if status else "" + source = event.get("source") or "event" + detail = compact_text(event.get("detail") or "", 140) + timeline_items += f''' +
    +
    {esc(fmt_dt(event.get('created_at')))}
    {esc(source)}
    +
    {esc(event.get('title') or 'Evento')}{status_html}
    {esc(detail or '—')}
    +
    + ''' + if not timeline_items: + timeline_items = _derived_timeline_html(opportunity_id) + if not timeline_items: + timeline_items = '
    Sem eventos registados.
    ' + + stage_options = "" + for value, label in OPPORTUNITY_STAGE_LABELS.items(): + selected = "selected" if value == opportunity.get("stage") else "" + stage_options += f'' + + customer_name = opportunity_customer_name(opportunity) + contact_name = opportunity_contact_name(opportunity) + customer_email = opportunity.get("customer_email") or "" + customer_phone = opportunity.get("customer_phone") or "" + conversation = opportunity.get("conversation_id") or "—" + # v4.6.2: não mostrar aviso por divergência de nome. Contacto pessoal e + # cliente fiscal/empresa podem ser diferentes e ainda assim estar corretos. + customer_mismatch_alert = "" + + linked_customer = None + customer_options = '' + try: + from app.commercial_service import get_customer_for_opportunity, list_customers + linked_customer = get_customer_for_opportunity(opportunity_id) + for c in list_customers(limit=150): + selected = "selected" if linked_customer and str(c.get("id")) == str(linked_customer.get("id")) else "" + label = f"{c.get('name') or 'Cliente'} · {c.get('tax_id') or 'sem NIF'}" + customer_options += f'' + except Exception: + linked_customer = None + + fiscal_suggestions_html = _render_fiscal_suggestions(opportunity_id, linked_customer) + email_identity_html = _render_email_identity_review(opportunity_id, linked_customer) + + fiscal_customer = opportunity_context_customer(opportunity, linked_customer) + fiscal_customer_href = f"/customers/{esc(fiscal_customer.get('id'))}" if fiscal_customer and fiscal_customer.get("id") else "" + fiscal_contact_html = fiscal_contact_panel_html( + fiscal_customer=fiscal_customer, + contact_name=contact_name, + contact_email=customer_email, + contact_phone=customer_phone, + conversation_id=opportunity.get("conversation_id"), + contact_id=opportunity.get("contact_id"), + customer_href=fiscal_customer_href, + ) + next_action_code = (next_action.get("action_code") if isinstance(next_action, dict) else None) or opportunity.get("last_action_code") + current_blockers = opportunity_blockers(opportunity, linked_customer, action_code=next_action_code) + document_already_issued = bool( + primary_document + or linked_documents + or stage in {"QUOTE_SENT", "PROFORMA_SENT", "INVOICE_SENT", "WAITING_PAYMENT", "PAYMENT_CONFIRMED", "WON"} + ) + blockers_html = ( + '
    Avisos para revisão' + + '
    Existe documento emitido/ligado; estes dados devem ser revistos para próximos documentos ou correção administrativa.
    ' + if current_blockers and document_already_issued + else blocker_alert_html(current_blockers) + ) + fiscal_readiness_html = readiness_checklist_html( + title="Prontidão para documentos", + missing=fiscal_customer_missing_fields(fiscal_customer), + ok_text="Cliente fiscal pronto para orçamento, pró-forma ou fatura.", + blocked_text=("Dados fiscais incompletos no ClientFlow; rever para próximos documentos." if document_already_issued else "Dados fiscais incompletos no ClientFlow; rever antes de emitir novo documento."), + ) + shipment_readiness_html = readiness_checklist_html( + title="Prontidão para envio", + missing=shipment_missing_fields(fiscal_customer, opportunity), + ok_text="Dados mínimos de envio completos.", + blocked_text="Envio deve aguardar correção destes dados.", + ) + consistency_alert_html = _opportunity_consistency_alert_html(opportunity, tasks, opportunity_items, opportunity_id) + if primary_document: + document_label = commercial_document_display_number(primary_document, fallback="número por atualizar") + document_kind = { + "quotation": "Orçamento", + "proforma": "Pró-forma", + "invoice": "Fatura", + }.get(str(primary_document.get("document_kind") or ""), "Documento") + document_state = f"{document_kind} · {document_label}" + document_chip = 'ligado' + else: + document_state = "Sem documento principal" + document_chip = 'pendente' + fiscal_state = (linked_customer.get("name") if linked_customer else "Por associar") + fiscal_chip = 'validado' if linked_customer else 'bloqueia documentos' + task_state = f"{len(pending_tasks)} pendente(s)" if pending_tasks else "Sem tarefas pendentes" + task_chip = 'requer ação' if pending_tasks else 'limpo' + operator_summary_html = f''' +
    +
    +
    +

    Mapa operacional

    Leitura rápida do processo: cliente fiscal, documento principal, task e próxima ação.
    + Ver reconciliação +
    +
    +
    Cliente fiscal{esc(fiscal_state)}{fiscal_chip}
    +
    Documento principal{esc(document_state)}{document_chip}
    +
    Tasks{esc(task_state)}{task_chip}
    +
    Decisão seguinte{esc(primary_action)}{esc(next_action.get('action_code') or opportunity.get('last_action_code') or 'FOLLOW_UP')}
    +
    +
    + Ações avançadas +
    + Documentos Jasmin + Detalhes técnicos + Tasks desta oportunidade +
    +
    +
    +
    + ''' + + technical_html = f''' +
    +
    ID
    {esc(opportunity_id)}
    +
    Conversa
    {esc(conversation)}
    +
    Última action
    {esc(opportunity.get('last_action_code') or '—')}
    +
    Atualizada
    {esc(fmt_dt(opportunity.get('updated_at')))}
    +
    + ''' + + # "Bloqueios atuais" permanece como conceito de UI/teste, mas o título duplicado foi removido. + body = f''' + + + ← Voltar a oportunidades + {notice_html} + {legacy_notice_html} + {customer_mismatch_alert} + {consistency_alert_html} + + +
    +
    +
    Oportunidade

    {esc(opportunity.get('title') or 'Oportunidade')}

    {esc(customer_name)} · {esc(opportunity.get('product_interest') or 'Interesse por definir')}
    {opportunity_stage_badge(stage)}{opportunity_priority_chip(opportunity)}
    Próxima ação

    {esc(primary_action)}

    {esc(primary_note)}
    {primary_button}
    + + {operator_summary_html} + + {f'
    {blockers_html}
    ' if current_blockers else ''} + + {fiscal_contact_html} + +
    {fiscal_readiness_html}
    {shipment_readiness_html}
    + +

    Resumo essencial

    {'Valor principal' if document_value else ('Valor reconstruído' if legacy_mode else 'Valor estimado')}{money_html(estimated_value)}
    {esc(value_source)}
    Tarefas pendentes{len(pending_tasks)}
    Atualizada{esc(fmt_dt(opportunity.get('updated_at')))}
    + +

    Pipeline

    {stage_progress_html(stage)}
    + + {operation_cockpit_html(opportunity_id, opportunity, operation_snapshot)} + +
    {opportunity_integrations_panel_html(opportunity_id)}
    + +
    {jasmin_documents_html(opportunity_id)}
    + +
    {opportunity_products_panel_html(opportunity_id)}
    + +

    Tasks relacionadas

    Ações humanas já criadas para esta oportunidade.
    {task_rows}
    AçãoFilaEstadoCriada
    + +

    Mensagens Chatwoot

    Mensagens relevantes ligadas a esta oportunidade. A resposta continua no Chatwoot.
    {communication_rows}
    MensagemClassificaçãoEstadoRecebida
    + +

    Timeline recente

    {timeline_items}
    + +
    Ver detalhes técnicos e edição avançada
    {technical_html}
    +
    + + +
    + ''' + return layout(str(opportunity.get("title") or "Oportunidade"), "Detalhe comercial com informação essencial", body, "opportunities") + + +@router.get("/opportunities/{opportunity_id}/partials/jasmin-documents", response_class=HTMLResponse) +async def opportunity_jasmin_documents_partial(opportunity_id: str): + return HTMLResponse(jasmin_documents_html(opportunity_id)) + + +@router.get("/opportunities/{opportunity_id}/partials/products", response_class=HTMLResponse) +async def opportunity_products_partial(opportunity_id: str): + return HTMLResponse(opportunity_products_panel_html(opportunity_id)) + + +@router.post("/commercial-documents/{document_id}/refresh") +async def commercial_document_refresh(document_id: str, request: Request): + form = await request.form() + opportunity_id = str(form.get("opportunity_id") or "").strip() + try: + from app.jasmin_service import refresh_commercial_document_from_jasmin + await refresh_commercial_document_from_jasmin(document_id) + except Exception as exc: + if opportunity_id and is_htmx(request): + return HTMLResponse(jasmin_documents_html(opportunity_id, error_notice=f"Erro ao atualizar documento: {exc}"), status_code=409) + return PlainTextResponse(f"Erro ao atualizar documento: {exc}", status_code=500) + if opportunity_id and is_htmx(request): + return HTMLResponse(jasmin_documents_html(opportunity_id, notice="Documento atualizado a partir do Jasmin.")) + return RedirectResponse(f"/opportunities/{opportunity_id}" if opportunity_id else "/outbox", status_code=303) + + +@router.get("/commercial-documents/{document_id}/pdf") +async def commercial_document_pdf(document_id: str): + try: + from app.jasmin_service import get_commercial_document_pdf + doc, data, content_type = await get_commercial_document_pdf(document_id) + except Exception as exc: + return PlainTextResponse(f"Erro ao obter PDF Jasmin: {exc}", status_code=500) + name = doc.get("document_number") or doc.get("external_id") or document_id + safe_name = "".join(ch if ch.isalnum() or ch in "-_" else "_" for ch in str(name))[:80] or "documento" + headers = {"Content-Disposition": f'inline; filename="{safe_name}.pdf"'} + return Response(content=data, media_type=content_type or "application/pdf", headers=headers) + + +@router.post("/opportunities/{opportunity_id}/stage") +async def update_opportunity_stage_action(opportunity_id: str, request: Request): + form = await request.form() + stage = str(form.get("stage") or "").strip() + note = str(form.get("note") or "").strip() + if stage: + set_opportunity_stage(opportunity_id, stage, note=note, created_by="operator") + return RedirectResponse(f"/opportunities/{opportunity_id}", status_code=303) + + +@router.post("/opportunities/{opportunity_id}/customer") +async def opportunity_link_customer_action(opportunity_id: str, request: Request): + form = await request.form() + customer_id = str(form.get("customer_id") or "").strip() + try: + from app.commercial_service import link_customer_to_opportunity, unlink_customer_from_opportunity + if customer_id: + link_customer_to_opportunity(customer_id, opportunity_id) + else: + unlink_customer_from_opportunity(opportunity_id) + except Exception as exc: + return PlainTextResponse(f"Erro ao associar cliente: {exc}", status_code=500) + return RedirectResponse(f"/opportunities/{opportunity_id}", status_code=303) + + +@router.post("/opportunities/{opportunity_id}/email-identity/extract") +async def opportunity_email_identity_extract_action(opportunity_id: str, request: Request): + try: + from app.email_identity_extraction_service import extract_identity_for_opportunity + result = extract_identity_for_opportunity(opportunity_id, refresh=True, use_llm=True) + except Exception as exc: + return PlainTextResponse(f"Erro ao extrair identidade do email: {exc}", status_code=500) + if not result: + notice = "Sem mensagem associada para extrair identidade." + else: + companies = result.get("company_mentions") or [] + notice = "Identidade extraída" + (f": {', '.join(companies[:2])}" if companies else ".") + return RedirectResponse(f"/opportunities/{opportunity_id}?notice={esc(notice)}", status_code=303) + + +@router.post("/opportunities/{opportunity_id}/email-identity/assist") +async def opportunity_email_identity_assist_action(opportunity_id: str, request: Request): + try: + from app.fiscal_enrichment_service import assist_email_identity_enrichment + result = assist_email_identity_enrichment(opportunity_id, refresh=True, apply_safe=False) + except Exception as exc: + return PlainTextResponse(f"Erro ao procurar cliente fiscal por identidade: {exc}", status_code=500) + if result.get("conflict"): + notice = "Possível conflito fiscal detetado pela identidade extraída." + elif result.get("status") == "email_identity_matches_current_fiscal_customer": + notice = "Identidade extraída confirma o cliente fiscal atual." + elif result.get("suggested"): + notice = "Sugestão fiscal criada a partir da identidade extraída." + else: + notice = "Identidade extraída, mas sem cliente fiscal compatível encontrado." + return RedirectResponse(f"/opportunities/{opportunity_id}?notice={esc(notice)}", status_code=303) + + +@router.post("/opportunities/{opportunity_id}/email-identity/cleanup-invalid") +async def opportunity_email_identity_cleanup_invalid_action(opportunity_id: str, request: Request): + try: + from app.email_identity_cleanup_service import cleanup_invalid_email_identity_state + result = cleanup_invalid_email_identity_state( + opportunity_id=opportunity_id, + include_accepted=True, + fix_extractions=True, + apply=True, + ) + except Exception as exc: + return PlainTextResponse(f"Erro ao limpar identidade inválida: {exc}", status_code=500) + notice = ( + f"Limpeza de identidade: {result.get('rejected', 0)} sugestão(ões) rejeitada(s), " + f"{result.get('fixed_extractions', 0)} extração(ões) corrigida(s)." + ) + return RedirectResponse(f"/opportunities/{opportunity_id}?notice={esc(notice)}", status_code=303) + + +@router.post("/opportunities/{opportunity_id}/fiscal-enrich") +async def opportunity_fiscal_enrich_action(opportunity_id: str, request: Request): + try: + from app.fiscal_enrichment_service import enrich_opportunity + result = enrich_opportunity(opportunity_id, apply_safe=True) + except Exception as exc: + return PlainTextResponse(f"Erro ao enriquecer cliente fiscal: {exc}", status_code=500) + if result.get("auto_applied"): + notice = "Cliente fiscal auto-associado por enriquecimento." + elif result.get("suggested"): + notice = "Sugestão fiscal criada para revisão." + else: + notice = f"Sem sugestão fiscal: {result.get('reason') or 'sem correspondência'}" + return RedirectResponse(f"/opportunities/{opportunity_id}?notice={esc(notice)}", status_code=303) + + +@router.post("/fiscal-suggestions/{suggestion_id}/accept") +async def fiscal_suggestion_accept_action(suggestion_id: str, request: Request): + try: + from app.fiscal_enrichment_service import apply_fiscal_suggestion + result = apply_fiscal_suggestion(suggestion_id, actor="operator_ui") + except Exception as exc: + return PlainTextResponse(f"Erro ao aplicar sugestão fiscal: {exc}", status_code=500) + opportunity_id = result.get("opportunity_id") or "" + if not result.get("applied"): + return PlainTextResponse(f"Sugestão não aplicada: {result.get('reason')}", status_code=409) + return RedirectResponse(f"/opportunities/{esc(opportunity_id)}?notice=Sugest%C3%A3o%20fiscal%20aplicada", status_code=303) + + +@router.post("/fiscal-suggestions/{suggestion_id}/reject") +async def fiscal_suggestion_reject_action(suggestion_id: str, request: Request): + try: + from app.fiscal_enrichment_service import reject_fiscal_suggestion + reject_fiscal_suggestion(suggestion_id, actor="operator_ui") + except Exception as exc: + return PlainTextResponse(f"Erro ao rejeitar sugestão fiscal: {exc}", status_code=500) + referer = request.headers.get("referer") or "/opportunities" + return RedirectResponse(referer, status_code=303) + + +@router.post("/opportunities/{opportunity_id}/jasmin/sync-candidates") +async def opportunity_jasmin_sync_candidates_action(opportunity_id: str, request: Request): + try: + from app.external_reconciliation_sync import sync_jasmin_reconciliation_candidates + result = await sync_jasmin_reconciliation_candidates(limit=100, days=30) + except Exception as exc: + if request.headers.get("hx-request"): + return HTMLResponse(jasmin_documents_html(opportunity_id, error_notice=f"Erro ao sincronizar Jasmin: {exc}"), status_code=409) + return PlainTextResponse(f"Erro ao sincronizar Jasmin: {exc}", status_code=500) + seen = result.get("seen", 0) + created = result.get("created_or_updated", 0) + notice = f"Jasmin sincronizado: {seen} documento(s) visto(s), {created} criado(s)/atualizado(s)." + if request.headers.get("hx-request"): + return HTMLResponse(jasmin_documents_html(opportunity_id, notice=notice)) + return RedirectResponse(f"/opportunities/{opportunity_id}?notice=Jasmin%20sincronizado", status_code=303) + + +@router.post("/opportunities/{opportunity_id}/jasmin/reimport-details") +async def opportunity_jasmin_reimport_details_action(opportunity_id: str, request: Request): + try: + from app.jasmin_backfill_service import backfill_jasmin_opportunity_details_async + result = await backfill_jasmin_opportunity_details_async( + opportunity_id=opportunity_id, + fetch_detail=True, + actor="operator_ui_reimport", + dry_run=False, + ) + except Exception as exc: + if is_htmx(request): + return HTMLResponse(jasmin_documents_html(opportunity_id, error_notice=f"Erro ao reimportar detalhes Jasmin: {exc}"), status_code=409) + return PlainTextResponse(f"Erro ao reimportar detalhes Jasmin: {exc}", status_code=500) + if not result.get("ok"): + msg = result.get("error") or "sem itens Jasmin para reimportar" + if is_htmx(request): + return HTMLResponse(jasmin_documents_html(opportunity_id, error_notice=f"Não foi possível reimportar: {msg}"), status_code=409) + return RedirectResponse(f"/opportunities/{opportunity_id}?notice=N%C3%A3o%20foi%20poss%C3%ADvel%20reimportar%20Jasmin", status_code=303) + import_result = result.get("import_result") or {} + docs = int(import_result.get("documents") or 0) + lines = int(import_result.get("lines") or 0) + notice = f"Detalhes Jasmin reimportados: {docs} documento(s), {lines} linha(s). Recarregue a página para atualizar produtos/valor no topo." + if is_htmx(request): + return HTMLResponse(jasmin_documents_html(opportunity_id, notice=notice)) + return RedirectResponse(f"/opportunities/{opportunity_id}?notice={esc(notice)}", status_code=303) + + +@router.post("/opportunities/{opportunity_id}/jasmin/link-candidate/{item_id}") +async def opportunity_jasmin_link_candidate_action(opportunity_id: str, item_id: str, request: Request): + conflict_msg = _jasmin_candidate_tax_conflict_message(opportunity_id, item_id) + if conflict_msg: + if request.headers.get("hx-request"): + return HTMLResponse(jasmin_documents_html(opportunity_id, error_notice=conflict_msg), status_code=409) + return PlainTextResponse(conflict_msg, status_code=409) + try: + from app.jasmin_backfill_service import link_and_import_jasmin_candidate_async + result = await link_and_import_jasmin_candidate_async( + opportunity_id=opportunity_id, + item_id=item_id, + actor="operator_ui_link_existing_jasmin", + ) + except Exception as exc: + if request.headers.get("hx-request"): + return HTMLResponse(jasmin_documents_html(opportunity_id, error_notice=f"Erro ao associar documento Jasmin: {exc}"), status_code=409) + return PlainTextResponse(f"Erro ao associar documento Jasmin: {exc}", status_code=500) + if not result.get("ok"): + msg = result.get("error") or "não foi possível associar documento Jasmin" + if request.headers.get("hx-request"): + return HTMLResponse(jasmin_documents_html(opportunity_id, error_notice=f"Não foi possível associar: {msg}"), status_code=409) + return RedirectResponse(f"/opportunities/{opportunity_id}?notice=N%C3%A3o%20foi%20poss%C3%ADvel%20associar%20Jasmin", status_code=303) + import_result = result.get("import_result") or {} + docs = import_result.get("documents", 0) + lines = import_result.get("lines", 0) + notice = f"Documento Jasmin associado e importado: {docs} documento(s), {lines} linha(s). Recarregue a página para atualizar valor/produtos no topo." + if request.headers.get("hx-request"): + return HTMLResponse(jasmin_documents_html(opportunity_id, notice=notice)) + return RedirectResponse(f"/opportunities/{opportunity_id}?notice=Documento%20Jasmin%20associado", status_code=303) + + +@router.post("/opportunities/{opportunity_id}/jasmin/replace-candidate/{item_id}") +async def opportunity_jasmin_replace_candidate_action(opportunity_id: str, item_id: str, request: Request): + conflict_msg = _jasmin_candidate_tax_conflict_message(opportunity_id, item_id) + if conflict_msg: + if request.headers.get("hx-request"): + return HTMLResponse(jasmin_documents_html(opportunity_id, error_notice=conflict_msg), status_code=409) + return PlainTextResponse(conflict_msg, status_code=409) + try: + from app.jasmin_backfill_service import replace_jasmin_document_for_opportunity_async + result = await replace_jasmin_document_for_opportunity_async( + opportunity_id=opportunity_id, + item_id=item_id, + actor="operator_ui_replace_existing_jasmin", + dry_run=False, + ) + except Exception as exc: + if request.headers.get("hx-request"): + return HTMLResponse(jasmin_documents_html(opportunity_id, error_notice=f"Erro ao substituir documento Jasmin: {exc}"), status_code=409) + return PlainTextResponse(f"Erro ao substituir documento Jasmin: {exc}", status_code=500) + if not result.get("ok"): + msg = result.get("error") or "não foi possível substituir documento Jasmin" + if request.headers.get("hx-request"): + return HTMLResponse(jasmin_documents_html(opportunity_id, error_notice=f"Não foi possível substituir: {msg}"), status_code=409) + return RedirectResponse(f"/opportunities/{opportunity_id}?notice=N%C3%A3o%20foi%20poss%C3%ADvel%20substituir%20Jasmin", status_code=303) + import_result = result.get("import_result") or {} + docs = import_result.get("documents", 0) + lines = import_result.get("lines", 0) + removed_docs = result.get("removed_documents", 0) + notice = f"Documento Jasmin substituído: {removed_docs} anterior(es) removido(s), {docs} documento(s), {lines} linha(s) importada(s). Recarregue a página para atualizar valor/produtos no topo." + if request.headers.get("hx-request"): + return HTMLResponse(jasmin_documents_html(opportunity_id, notice=notice)) + return RedirectResponse(f"/opportunities/{opportunity_id}?notice=Documento%20Jasmin%20substitu%C3%ADdo", status_code=303) + + +@router.post("/opportunities/{opportunity_id}/jasmin/create-quotation") +async def opportunity_jasmin_create_quotation(opportunity_id: str, request: Request): + try: + if settings.jasmin_enabled: + from app.jasmin_service import enqueue_create_quotation + enqueue_create_quotation(opportunity_id, created_by="operator") + else: + return PlainTextResponse("JASMIN_ENABLED=false", status_code=409) + except Exception as exc: + print(f"ClientFlow Jasmin create quotation failed: {exc}", flush=True) + if is_htmx(request): + return HTMLResponse(jasmin_documents_html(opportunity_id, error_notice=str(exc)), status_code=409) + return PlainTextResponse(f"Erro ao criar pedido de orçamento Jasmin: {exc}", status_code=500) + if is_htmx(request): + return HTMLResponse(jasmin_documents_html(opportunity_id, notice="Pedido de orçamento enviado para a outbox Jasmin.")) + return RedirectResponse(f"/opportunities/{opportunity_id}?notice=Pedido%20de%20or%C3%A7amento%20enviado%20para%20a%20outbox%20Jasmin", status_code=303) + + +@router.post("/opportunities/{opportunity_id}/jasmin/convert-invoice") +async def opportunity_jasmin_convert_invoice(opportunity_id: str, request: Request): + try: + if settings.jasmin_enabled: + from app.jasmin_service import enqueue_convert_latest_to_invoice + enqueue_convert_latest_to_invoice(opportunity_id, created_by="operator") + else: + return PlainTextResponse("JASMIN_ENABLED=false", status_code=409) + except Exception as exc: + print(f"ClientFlow Jasmin convert invoice failed: {exc}", flush=True) + if is_htmx(request): + return HTMLResponse(jasmin_documents_html(opportunity_id, error_notice=str(exc)), status_code=409) + return PlainTextResponse(f"Erro ao criar pedido de fatura Jasmin: {exc}", status_code=500) + if is_htmx(request): + return HTMLResponse(jasmin_documents_html(opportunity_id, notice="Pedido de fatura enviado para a outbox Jasmin.")) + return RedirectResponse(f"/opportunities/{opportunity_id}?notice=Pedido%20de%20fatura%20enviado%20para%20a%20outbox%20Jasmin", status_code=303) + + +@router.post("/opportunities/{opportunity_id}/operations/{action_key}") +async def opportunity_operation_action(opportunity_id: str, action_key: str, request: Request): + form = await request.form() + external_id = str(form.get("external_id") or "").strip() + external_name = str(form.get("external_name") or form.get("external_ref") or form.get("title") or "").strip() + external_url = str(form.get("external_url") or "").strip() + note = str(form.get("note") or "").strip() + try: + # Jasmin e Packlink, sem referência manual, criam itens de outbox para a API real. + # Se o operador preencher external_id/external_name, mantém o modo manual/fallback. + if action_key == "jasmin_quotation" and not external_id and not external_name: + if settings.jasmin_enabled: + from app.jasmin_service import enqueue_create_quotation + enqueue_create_quotation(opportunity_id, created_by="operator") + else: + register_operation_action(opportunity_id, action_key, external_id=external_id, external_name=external_name, external_url=external_url, note=note, created_by="operator") + elif action_key == "packlink_shipment" and not external_id and not external_name: + if settings.packlink_enabled: + from app.packlink_service import enqueue_packlink_shipment + enqueue_packlink_shipment(opportunity_id, created_by="operator") + else: + register_operation_action(opportunity_id, action_key, external_id=external_id, external_name=external_name, external_url=external_url, note=note, created_by="operator") + else: + register_operation_action(opportunity_id, action_key, external_id=external_id, external_name=external_name, external_url=external_url, note=note, created_by="operator") + except OperationActionBlocked as exc: + return PlainTextResponse(f"Ação bloqueada: {exc}", status_code=409) + except Exception as exc: + print(f"ClientFlow operation action failed: {exc}", flush=True) + return PlainTextResponse(f"Erro ao registar ação: {exc}", status_code=500) + return RedirectResponse(f"/opportunities/{opportunity_id}", status_code=303) + + +@router.post("/opportunities/{opportunity_id}/odoo/sync-status") +async def opportunity_odoo_sync_status_action(opportunity_id: str, request: Request): + try: + sync_opportunity_odoo_status(opportunity_id) + except Exception: + pass + return RedirectResponse(url=f"/opportunities/{opportunity_id}", status_code=303) + + diff --git a/app/admin_ui/pages/orders.py b/app/admin_ui/pages/orders.py new file mode 100644 index 0000000..f750b2b --- /dev/null +++ b/app/admin_ui/pages/orders.py @@ -0,0 +1,163 @@ +"""Order and fulfilment routes. + +Moved from app.admin_dashboard in v4.7.2. The handlers still reuse +legacy helpers to keep this refactor behavior-preserving. +""" +from fastapi import APIRouter +import app.admin_dashboard as legacy +from app.admin_dashboard import * # noqa: F401,F403 + +router = APIRouter() + + +@router.get("/orders", response_class=HTMLResponse) +@router.get("/encomendas", response_class=HTMLResponse) +async def orders_page(status: Optional[str] = None, q: Optional[str] = None): + opportunities = list_opportunities(status="all", q=q, limit=300) + order_stages = {"PAYMENT_CONFIRMED", "ORDER_PREPARATION", "SHIPPED", "WON"} + if status: + if status == "preparing": + stages = {"PAYMENT_CONFIRMED", "ORDER_PREPARATION"} + elif status == "shipped": + stages = {"SHIPPED", "WON"} + else: + stages = order_stages + else: + stages = order_stages + orders = [opp for opp in opportunities if str(opp.get("stage") or "") in stages] + + rows = "" + for opp in orders: + oid = str(opp.get("id") or "") + stage = str(opp.get("stage") or "") + customer = opp.get("customer_name") or opp.get("customer_email") or "Cliente" + product = opp.get("product_interest") or opp.get("title") or "—" + state_chip = opportunity_stage_badge(stage) + next_step = { + "PAYMENT_CONFIRMED": "Criar/preparar encomenda", + "ORDER_PREPARATION": "Preparar material e envio", + "SHIPPED": "Acompanhar entrega", + "WON": "Concluída", + "NO_INTEREST": "Sem interesse", + }.get(stage, "Acompanhar") + rows += f""" + + #{esc(oid[:8])} + {esc(customer)}
    {esc(opp.get('customer_email') or '')}
    + {esc(product)} + {state_chip} + {esc(next_step)} + {esc(opp.get('updated_at') or '—')} + Abrir + + """ + if not rows: + rows = 'Ainda não existem encomendas em preparação/envio.' + + body = f""" +
    + {kpi_card('A preparar', sum(1 for o in opportunities if str(o.get('stage')) in {'PAYMENT_CONFIRMED','ORDER_PREPARATION'}), '/orders', 'pagas/preparação', 'bi-box-seam')} + {kpi_card('Enviadas', sum(1 for o in opportunities if str(o.get('stage')) == 'SHIPPED'), '/orders?status=shipped', 'em trânsito', 'bi-truck', 'cf-kpi-tone-green')} + {kpi_card('Pipeline', len(opportunities), '/opportunities', 'oportunidades', 'bi-funnel')} + {kpi_card('Tarefas operações', '→', '/tasks?status=pending&route=operacoes', 'abrir fila', 'bi-list-check')} +
    + +
    +
    +
    + + +
    +
    + + +
    +
    Limpar
    +
    +
    + +
    +
    +

    Encomendas e preparação

    Derivado das oportunidades com pagamento confirmado/preparação/envio.
    +
    + + + {rows} +
    EncomendaClienteProdutoEstadoPróximo passoAtualizada
    +
    +
    +
    + """ + return layout("Encomendas", "O que falta preparar, enviar ou acompanhar?", body, "orders") + + +@router.get("/orders/{opportunity_id}", response_class=HTMLResponse) +async def order_detail_page(opportunity_id: str): + opp = get_opportunity(opportunity_id) + if not opp: + return layout("Encomenda não encontrada", "Encomendas", '
    Encomenda não encontrada.
    ', "orders") + + stage = str(opp.get("stage") or "") + tasks = list_opportunity_tasks(opportunity_id, limit=20) + opportunity_items = list_opportunity_items(opportunity_id) + material_rows = "" + for item in opportunity_items: + material_rows += f""" + + {esc(item.get('product_name') or 'Produto')}
    SKU/Odoo {esc(item.get('sku') or '—')}
    Jasmin {esc(item.get('jasmin_sales_item') or '—')}
    + {esc(item.get('quantity') or '1')} + {money_html(item.get('unit_price'))} + {item_status_badge(item.get('status'))} + + """ + if not material_rows: + material_rows = f'Sem produtos definidos. Adicionar na oportunidade.' + + task_rows = "" + for task in tasks: + task_rows += f'
  • {esc(action_label(task.get("action_code")))} · {status_badge(task.get("status"))}
    • ' + if not task_rows: + task_rows = '
  • Sem tarefas associadas.
    • ' + + body = f""" + ← Voltar a encomendas +
    +
    +
    + Encomenda +

    {esc(opp.get('title') or 'Encomenda')}

    +
    {opportunity_stage_badge(stage)} Ver oportunidade
    +
    +
    +

    Material

    Produtos aceites/em preparação derivados da oportunidade.
    +
    + + + {material_rows} +
    ProdutoQtd.PreçoEstado
    +
    +
    +
    +

    Checklist

    +
    + + + + +
    +
    +
    + +
    + """ + return layout(str(opp.get("title") or "Encomenda"), "Preparação e envio", body, "orders") + + diff --git a/app/admin_ui/pages/outbox.py b/app/admin_ui/pages/outbox.py new file mode 100644 index 0000000..a2e1376 --- /dev/null +++ b/app/admin_ui/pages/outbox.py @@ -0,0 +1,242 @@ +"""Integration outbox routes and actions. + +Moved from app.admin_dashboard in v4.7.2. v4.7.4 adds a dedicated +HTMX table partial so filters and operator actions can refresh the outbox +without replacing the full page. +""" +from fastapi import APIRouter, Request +from fastapi.responses import HTMLResponse +import app.admin_dashboard as legacy +from app.admin_dashboard import * # noqa: F401,F403 +from app.admin_ui.guidance import outbox_operator_message + +router = APIRouter() + + +def _outbox_tab_url(status: str = "all", target_system: str = "all") -> str: + parts = [] + if status and status != "all": + parts.append(f"status={esc(status)}") + if target_system and target_system != "all": + parts.append(f"target_system={esc(target_system)}") + return "/outbox" + ("?" + "&".join(parts) if parts else "") + + +def _outbox_partial_url(status: str = "all", target_system: str = "all") -> str: + href = _outbox_tab_url(status=status, target_system=target_system) + return "/outbox/partials/table" + (href[href.find("?"):] if "?" in href else "") + + +def _outbox_items(status: str = "all", target_system: str = "all", limit: int = 100) -> list[dict]: + query_status = None if status == "all" else status + query_target = None if target_system == "all" else target_system + return list_outbox(status=query_status, target_system=query_target, limit=limit) + + +def render_outbox_table_partial(items: list[dict], status: str = "all", target_system: str = "all") -> str: + rows = "" + for item in items: + payload = item.get("payload") or {} + if isinstance(payload, str): + try: + payload = json.loads(payload) + except Exception: + payload = {} + opp_id = payload.get("opportunity_id") or "" + conv = payload.get("conversation_id") or "" + err = str(item.get("last_error") or "") + human_error = outbox_operator_message(item) + short_err = err[:220] + "…" if len(err) > 220 else err + origin = f'Oportunidade' if opp_id else '—' + conv_html = f'Conversa #{esc(conv)}' if conv else '' + item_id = str(item.get("id") or "") + oid = esc(item_id) + rows += f''' + +
    {esc(fmt_dt(item.get('created_at')))}
    {oid} + {operation_status_badge(str(item.get('status') or 'pending'))}
    {esc(fmt_dt(item.get('locked_at')) if item.get('locked_at') else '')}
    {esc(item.get('lock_owner') or '')}
    + {esc(item.get('target_system'))}
    {esc(item.get('action_type'))}
    + {origin}
    {conv_html}
    + +
    + {esc(human_error.get('title'))} +
    Motivo provável: {esc(human_error.get('probable'))}
    + {f'
    Técnico: {esc(short_err)}
    ' if short_err else ''} +
    + + +
    + Ver +
    +
    +
    + + + ''' + if not rows: + rows = 'Sem itens de outbox para os filtros selecionados.' + + return f''' +
    +
    + {len(items)} resultado(s) + A atualizar… +
    +
    + + + {rows} +
    Criado / IDEstadoSistema / AçãoOrigemErroAções
    +
    +
    + ''' + + +@router.get("/outbox/partials/table", response_class=HTMLResponse) +async def outbox_table_partial(status: Optional[str] = None, target_system: Optional[str] = None, limit: int = 100): + status = status or "all" + target_system = target_system or "all" + items = _outbox_items(status=status, target_system=target_system, limit=limit) + return HTMLResponse(render_outbox_table_partial(items, status=status, target_system=target_system)) + + +@router.get("/outbox", response_class=HTMLResponse) +async def outbox_page( + status: Optional[str] = None, + target_system: Optional[str] = None, + limit: int = 100, +): + status = status or "all" + target_system = target_system or "all" + items = _outbox_items(status=status, target_system=target_system, limit=limit) + + status_filters = "".join( + f'{esc(label)}' + for value, label in [("all", "Todas"), ("pending", "Pendentes"), ("processing", "A processar"), ("sent", "Processadas"), ("failed", "Falhadas"), ("blocked", "Bloqueadas"), ("stale", "Stale"), ("dry_run", "Dry-run"), ("ignored", "Ignoradas"), ("cancelled", "Canceladas")] + ) + system_filters = "".join( + f'{esc(label)}' + for value, label in [("all", "Todos"), ("jasmin", "Jasmin"), ("packlink", "Packlink"), ("chatwoot", "Chatwoot"), ("mautic", "Mautic"), ("odoo", "Odoo")] + ) + + body = f''' +
    +
    +
    Estado:{status_filters}
    +
    Sistema:{system_filters}
    +
    +
    + +
    +
    +
    +

    Outbox operacional

    Reprocessa ou ignora ações sem usar SQL no terminal. Filtros e ações atualizam por HTMX.
    + Integrações +
    + {render_outbox_table_partial(items, status=status, target_system=target_system)} +
    +
    + ''' + return layout("Outbox", "Ações pendentes e histórico de integrações externas", body, "outbox") + + +@router.get("/outbox/{outbox_id}", response_class=HTMLResponse) +async def outbox_detail(outbox_id: str): + item = get_outbox_item(outbox_id) + + if not item: + return layout("Outbox item", "Item não encontrado.", "

    Não encontrado.

    ", "outbox") + + payload = item.get("payload") or {} + if isinstance(payload, str): + try: + payload = json.loads(payload) + except Exception: + payload = {} + opp_id = payload.get("opportunity_id") or "" + origin_href = ("/opportunities/" + esc(opp_id)) if opp_id else "/outbox" + origin_label = "Oportunidade" if opp_id else "—" + human_error = outbox_operator_message(item) + fix_href = origin_href if opp_id else "/outbox" + body = f''' +

    ← Voltar à outbox

    + +
    + {kpi_card('Estado', esc(item.get('status')), None, 'atual', 'bi-info-circle')} + {kpi_card('Sistema', esc(item.get('target_system')), None, esc(item.get('action_type')), 'bi-puzzle')} + {kpi_card('Retries', esc(item.get('retry_count')), None, 'tentativas', 'bi-arrow-clockwise')} + {kpi_card('Origem', origin_label, origin_href, esc(opp_id), 'bi-link-45deg')} +
    + +
    +

    Leitura operacional

    +
    + {esc(human_error.get('title'))} +
    Motivo provável: {esc(human_error.get('probable'))}
    + {esc(human_error.get('fix_label'))} +
    +
    + +
    +

    Payload

    + 
    {esc(pretty_json(payload))}
    +

    Erro técnico

    + 
    {esc(item.get('last_error') or '')}
    +
    +
    +
    +
    +
    +
    + ''' + return layout("Outbox item", f"Detalhe {outbox_id}", body, "outbox") + + +@router.post("/outbox/{outbox_id}/retry") +async def outbox_retry(outbox_id: str, request: Request): + form = await request.form() + opportunity_id = str(form.get("opportunity_id") or "").strip() + set_outbox_status(outbox_id=outbox_id, status="pending") + if opportunity_id and is_htmx(request): + return HTMLResponse(jasmin_documents_html(opportunity_id, notice="Item de outbox reposto para pending.")) + if opportunity_id: + return RedirectResponse(f"/opportunities/{opportunity_id}?notice=Item%20de%20outbox%20reposto%20para%20pending", status_code=303) + return RedirectResponse("/outbox", status_code=303) + + +def _outbox_htmx_or_redirect(request: Request, status: str = "all", target_system: str = "all"): + if is_htmx(request): + return HTMLResponse(render_outbox_table_partial(_outbox_items(status=status, target_system=target_system))) + return RedirectResponse("/outbox", status_code=303) + + +@router.post("/outbox/{outbox_id}/pending") +async def outbox_pending(outbox_id: str, request: Request): + set_outbox_status(outbox_id=outbox_id, status="pending") + from app.operator_audit_service import record_operator_action_best_effort + record_operator_action_best_effort(action="outbox_reprocess_requested", entity_type="outbox", entity_id=outbox_id, actor="operator", after={"status": "pending"}) + return _outbox_htmx_or_redirect(request) + + +@router.post("/outbox/{outbox_id}/sent") +async def outbox_sent(outbox_id: str, request: Request): + set_outbox_status(outbox_id=outbox_id, status="sent") + from app.operator_audit_service import record_operator_action_best_effort + record_operator_action_best_effort(action="outbox_marked_sent", entity_type="outbox", entity_id=outbox_id, actor="operator", after={"status": "sent"}) + return _outbox_htmx_or_redirect(request) + + +@router.post("/outbox/{outbox_id}/failed") +async def outbox_failed(outbox_id: str, request: Request): + set_outbox_status(outbox_id=outbox_id, status="failed", error="Marcado manualmente como failed.") + from app.operator_audit_service import record_operator_action_best_effort + record_operator_action_best_effort(action="outbox_marked_failed", entity_type="outbox", entity_id=outbox_id, actor="operator", after={"status": "failed"}) + return _outbox_htmx_or_redirect(request) + + +@router.post("/outbox/{outbox_id}/ignored") +async def outbox_ignored(outbox_id: str, request: Request): + set_outbox_status(outbox_id=outbox_id, status="ignored", error="Ignorado manualmente pelo operador.") + from app.operator_audit_service import record_operator_action_best_effort + record_operator_action_best_effort(action="outbox_ignored", entity_type="outbox", entity_id=outbox_id, actor="operator", after={"status": "ignored"}) + return _outbox_htmx_or_redirect(request) diff --git a/app/admin_ui/pages/products.py b/app/admin_ui/pages/products.py new file mode 100644 index 0000000..c576dd5 --- /dev/null +++ b/app/admin_ui/pages/products.py @@ -0,0 +1,301 @@ +"""Product catalog and opportunity item routes. + +Moved from app.admin_dashboard in v4.7.2. The handlers still reuse +legacy helpers to keep this refactor behavior-preserving. +""" +from fastapi import APIRouter +import app.admin_dashboard as legacy +from app.admin_dashboard import * # noqa: F401,F403 + +router = APIRouter() + + +@router.get("/products", response_class=HTMLResponse) +@router.get("/produtos", response_class=HTMLResponse) +async def products_page( + q: Optional[str] = None, + category: Optional[str] = "all", + active: Optional[str] = "true", +): + products = list_products(q=q, category=category, active=("true" if active == "missing_jasmin" else active), limit=500) + if active == "missing_jasmin": + products = [p for p in products if not p.get("jasmin_sales_item")] + categories = list_product_categories() + + category_options = '' + for cat in categories: + selected = "selected" if category == cat else "" + category_options += f'' + + active_options = "" + for value, label in [("true", "Ativos"), ("missing_jasmin", "Ativos sem Artigo Jasmin"), ("false", "Inativos"), ("all", "Todos")]: + selected = "selected" if (active or "true") == value else "" + active_options += f'' + + rows = "" + for product in products: + rows += f''' + + {esc(product.get('sku') or '—')} + {esc(product.get('jasmin_sales_item') or '—')} + {esc(product.get('name') or 'Produto')}
    {esc(product.get('description') or '')}
    + {esc(product.get('category') or '—')} + {money_html(product.get('default_unit_price'))} + {esc(product.get('vat_rate') or '23')}% + {product_status_badge(product.get('active'))} + Abrir + + ''' + if not rows: + rows = 'Sem produtos encontrados.' + + body = f''' +
    + {kpi_card('Produtos listados', len(products), '/products', 'catálogo', 'bi-box-seam')} + {kpi_card('Ativos', sum(1 for p in products if p.get('active')), '/products?active=true', 'para orçamento', 'bi-check2-circle', 'cf-kpi-tone-green')} + {kpi_card('Categorias', len(categories), '/products?category=Carregadores', 'organização', 'bi-tags')} + {kpi_card('Novo produto', '+', '/products/new', 'adicionar', 'bi-plus-circle')} +
    + +
    +
    +
    + + +
    +
    + + +
    +
    + + +
    +
    + + Limpar +
    +
    +
    + +
    +
    +
    +
    +

    Catálogo de produtos

    +
    Produtos e serviços usados nas oportunidades, orçamentos e encomendas.
    +
    +
    Validar artigos Jasmin+ Novo produto
    +
    +
    + + + {rows} +
    SKU/OdooJasminProdutoCategoriaPreçoIVAEstado
    +
    +
    +
    + ''' + return layout("Produtos", "Catálogo simples sem variantes, usado para propostas e encomendas", body, "products") + + +@router.get("/products/validate-jasmin", response_class=HTMLResponse) +async def products_validate_jasmin_page(): + products = list_products(active="true", limit=500) + rows = "" + checked = 0 + ok_count = 0 + missing_count = 0 + invalid_count = 0 + client = None + if settings.jasmin_enabled: + try: + from app.jasmin_client import JasminClient + client = JasminClient() + except Exception as exc: + rows += f'Erro de configuração Jasmin: {esc(exc)}' + for product in products: + jasmin_item = str(product.get("jasmin_sales_item") or "").strip() + status = "Sem Artigo Jasmin" + detail = "" + chip = "cf-chip-orange" + if not jasmin_item: + missing_count += 1 + elif not client: + status = "Não testado" + detail = "Jasmin desativado ou configuração inválida" + chip = "cf-chip-gray" + else: + checked += 1 + try: + info = await client.get_sales_item(jasmin_item) + if isinstance(info, dict) and info.get("itemKey"): + status = "OK" + detail = info.get("description") or info.get("itemKey") or "" + chip = "cf-chip-green" + ok_count += 1 + else: + status = "Resposta inesperada" + detail = str(info)[:180] + chip = "cf-chip-red" + invalid_count += 1 + except Exception as exc: + status = "Não existe / erro" + detail = str(exc)[:220] + chip = "cf-chip-red" + invalid_count += 1 + rows += f''' + + {esc(product.get('sku') or '—')} + {esc(jasmin_item or '—')} + {esc(product.get('name') or 'Produto')} + {esc(status)} + {esc(detail)} + + ''' + if not rows: + rows = 'Sem produtos ativos.' + body = f''' + ← Voltar a produtos +
    + {kpi_card('OK', ok_count, None, 'artigos válidos', 'bi-check2-circle', 'cf-kpi-tone-green')} + {kpi_card('Sem artigo', missing_count, None, 'precisam mapeamento', 'bi-box-seam', 'cf-kpi-tone-orange')} + {kpi_card('Inválidos/erro', invalid_count, None, 'corrigir antes de orçamentar', 'bi-exclamation-triangle', 'cf-kpi-tone-red')} + {kpi_card('Testados', checked, None, 'consultas Jasmin', 'bi-patch-check')} +
    +
    +

    Validação de artigos Jasmin

    Confirma se products.jasmin_sales_item existe no Jasmin.
    +
    {rows}
    SKU/OdooArtigo JasminProdutoEstadoDetalhe
    +
    + ''' + return layout("Validar artigos Jasmin", "Verificação de mapeamentos de produtos", body, "products") + + +@router.get("/products/new", response_class=HTMLResponse) +async def product_new_page(): + body = f''' + ← Voltar a produtos +
    +

    Novo produto

    + {product_form_html({}, action="/products", submit_label="Criar produto")} +
    + ''' + return layout("Novo produto", "Adicionar produto ou serviço ao catálogo", body, "products") + + +@router.post("/products") +async def product_create(request: Request): + data = dict(await request.form()) + try: + product_id = create_product(data) + return RedirectResponse(f"/products/{product_id}", status_code=303) + except Exception as exc: + body = f''' + ← Voltar a produtos +
    Erro ao criar produto: {esc(exc)}
    +
    {product_form_html(data, action="/products", submit_label="Criar produto")}
    + ''' + return layout("Novo produto", "Corrige os campos e tenta novamente", body, "products") + + +@router.get("/products/{product_id}", response_class=HTMLResponse) +async def product_detail_page(product_id: str): + product = get_product(product_id) + if not product: + return layout("Produto não encontrado", "Catálogo", '
    Produto não encontrado.
    ', "products") + + toggle_label = "Desativar" if product.get("active") else "Ativar" + toggle_active = "false" if product.get("active") else "true" + body = f''' + ← Voltar a produtos +
    +
    +
    +
    + Produto +

    {esc(product.get('name') or 'Produto')}

    +
    SKU/Odoo {esc(product.get('sku') or '—')} · Jasmin {esc(product.get('jasmin_sales_item') or '—')} · {esc(product.get('category') or 'Geral')}
    +
    + {product_status_badge(product.get('active'))} +
    + {product_form_html(product, action=f"/products/{product_id}/update", submit_label="Guardar alterações")} +
    + +
    + ''' + return layout(str(product.get("name") or "Produto"), "Editar produto do catálogo", body, "products") + + +@router.post("/products/{product_id}/update") +async def product_update(product_id: str, request: Request): + data = dict(await request.form()) + try: + update_product(product_id, data) + return RedirectResponse(f"/products/{product_id}", status_code=303) + except Exception as exc: + product = get_product(product_id) or data + body = f''' + ← Voltar ao produto +
    Erro ao guardar produto: {esc(exc)}
    +
    {product_form_html(product, action=f"/products/{product_id}/update", submit_label="Guardar alterações")}
    + ''' + return layout("Editar produto", "Corrige os campos e tenta novamente", body, "products") + + +@router.post("/products/{product_id}/toggle") +async def product_toggle(product_id: str, request: Request): + data = dict(await request.form()) + set_product_active(product_id, str(data.get("active") or "false").lower() == "true") + return RedirectResponse(f"/products/{product_id}", status_code=303) + + +@router.post("/opportunities/{opportunity_id}/items/add") +async def opportunity_item_add(opportunity_id: str, request: Request): + data = dict(await request.form()) + try: + add_opportunity_item( + opportunity_id, + product_id=data.get("product_id") or None, + product_name=data.get("product_name") or None, + jasmin_sales_item=data.get("jasmin_sales_item") or None, + quantity=data.get("quantity") or "1", + unit_price=data.get("unit_price") or None, + discount_amount=data.get("discount_amount") or "0", + status=data.get("status") or "INTERESTED", + ) + except Exception as exc: + print(f"ClientFlow add opportunity item failed: {exc}", flush=True) + if is_htmx(request): + return HTMLResponse(opportunity_products_panel_html(opportunity_id, error_notice=str(exc)), status_code=409) + if is_htmx(request): + return HTMLResponse(opportunity_products_panel_html(opportunity_id, notice="Produto adicionado à oportunidade.")) + return RedirectResponse(f"/opportunities/{opportunity_id}?notice=Produto%20adicionado%20%C3%A0%20oportunidade", status_code=303) + + +@router.post("/opportunities/{opportunity_id}/items/{item_id}/delete") +async def opportunity_item_delete(opportunity_id: str, item_id: str, request: Request): + try: + delete_opportunity_item(item_id) + except Exception as exc: + print(f"ClientFlow delete opportunity item failed: {exc}", flush=True) + if is_htmx(request): + return HTMLResponse(opportunity_products_panel_html(opportunity_id, error_notice=str(exc)), status_code=409) + if is_htmx(request): + return HTMLResponse(opportunity_products_panel_html(opportunity_id, notice="Produto removido.")) + return RedirectResponse(f"/opportunities/{opportunity_id}", status_code=303) + + diff --git a/app/admin_ui/pages/queues.py b/app/admin_ui/pages/queues.py new file mode 100644 index 0000000..4f8ab6a --- /dev/null +++ b/app/admin_ui/pages/queues.py @@ -0,0 +1,83 @@ +"""Operational queue diagnostic routes. + +Moved from app.admin_dashboard in v4.7.2. The handlers still reuse +legacy helpers to keep this refactor behavior-preserving. +""" +from fastapi import APIRouter +import app.admin_dashboard as legacy +from app.admin_dashboard import * # noqa: F401,F403 + +router = APIRouter() + + +@router.get("/queues", response_class=HTMLResponse) +@router.get("/filas", response_class=HTMLResponse) +async def queues_page(): + metrics = get_admin_dashboard_metrics() + queue_defs = [ + ("vendas", "Comercial", "Orçamentos, propostas e oportunidades", "pending_vendas", "cf-chip-blue"), + ("financeiro", "Financeiro", "Pagamentos, faturas e comprovativos", "pending_financeiro", "cf-chip-green"), + ("operacoes", "Logística", "Envios, recolhas e entregas", "pending_operacoes", "cf-chip-orange"), + ("suporte", "Suporte", "Técnico, garantia e pós-venda", "pending_suporte", "cf-chip-purple"), + ("rever", "Rever", "Casos ambíguos ou baixa confiança", "pending_rever", "cf-chip-gray"), + ] + + rows = "" + chart_items = "" + for route_key, label, description, metric_key, chip_class in queue_defs: + count = int(metrics.get(metric_key) or 0) + active_tasks = [] + try: + active_tasks = list_tasks(status="pending", route=route_key, limit=1) + except Exception: + active_tasks = [] + priority = "Alta" if route_key in {"financeiro", "operacoes"} and count else ("Média" if count else "Baixa") + rows += f''' + + {esc(label)} + {esc(description)} + {count} + {esc(priority)} + + Abrir + + + ''' + chart_items += f''' +
    +
    {esc(label)}{count}
    +
    +
    + ''' + + body = f''' +
    +
    +
    +
    +
    +

    Filas

    +
    Gestão de distribuição de tarefas por equipa.
    +
    +
    + + + {rows} +
    FilaDescriçãoTarefasPrioridadeAções
    +
    +
    +
    +
    +
    +
    +
    +

    Distribuição de tarefas

    + {chart_items} +
    +
    +
    +
    + ''' + return layout("Filas", "Gestão de filas e distribuição de tarefas", body, "queues") + + diff --git a/app/admin_ui/pages/reconciliation.py b/app/admin_ui/pages/reconciliation.py new file mode 100644 index 0000000..c760e47 --- /dev/null +++ b/app/admin_ui/pages/reconciliation.py @@ -0,0 +1,746 @@ +"""Operational reconciliation and external intake UI. + +This page is a staging area for information found outside ClientFlow. It does +not replace Operations; it prepares loose Jasmin/Odoo/payment/manual evidence so +an operator can link, create or ignore it deliberately. +""" +from __future__ import annotations + +import logging +from typing import Optional + +from fastapi import APIRouter, Request +from fastapi.responses import HTMLResponse, RedirectResponse + +from app.admin_ui.components import esc, fmt_dt, kpi_card, money, status_chip +from app.admin_ui.layout import layout +from app.fiscal_enrichment_service import enrich_open_opportunities, fiscal_enrichment_summary +from app.external_reconciliation_sync import ( + sync_all_external_reconciliation_candidates, + sync_jasmin_reconciliation_candidates, + sync_odoo_reconciliation_candidates, + sync_packlink_reconciliation_candidates, +) +from app.reconciliation_decision_service import ( + classify_reconciliation_item, + classify_reconciliation_process, + reconciliation_decision_summary, + sort_items_for_operator, +) +from app.reconciliation_service import ( + cleanup_reconciliation_outside_window, + create_external_request, + create_opportunity_from_reconciliation, + create_payment_proof, + link_reconciliation_to_opportunity, + list_reconciliation_items, + list_reconciliation_process_candidates, + create_opportunity_from_reconciliation_process, + link_reconciliation_process_to_opportunity, + reconciliation_summary, + recent_window_start, + reset_generated_reconciliation_items, + set_reconciliation_status, + sync_local_documents_without_opportunity, + upsert_reconciliation_item, +) + +logger = logging.getLogger(__name__) + +router = APIRouter() + + +TYPE_LABELS = { + "jasmin_quotation": "Orçamento Jasmin", + "jasmin_proforma": "Pró-forma Jasmin", + "jasmin_invoice": "Fatura Jasmin", + "odoo_sale_order": "Venda Odoo", + "payment_proof": "Comprovativo", + "packlink_shipment": "Envio Packlink", + "manual_request": "Pedido externo", + "external_record": "Registo externo", +} + + +def _type_label(value: str | None) -> str: + return TYPE_LABELS.get(str(value or ""), str(value or "—")) + + +def _safe_days(value: object, default: int = 3) -> int: + try: + return min(max(int(value or default), 1), 90) + except Exception: + return default + + +def _filter_url(status: str = "open", external_type: str = "all", days: int = 3) -> str: + return f"/reconciliation?status={esc(status)}&external_type={esc(external_type)}&days={int(days or 3)}" + + + +def _operation_label(value: str | None) -> str: + raw = str(value or "") + if not raw: + return "" + labels = { + "odoo_sale_order": "Venda Odoo", + "jasmin_quotation": "Orçamento Jasmin", + "jasmin_proforma": "Pró-forma Jasmin", + "jasmin_invoice": "Fatura Jasmin", + "document": "Documento", + } + if ":" in raw: + key, ref = raw.split(":", 1) + return f"{labels.get(key, key)} {ref}" + return raw + +def _render_item_rows(items: list[dict]) -> str: + rows = "" + for item in items: + item_id = str(item.get("id") or "") + opportunity_link = "—" + if item.get("opportunity_id"): + opportunity_label = item.get("opportunity_title") or "Ver oportunidade" + opportunity_link = f'{esc(opportunity_label)}' + + amount = "—" + if item.get("amount") is not None: + amount = money(item.get("amount"), item.get("currency") or "EUR") + + actions = "" + if str(item.get("status") or "") in {"open", "needs_review"}: + suggestions = item.get("operation_suggestions") or [] + suggestion_html = "" + if suggestions: + suggestion_cards = "" + for suggestion in suggestions[:2]: + suggested_opp_id = str(suggestion.get("opportunity_id") or "") + suggested_title = suggestion.get("opportunity_title") or "Oportunidade aberta" + suggested_customer = suggestion.get("customer_name") or "" + suggested_action = suggestion.get("action") or suggestion.get("action_code") or "operação aberta" + suggested_reason = suggestion.get("reason") or "possível correspondência" + suggestion_cards += f""" +
    + +
    Sugestão encontrada em Operations
    +
    {esc(suggested_title)}
    +
    {esc(suggested_customer)}
    +
    {esc(suggested_action)} · {esc(suggested_reason)}
    + +
    + """ + suggestion_html = f'
    {suggestion_cards}
    ' + + create_button_class = "btn-outline-primary" if suggestions else "btn-primary" + search_query = item.get("customer_tax_id") or item.get("customer_email") or item.get("customer_name") or item.get("document_number") or "" + search_href = f"/opportunities?q={esc(str(search_query))}" if search_query else "/opportunities" + manual_link = f""" +
    + Escolher outra oportunidade +
    + + +
    +
    + """ + actions = f""" +
    + {suggestion_html} +
    + Pesquisar oportunidade +
    + +
    +
    + +
    +
    + +
    +
    + +
    +
    + {manual_link} +
    + """ + else: + actions = 'Sem ações pendentes' + + decision = classify_reconciliation_item(item) + decision_chip = f'{esc(decision.get("label") or "Decisão")}' + rows += f""" + + +
    {esc(item.get('title'))}
    +
    {esc(item.get('description') or '')}
    +
    {esc(item.get('source_system'))} · {esc(item.get('external_id') or 'sem external_id')}
    + + {esc(_type_label(item.get('external_type')))}
    {esc(item.get('document_number') or '')}
    + {status_chip(item.get('status'))}
    {esc(item.get('priority') or 'normal')}
    {decision_chip}
    {esc(decision.get('primary_decision') or '')}
    +
    {esc(item.get('customer_name') or item.get('linked_customer_name') or '—')}
    {esc(item.get('customer_email') or '')}
    {f'
    NIF {esc(item.get("customer_tax_id"))}
    ' if item.get('customer_tax_id') else ''} + {amount}
    {esc(fmt_dt(item.get('document_date')))}
    + {opportunity_link} + {actions} + + """ + if not rows: + rows = 'Sem itens de reconciliação para este filtro.' + return rows + + + +def _render_process_candidates(candidates: list[dict], *, days: int = 3) -> str: + """Render grouped timeline reconstruction proposals. + + These cards are intentionally more useful than isolated rows: they show the + probable sequence found across Jasmin/Odoo/Packlink/payment evidence and let + the operator create or link one process deliberately. + """ + if not candidates: + return """ +
    +
    +

    Processos candidatos

    +
    Sem processos reconstruíveis na janela ativa. Itens isolados continuam disponíveis abaixo.
    +
    +
    + """ + + cards = "" + for candidate in candidates[:8]: + item_ids = ",".join(str(x) for x in candidate.get("item_ids") or []) + tax_line = f'
    NIF {esc(candidate.get("customer_tax_id"))}
    ' if candidate.get("customer_tax_id") else "" + email_line = f'
    {esc(candidate.get("customer_email"))}
    ' if candidate.get("customer_email") else "" + amount = "" + if candidate.get("amount") is not None: + amount = f'{money(candidate.get("amount"), candidate.get("currency") or "EUR")}' + identity_line = f'
    Cliente fiscal: {esc(candidate.get("identity_reason") or candidate.get("match_key") or "—")}
    ' + operation_label = _operation_label(candidate.get("operation_key")) + operation_line = f'
    Compra/processo: {esc(operation_label)}
    ' if operation_label else "" + review_status = str(candidate.get("review_status") or "needs_review") + review_label = {"ready": "pronto", "needs_review": "rever", "conflict": "conflito"}.get(review_status, review_status) + review_class = "green" if review_status == "ready" else "orange" if review_status == "needs_review" else "red" + review_chip = f'{esc(review_label)}' + decision = classify_reconciliation_process(candidate) + decision_chip = f'{esc(decision.get("label") or "Decisão")}' + reasons = "".join(f'
  • {esc(reason)}
    • ' for reason in (candidate.get("reasons") or [])[:5]) + risks = "".join(f'
  • {esc(risk)}
    • ' for risk in (candidate.get("risks") or [])[:5]) + explanation_html = ( + '
    ' + f'
    Motivos
      {reasons or "
    • candidato reconstruído por identidade fiscal/processo
      • "}
    ' + f'
    Riscos
      {risks or "
    • sem riscos relevantes detetados
      • "}
    ' + '
    ' + ) + timeline = "" + for step in candidate.get("timeline") or []: + timeline += f""" +
    +
    {esc(fmt_dt(step.get('document_date')))}
    +
    {esc(step.get('label'))}
    +
    {esc(step.get('document_number') or step.get('title') or '')}
    +
    + """ + suggestion_html = "" + suggestions = candidate.get("suggestions") or [] + if suggestions: + suggestion = suggestions[0] + suggestion_html = f""" +
    + + +
    Sugestão de oportunidade existente
    +
    {esc(suggestion.get('opportunity_title') or 'Oportunidade aberta')}
    +
    {esc(suggestion.get('customer_name') or '')} · {esc(suggestion.get('reason') or 'possível correspondência')}
    + +
    + """ + cards += f""" +
    +
    +
    +

    {esc(candidate.get('customer_name') or 'Processo externo')}

    + {email_line}{tax_line} +
    +
    + {len(candidate.get('items') or [])} evidência(s) + confiança {esc(candidate.get('confidence') or 'média')} + {review_chip} + {decision_chip} + {amount} +
    +
    + {identity_line}{operation_line} +
    {esc(decision.get('primary_decision') or 'Decisão pendente')}{esc(decision.get('description') or '')}
    + {explanation_html} +
    +
    Estado sugerido{esc(candidate.get('suggested_stage') or 'REVIEW')}
    +
    Próxima ação{esc(candidate.get('suggested_action') or 'REVIEW_MANUALLY')}
    +
    +
    {timeline}
    +
    + {suggestion_html} +
    + + +
    +
    + + +
    +
    + + +
    +
    + + +
    +
    + Ligar a outra oportunidade +
    + + + +
    +
    +
    +
    + """ + return f""" +
    +
    +

    Processos candidatos

    +
    O sistema agrupa primeiro por cliente fiscal e depois separa por compra/processo antes de criar ou ligar a oportunidade.
    +
    +
    {cards}
    +
    + """ + +def render_reconciliation_decision_board(items: list[dict], candidates: list[dict], *, status: str, external_type: str, days: int) -> str: + summary = reconciliation_decision_summary(items, candidates) + cards = [ + ("actionable", "Ação recomendada", "Ligar/criar com evidência suficiente", "open"), + ("review", "Requer revisão", "Conflitos ou baixa confiança", "needs_review"), + ("historical", "Histórico", "Sem ação operacional", "historical"), + ("ignored", "Ignorados", "Fora da fila", "ignored"), + ("resolved", "Resolvidos", "Ligados ou fechados", "linked"), + ] + html = "" + for key, label, description, target_status in cards: + active = " active" if ((key == "actionable" and status == "open") or status == target_status) else "" + href = _filter_url(target_status, external_type, days) + html += f''' + + {esc(label)} + {int(summary.get(key, 0))} + {esc(description)} + + ''' + return f''' +
    + {html} +
    + ''' + + +def render_reconciliation_table(items: list[dict]) -> str: + return f""" +
    + + + {_render_item_rows(items)} +
    ItemTipoEstadoClienteValor/dataOportunidadeAções
    +
    + """ + + +@router.get("/reconciliation", response_class=HTMLResponse) +@router.get("/reconciliacao", response_class=HTMLResponse) +async def reconciliation_page(status: Optional[str] = "open", external_type: Optional[str] = "all", notice: Optional[str] = None, days: Optional[int] = 3): + status = status or "open" + external_type = external_type or "all" + item_type = None if external_type == "all" else external_type + recent_days = min(max(int(days or 3), 1), 90) + items = sort_items_for_operator(list_reconciliation_items(status=status, external_type=item_type, limit=100, days=recent_days)) + process_candidates = list_reconciliation_process_candidates(status="open", days=recent_days, limit=8) + summary = reconciliation_summary(days=recent_days) + enrichment_summary = fiscal_enrichment_summary() + + notice_html = f'
    {esc(notice)}
    ' if notice else "" + status_filters = "".join( + f'{esc(label)}' + for value, label in [("open", "Abertos"), ("needs_review", "Revisão"), ("conflict", "Conflitos"), ("historical", "Histórico"), ("linked", "Ligados"), ("resolved", "Resolvidos"), ("ignored", "Ignorados"), ("all", "Todos")] + ) + type_filters = "".join( + f'{esc(label)}' + for value, label in [("all", "Todos"), ("jasmin_quotation", "Orçamentos"), ("jasmin_invoice", "Faturas"), ("payment_proof", "Comprovativos"), ("odoo_sale_order", "Odoo"), ("packlink_shipment", "Packlink"), ("manual_request", "Externos")] + ) + + body = f""" + {notice_html} +
    + Reconciliação = informação solta para organizar. + O sistema cria candidatos por cliente fiscal e por compra/processo; o operador decide ligar, criar oportunidade ou ignorar. Janela ativa: últimos {recent_days} dias operacionais, desde {esc(recent_window_start(recent_days))}. Comprovativo recebido não confirma pagamento. +
    + +
    + {kpi_card('Itens abertos', summary.get('open', 0), '/reconciliation', 'pendências para ligar/criar/ignorar', 'bi-diagram-3')} + {kpi_card('Conflitos', summary.get('conflict', 0), '/reconciliation?status=conflict', 'cliente/processo com risco', 'bi-exclamation-triangle')} + {kpi_card('Histórico', summary.get('historical', 0), '/reconciliation?status=historical', 'fora da operação diária', 'bi-archive')} + {kpi_card('Resolvidos', summary.get('resolved', 0), '/reconciliation?status=linked', 'ligados ou fechados', 'bi-check2-circle')} + {kpi_card('Sugestões fiscais', enrichment_summary.get('pending_suggestions', 0), '/opportunities?scope=blocked', 'pendentes antes da reconciliação', 'bi-person-vcard')} +
    + + {render_reconciliation_decision_board(items, process_candidates, status=status, external_type=external_type, days=recent_days)} + +
    +
    Estado:{status_filters}
    +
    Tipo:{type_filters}
    +
    Janela: + {''.join(f'{label}' for value, label in [(1, 'Hoje'), (3, '3 dias'), (7, '7 dias'), (30, '30 dias')])} +
    +
    +
    + +
    +
    + +
    +
    + +
    +
    + +
    +
    + +
    +
    + +
    +
    + +
    +
    + +
    +
    +
    + + {_render_process_candidates(process_candidates, days=recent_days)} + +
    +
    +

    Itens soltos

    Itens individuais que ainda precisam de decisão ou que não formaram um processo candidato.
    + {render_reconciliation_table(items)} +
    +
    + +
    +
    +
    +

    Registar pedido externo

    +
    +
    +
    +
    +
    +
    +
    +
    +
    + +
    +
    +

    Adicionar comprovativo

    +
    +
    Regra: comprovativo recebido abre validação financeira; não confirma pagamento.
    +
    +
    +
    +
    +
    +
    +
    +
    + +
    +

    Criar candidato manual de reconciliação

    +
    +
    +
    +
    +
    +
    +
    +
    +
    + """ + return layout("Reconciliação", "Organizar documentos, vendas e comprovativos fora do ClientFlow", body, "reconciliation") + + +@router.post("/reconciliation/sync-local-documents") +async def reconciliation_sync_local_documents(): + result = sync_local_documents_without_opportunity(limit=200) + notice = f"Documentos analisados: {result.get('seen', 0)} · candidatos criados/atualizados: {result.get('created_or_updated', 0)}" + return RedirectResponse(f"/reconciliation?notice={esc(notice)}", status_code=303) + + +@router.post("/reconciliation/sync-jasmin") +async def reconciliation_sync_jasmin(request: Request): + form = await request.form() + days = _safe_days(form.get("days"), 3) + result = await sync_jasmin_reconciliation_candidates(limit=50, days=days) + notice = f"Jasmin últimos {result.get('days', days)} dias: analisados {result.get('seen', 0)} · candidatos criados/atualizados {result.get('created_or_updated', 0)}" + if result.get("errors"): + notice += " · erros: " + "; ".join(str(x) for x in result.get("errors", [])[:2]) + if result.get("skipped"): + notice += " · " + str(result.get("skipped")) + return RedirectResponse(f"/reconciliation?days={days}¬ice={esc(notice)}", status_code=303) + + +@router.post("/reconciliation/sync-odoo") +async def reconciliation_sync_odoo(request: Request): + form = await request.form() + days = _safe_days(form.get("days"), 3) + result = sync_odoo_reconciliation_candidates(limit=50, days=days) + notice = f"Odoo últimos {result.get('days', days)} dias: analisadas {result.get('seen', 0)} vendas · candidatos criados/atualizados {result.get('created_or_updated', 0)}" + if result.get("skipped"): + notice += " · " + str(result.get("skipped")) + return RedirectResponse(f"/reconciliation?days={days}¬ice={esc(notice)}", status_code=303) + + +@router.post("/reconciliation/sync-packlink") +async def reconciliation_sync_packlink(request: Request): + form = await request.form() + days = _safe_days(form.get("days"), 3) + result = await sync_packlink_reconciliation_candidates(limit=50, days=days) + notice = f"Packlink últimos {result.get('days', days)} dias: analisados {result.get('seen', 0)} envios · candidatos criados/atualizados {result.get('created_or_updated', 0)}" + if result.get("errors"): + notice += " · erros: " + "; ".join(str(x) for x in result.get("errors", [])[:2]) + if result.get("skipped"): + notice += " · " + str(result.get("skipped")) + return RedirectResponse(f"/reconciliation?days={days}¬ice={esc(notice)}", status_code=303) + + +@router.post("/reconciliation/enrich-fiscal") +async def reconciliation_enrich_fiscal(request: Request): + result = enrich_open_opportunities(limit=200, apply_safe=True, mode="operator_ui") + notice = ( + f"Enriquecimento fiscal: analisadas {result.get('seen', 0)} oportunidades" + f" · sugestões {result.get('suggested', 0)}" + f" · auto-associadas {result.get('auto_applied', 0)}" + f" · ignoradas {result.get('skipped', 0)}" + ) + if result.get("errors"): + notice += " · erros: " + "; ".join(str(x) for x in result.get("errors", [])[:2]) + return RedirectResponse(f"/reconciliation?notice={esc(notice)}", status_code=303) + + +@router.post("/reconciliation/sync-external") +async def reconciliation_sync_external(request: Request): + form = await request.form() + days = _safe_days(form.get("days"), 3) + result = await sync_all_external_reconciliation_candidates(limit=50, days=days) + notice = ( + f"APIs externas últimos {days} dias: enriquecidas {result.get('enrichment_auto_applied', 0)} oportunidades" + f" · sugestões fiscais {result.get('enrichment_suggested', 0)}" + f" · clientes fiscais analisados {result.get('customer_seen', 0)}" + f" · clientes criados/atualizados {result.get('customers_created_or_updated', 0)}" + f" · documentos/vendas analisados {result.get('seen', 0)}" + f" · candidatos criados/atualizados {result.get('created_or_updated', 0)}" + ) + return RedirectResponse(f"/reconciliation?days={days}¬ice={esc(notice)}", status_code=303) + + +@router.post("/reconciliation/rebuild") +async def reconciliation_rebuild(request: Request): + form = await request.form() + days = _safe_days(form.get("days"), 3) + reset = reset_generated_reconciliation_items(days=days, limit=2000, actor="operator_ui", apply=True) + result = await sync_all_external_reconciliation_candidates(limit=200, days=days) + notice = ( + f"Reconciliação reconstruída para {days} dias: apagados {reset.get('deleted', 0)} candidatos gerados" + f" · backup {reset.get('backup_table') or 'sem alterações'}" + f" · enriquecidas {result.get('enrichment_auto_applied', 0)} oportunidades" + f" · sugestões fiscais {result.get('enrichment_suggested', 0)}" + f" · clientes fiscais analisados {result.get('customer_seen', 0)}" + f" · clientes criados/atualizados {result.get('customers_created_or_updated', 0)}" + f" · documentos/vendas analisados {result.get('seen', 0)}" + f" · candidatos criados/atualizados {result.get('created_or_updated', 0)}" + ) + errors = [] + for source_result in result.get("results", []) or []: + errors.extend(str(err) for err in source_result.get("errors", []) or []) + if errors: + notice += " · erros: " + "; ".join(errors[:2]) + return RedirectResponse(f"/reconciliation?days={days}¬ice={esc(notice)}", status_code=303) + + +@router.post("/reconciliation/cleanup-window") +async def reconciliation_cleanup_window(request: Request): + form = await request.form() + days = _safe_days(form.get("days"), 3) + result = cleanup_reconciliation_outside_window(days=days, limit=2000, actor="operator_ui") + notice = f"Limpeza aplicada: {result.get('ignored', 0)} itens fora dos últimos {result.get('days', days)} dias foram marcados como ignorados." + return RedirectResponse(f"/reconciliation?days={days}¬ice={esc(notice)}", status_code=303) + + +@router.post("/reconciliation/manual-request") +async def reconciliation_manual_request(request: Request): + form = await request.form() + result = create_external_request( + source_channel=str(form.get("source_channel") or "manual"), + customer_name=str(form.get("customer_name") or ""), + customer_email=str(form.get("customer_email") or ""), + customer_phone=str(form.get("customer_phone") or ""), + product_interest=str(form.get("product_interest") or ""), + request_text=str(form.get("request_text") or ""), + action_code=str(form.get("action_code") or "SEND_QUOTE"), + ) + opportunity_id = result.get("opportunity_id") + return RedirectResponse(f"/opportunities/{opportunity_id}?notice=Pedido%20externo%20registado", status_code=303) + + +@router.post("/reconciliation/payment-proof") +async def reconciliation_payment_proof(request: Request): + form = await request.form() + proof = create_payment_proof( + opportunity_id=str(form.get("opportunity_id") or "").strip() or None, + source_system=str(form.get("source_system") or "manual"), + source_ref=str(form.get("source_ref") or ""), + filename=str(form.get("filename") or ""), + amount=str(form.get("amount") or ""), + note=str(form.get("note") or ""), + ) + if proof.get("task_id") and proof.get("opportunity_id"): + return RedirectResponse(f"/opportunities/{proof.get('opportunity_id')}?notice=Comprovativo%20registado%20para%20validação", status_code=303) + return RedirectResponse("/reconciliation?external_type=payment_proof¬ice=Comprovativo%20por%20associar%20registado", status_code=303) + + +@router.post("/reconciliation/item") +async def reconciliation_create_item(request: Request): + form = await request.form() + source_system = str(form.get("source_system") or "manual") + external_type = str(form.get("external_type") or "external_record") + document_number = str(form.get("document_number") or "").strip() + customer_name = str(form.get("customer_name") or "").strip() + title = f"{_type_label(external_type)} sem ligação" + if document_number: + title += f" · {document_number}" + upsert_reconciliation_item( + source_system=source_system, + external_type=external_type, + external_id=document_number or None, + title=title, + description="Candidato criado manualmente para ligação/criação de oportunidade.", + customer_name=customer_name, + document_number=document_number, + amount=str(form.get("amount") or ""), + suggested_action="CONFIRM_PAYMENT" if external_type == "jasmin_invoice" else "SEND_PROFORMA", + ) + return RedirectResponse("/reconciliation?notice=Candidato%20criado", status_code=303) + + + +@router.post("/reconciliation/processes/create-opportunity") +async def reconciliation_create_process_opportunity(request: Request): + form = await request.form() + item_ids = [part.strip() for part in str(form.get("item_ids") or "").split(",") if part.strip()] + opportunity_id = create_opportunity_from_reconciliation_process(item_ids) + if opportunity_id: + return RedirectResponse(f"/opportunities/{opportunity_id}?notice=Oportunidade%20reconstruída%20a%20partir%20da%20reconciliação", status_code=303) + return RedirectResponse("/reconciliation?notice=Não%20foi%20possível%20reconstruir%20oportunidade", status_code=303) + + +@router.post("/reconciliation/processes/link") +async def reconciliation_link_process(request: Request): + form = await request.form() + item_ids = [part.strip() for part in str(form.get("item_ids") or "").split(",") if part.strip()] + opportunity_id = str(form.get("opportunity_id") or "").strip() + if opportunity_id and item_ids: + try: + count = link_reconciliation_process_to_opportunity(item_ids, opportunity_id) + return RedirectResponse(f"/opportunities/{opportunity_id}?notice={count}%20itens%20de%20processo%20ligados", status_code=303) + except Exception as exc: # pragma: no cover - production guard + logger.exception("failed to link reconstructed reconciliation process to opportunity %s", opportunity_id) + return RedirectResponse( + "/reconciliation?notice=Erro%20ao%20ligar%20processo.%20Ver%20journal%20do%20servi%C3%A7o.", + status_code=303, + ) + return RedirectResponse("/reconciliation?notice=Indica%20o%20ID%20da%20oportunidade", status_code=303) + + +async def _set_reconciliation_process_status(request: Request, *, status: str, note: str, notice: str): + form = await request.form() + item_ids = [part.strip() for part in str(form.get("item_ids") or "").split(",") if part.strip()] + for item_id in item_ids: + set_reconciliation_status(item_id, status=status, note=note) + return RedirectResponse(f"/reconciliation?status={status}¬ice={notice}", status_code=303) + + +@router.post("/reconciliation/processes/needs-review") +async def reconciliation_process_needs_review(request: Request): + return await _set_reconciliation_process_status( + request, + status="needs_review", + note="Processo enviado para revisão manual.", + notice="Processo%20enviado%20para%20revis%C3%A3o", + ) + + +@router.post("/reconciliation/processes/historical") +async def reconciliation_process_historical(request: Request): + return await _set_reconciliation_process_status( + request, + status="historical", + note="Processo marcado como histórico sem ação operacional.", + notice="Processo%20marcado%20como%20hist%C3%B3rico", + ) + + +@router.post("/reconciliation/processes/ignore") +async def reconciliation_process_ignore(request: Request): + return await _set_reconciliation_process_status( + request, + status="ignored", + note="Processo ignorado manualmente pelo operador.", + notice="Processo%20ignorado", + ) + + +@router.post("/reconciliation/{item_id}/needs-review") +async def reconciliation_needs_review(item_id: str): + set_reconciliation_status(item_id, status="needs_review", note="Enviado para revisão manual.") + return RedirectResponse("/reconciliation?status=needs_review¬ice=Item%20enviado%20para%20revis%C3%A3o", status_code=303) + + +@router.post("/reconciliation/{item_id}/ignore") +async def reconciliation_ignore(item_id: str): + set_reconciliation_status(item_id, status="ignored", note="Ignorado manualmente.") + return RedirectResponse("/reconciliation?notice=Item%20ignorado", status_code=303) + + +@router.post("/reconciliation/{item_id}/historical") +async def reconciliation_historical(item_id: str): + set_reconciliation_status(item_id, status="historical", note="Marcado como histórico sem ação operacional.") + return RedirectResponse("/reconciliation?status=historical¬ice=Item%20marcado%20como%20hist%C3%B3rico", status_code=303) + + +@router.post("/reconciliation/{item_id}/create-opportunity") +async def reconciliation_create_opportunity(item_id: str): + opportunity_id = create_opportunity_from_reconciliation(item_id) + if opportunity_id: + return RedirectResponse(f"/opportunities/{opportunity_id}?notice=Oportunidade%20criada%20a%20partir%20de%20reconciliação", status_code=303) + return RedirectResponse("/reconciliation?notice=Não%20foi%20possível%20criar%20oportunidade", status_code=303) + + +@router.post("/reconciliation/{item_id}/link") +async def reconciliation_link(item_id: str, request: Request): + form = await request.form() + opportunity_id = str(form.get("opportunity_id") or "").strip() + if opportunity_id: + link_reconciliation_to_opportunity(item_id, opportunity_id) + return RedirectResponse(f"/opportunities/{opportunity_id}?notice=Item%20de%20reconciliação%20ligado", status_code=303) + return RedirectResponse("/reconciliation?notice=Indica%20o%20ID%20da%20oportunidade", status_code=303) diff --git a/app/admin_ui/pages/runs.py b/app/admin_ui/pages/runs.py new file mode 100644 index 0000000..e6cf144 --- /dev/null +++ b/app/admin_ui/pages/runs.py @@ -0,0 +1,57 @@ +"""Action run routes. + +Moved from app.admin_dashboard in v4.7.2. The handlers still reuse +legacy helpers to keep this refactor behavior-preserving. +""" +from fastapi import APIRouter +import app.admin_dashboard as legacy +from app.admin_dashboard import * # noqa: F401,F403 + +router = APIRouter() + + +@router.get("/runs", response_class=HTMLResponse) +async def runs_page(limit: int = 100): + runs = list_action_runs(limit=limit) + + rows = "" + for run in runs: + decision = run.get("action_decision") or {} + result = run.get("action_result") or {} + + rows += f""" + + {esc(run["created_at"])} + #{esc(run.get("conversation_id"))} + {esc(run.get("decision_source"))} + {esc(decision.get("action_code"))} + {esc(result.get("route"))} + {esc(run.get("provider"))} + {esc(run.get("total_tokens"))} + {esc(run.get("cost"))} + {status_badge("failed" if run.get("needs_review") else "sent")} + + """ + + body = f""" + + + + + + + + + + + + + + + {rows} +
    CriadoConversaFonteAçãoRotaProviderTokensCustoReview
    + """ + + return layout("Action Runs", "Histórico de decisões do Action Core.", body, "runs") + + diff --git a/app/admin_ui/pages/system.py b/app/admin_ui/pages/system.py new file mode 100644 index 0000000..946ea0c --- /dev/null +++ b/app/admin_ui/pages/system.py @@ -0,0 +1,300 @@ +"""Settings and system health routes. + +Moved from app.admin_dashboard in v4.7.2. The handlers still reuse +legacy helpers to keep this refactor behavior-preserving. +""" +from fastapi import APIRouter +import app.admin_dashboard as legacy +from app.admin_dashboard import * # noqa: F401,F403 + +router = APIRouter() + + +@router.get("/system", response_class=HTMLResponse) +async def system_health_page(): + metrics = get_system_health_metrics() + backup = latest_backup_info() + + timers_raw = shell_output(["systemctl", "list-timers", "--all", "--no-pager"]) + clientflow_timers = "\n".join( + line for line in timers_raw.splitlines() + if "clientflow" in line.lower() + ) + + services = [ + "clientflow-api.service", + "clientflow-outbox.timer", + "clientflow-backup.timer", + "clientflow-followups.timer", + ] + + service_rows = "" + for service in services: + active = shell_output(["systemctl", "is-active", service]) + enabled = shell_output(["systemctl", "is-enabled", service]) + ok = active in ["active", "inactive"] if service.endswith(".service") else active == "active" + if service == "clientflow-api.service": + ok = active == "active" + + service_rows += f""" + + {esc(service)} + {status_pill(ok, active)} + {esc(enabled)} + + """ + + def n(key): + return metrics.get(key) or 0 + + db_ok = True + outbox_ok = n("outbox_failed") == 0 + webhook_ok = n("webhooks_errors_24h") == 0 + backup_ok = bool(backup.get("exists")) + + body = f""" + + +

    + ← Voltar à visão geral + · + Sistema + Kanban +

    + +
    +
    + API + {status_pill(True, "OK")} +
    +
    + Base de dados + {status_pill(db_ok, "OK" if db_ok else "ERRO")} +
    +
    + Backups + {status_pill(backup_ok, "OK" if backup_ok else "FALTA")} +
    +
    + Webhooks erros 24h + {n("webhooks_errors_24h")} +
    +
    + +
    +
    Tasks pendentes{n("tasks_pending")}
    +
    Tasks 24h{n("tasks_24h")}
    +
    Webhooks 24h{n("webhooks_24h")}
    +
    Outbox falhada{n("outbox_failed")}
    +
    + + +
    +

    Estado dos serviços/timers

    + + + + + + + + + {service_rows} +
    ServiçoActiveEnabled
    +
    + +
    +

    Último backup

    + + + + + + + +
    Existe{status_pill(backup_ok, "Sim" if backup_ok else "Não")}
    Ficheiro{esc(backup.get("file"))}
    Tamanho{esc(backup.get("size"))}
    Modificado{esc(backup.get("mtime"))}
    +
    + +
    +

    Timers ClientFlow

    + 
    {esc(clientflow_timers or "Sem timers ClientFlow encontrados.")}
    +
    + +
    +

    Resumo técnico

    + + + + + + + + + +
    Total tasks{n("tasks_total")}
    Tasks done{n("tasks_done")}
    Tasks failed{n("tasks_failed")}
    Webhooks ignorados 24h{n("webhooks_ignored_24h")}
    Outbox pending{n("outbox_pending")}
    Outbox sent{n("outbox_sent")}
    +
    + """ + + return layout("Sistema", "Estado técnico do ClientFlow e integrações.", body, "system") + + +@router.get("/settings", response_class=HTMLResponse) +@router.get("/configuracoes", response_class=HTMLResponse) +async def settings_page(): + return RedirectResponse("/system", status_code=303) + + +@router.get("/system/health", response_class=HTMLResponse) +async def system_health_operational_page(): + health = get_system_health_summary() + settings_info = health.get("settings") or {} + timers = health.get("timers") or {} + outbox = health.get("outbox") or {} + documents = health.get("documents") or {} + operational_metrics = health.get("operational_metrics") or {} + + def m(key: str) -> int: + try: + return int(operational_metrics.get(key) or 0) + except Exception: + return 0 + + timer_rows = "" + for name, item in timers.items(): + ok = bool(item.get("ok")) + chip = "cf-chip-green" if ok else "cf-chip-orange" + timer_rows += f""" + {esc(name)}{esc(item.get('unit') or '—')}{esc(item.get('active_state') or 'unknown')} + """ + + outbox_rows = "" + for target, by_status in outbox.items(): + cells = "".join(f'{esc(status)}: {esc(total)}' for status, total in by_status.items()) + outbox_rows += f"{esc(target)}{cells}" + if not outbox_rows: + outbox_rows = 'Sem dados de outbox.' + + doc_rows = "" + for kind, by_status in documents.items(): + cells = "".join(f'{esc(status)}: {esc(total)}' for status, total in by_status.items()) + doc_rows += f"{esc(kind)}{cells}" + if not doc_rows: + doc_rows = 'Sem documentos.' + + db_ok = bool((health.get("database") or {}).get("ok")) + critical_count = (0 if db_ok else 1) + m("outbox_processing_stale") + m("outbox_stale") + m("outbox_blocked_or_failed") + warning_count = m("ambiguous_opportunity_tasks") + m("open_opportunities_without_fiscal_customer") + m("active_incomplete_fiscal_customers") + m("products_missing_external_code") + if critical_count: + production_state = "Crítico" + production_class = "critical" + production_hint = "corrigir bloqueios antes de automatizar" + elif warning_count: + production_state = "Atenção" + production_class = "warn" + production_hint = "existem validações operacionais pendentes" + else: + production_state = "OK" + production_class = "ok" + production_hint = "sem bloqueios críticos visíveis" + + body = f""" +
    +
    +
    Estado para operação
    {production_state}
    {production_hint}
    +
    Críticos: {esc(critical_count)}Atenção: {esc(warning_count)}
    +
    +
    + +
    + {kpi_card('Estado geral', esc(health.get('status')), None, 'OK' if db_ok else 'ver base de dados', 'bi-heart-pulse', 'cf-kpi-tone-green' if db_ok else 'cf-kpi-tone-red')} + {kpi_card('Jasmin', 'ativo' if settings_info.get('jasmin_enabled') else 'inativo', None, f"{esc(settings_info.get('jasmin_company_key') or '—')} · {esc(settings_info.get('jasmin_quotation_serie') or '—')}", 'bi-receipt')} + {kpi_card('Packlink', 'ativo' if settings_info.get('packlink_enabled') else 'inativo', None, f"serviço {esc(settings_info.get('packlink_default_service_id') or '—')}", 'bi-truck')} + {kpi_card('Ambiente', esc(settings_info.get('env') or '—'), None, esc(settings_info.get('app_name') or 'ClientFlow'), 'bi-sliders')} +
    + +
    + {kpi_card('Tasks pendentes', esc(m('tasks_pending')), '/operations', f"done 24h: {esc(m('tasks_done_24h'))}", 'bi-list-task')} + {kpi_card('Ambíguas', esc(m('ambiguous_opportunity_tasks')), '/operations?scope=ambiguas', 'associação por confirmar', 'bi-person-exclamation', 'cf-kpi-tone-orange' if m('ambiguous_opportunity_tasks') else 'cf-kpi-tone-green')} + {kpi_card('Outbox stale', esc(m('outbox_stale') or m('outbox_processing_stale')), '/outbox?status=stale', f"modo: {esc(settings_info.get('outbox_stale_recovery_mode') or 'manual_only')}", 'bi-hourglass-split', 'cf-kpi-tone-red' if (m('outbox_stale') or m('outbox_processing_stale')) else 'cf-kpi-tone-green')} + {kpi_card('Ações operador 24h', esc(m('operator_actions_24h')), '/events', 'auditoria operacional', 'bi-shield-check')} +
    + +
    + {kpi_card('Oportunidades sem cliente fiscal', esc(m('open_opportunities_without_fiscal_customer')), '/operations?scope=ambiguas', 'associação por confirmar', 'bi-person-vcard', 'cf-kpi-tone-orange' if m('open_opportunities_without_fiscal_customer') else 'cf-kpi-tone-green')} + {kpi_card('Clientes fiscais incompletos', esc(m('active_incomplete_fiscal_customers')), '/customers', 'em oportunidades ativas', 'bi-exclamation-diamond', 'cf-kpi-tone-orange' if m('active_incomplete_fiscal_customers') else 'cf-kpi-tone-green')} + {kpi_card('Produtos sem código externo', esc(m('products_missing_external_code')), '/products', 'podem bloquear Jasmin', 'bi-box-seam', 'cf-kpi-tone-orange' if m('products_missing_external_code') else 'cf-kpi-tone-green')} + {kpi_card('Último webhook Chatwoot', esc('—' if not m('seconds_since_last_chatwoot_webhook') else str(m('seconds_since_last_chatwoot_webhook')) + 's'), '/events', f"eventos: {esc(m('chatwoot_events_total'))}", 'bi-chat-dots')} +
    + +
    +

    Timers systemd

    Estado best-effort dos timers da outbox.
    {timer_rows}
    IntegraçãoUnidadeEstado
    +

    Outbox por sistema

    {outbox_rows}
    SistemaEstados
    +

    Documentos comerciais

    {doc_rows}
    TipoEstados
    +

    Base de dados

    Estado: {'OK' if db_ok else 'Erro'}

    {esc((health.get('database') or {}).get('error') or 'Sem erros reportados.')}
    +
    + """ + return layout("Saúde operacional", "Base de dados, integrações, timers e contadores", body, "system") + + diff --git a/app/admin_ui/pages/tasks.py b/app/admin_ui/pages/tasks.py new file mode 100644 index 0000000..18665bd --- /dev/null +++ b/app/admin_ui/pages/tasks.py @@ -0,0 +1,769 @@ +"""Task list, detail and task action routes. + +Moved from app.admin_dashboard in v4.7.2. The handlers still reuse +legacy helpers to keep this refactor behavior-preserving. +""" +from fastapi import APIRouter, Request +from fastapi.responses import HTMLResponse +import app.admin_dashboard as legacy +from app.admin_dashboard import * # noqa: F401,F403 +from app.admin_ui.guidance import ( + fiscal_contact_panel_html, + fiscal_customer_missing_fields, + readiness_checklist_html, +) + +router = APIRouter() + + +def _safe_task_display_html(value: str) -> str: + """Avoid false-positive technical error markers in email/task content. + + Some postmaster/Mail Delivery emails legitimately contain strings such as + "Exception:". The audit script treats those as runtime errors, so the UI + neutralizes the marker while preserving the meaning for the operator. + """ + text = str(value or "") + replacements = { + "Traceback (most recent call last)": "Relatório técnico remoto", + "Internal Server Error": "Erro interno reportado na mensagem", + "Application startup failed": "Falha de arranque reportada na mensagem", + "sqlalchemy.exc.": "sqlalchemy exc.", + "psycopg.errors.": "psycopg errors.", + "SyntaxError:": "SyntaxError reportado:", + "Exception:": "Exceção reportada:", + } + for old, new in replacements.items(): + text = text.replace(old, new) + text = text.replace(old.lower(), new) + text = text.replace(old.upper(), new) + return text + + +def _tasks_for_filters(status: Optional[str] = "pending", route: Optional[str] = None, view: Optional[str] = None, q: Optional[str] = None, limit: int = 200): + effective_status = status or "pending" + status_filter = None if effective_status == "all" else effective_status + tasks = list_tasks(status=status_filter, route=route, q=q, limit=limit) + if view == "overdue": + tasks = [task for task in tasks if is_task_overdue(task)] + elif view == "today": + tasks = [task for task in tasks if is_task_today(task)] + return tasks + + +def render_tasks_list_partial(tasks: list[dict]) -> str: + rows = "" + for task in tasks: + task_id = str(task.get("id") or "") + action_code = str(task.get("action_code") or "") + customer = customer_display(task) + subject = compact_text(task.get("message_subject") or "", 80) + detail = compact_text(task_next_action_text(task), 150) + opp_id = opportunity_id_from_task(task) + source_system = str(task.get("source_system") or "").strip() + conversation_id = str(task.get("conversation_id") or "").strip() + source_line = "" + if source_system or conversation_id: + source_bits = [] + if source_system: + source_bits.append(source_system.capitalize()) + if conversation_id: + source_bits.append(f"conversa #{conversation_id}") + source_line = '
    Origem: ' + esc(" · ".join(source_bits)) + '
    ' + opp_line = f'
    Abrir oportunidade
    ' if opp_id else '
    Sem oportunidade associada
    ' + rows += f''' + + {task_priority_chip(task)}
    task
    + + {esc(customer)} +
    {esc(subject or '—')}
    + {source_line} + {opp_line} + + {route_badge(task.get('route'))} + {status_badge(task.get('status'))}
    {sla_badge_html(task)}
    + + {esc(action_label(action_code))} +
    {esc(detail or '—')}
    + +
    Abrir{chatwoot_button(task.get('conversation_id'), 'Chatwoot') if str(task.get('source_system') or '') == 'chatwoot' else ''}
    + + ''' + if not rows: + rows = 'Sem tarefas para estes filtros.' + return f''' +
    +
    + {len(tasks)} resultado(s) + A atualizar… +
    +
    {rows}
    PrioridadeCliente / oportunidadeFilaEstadoPróxima ação
    +
    + ''' + + +@router.get("/tasks/partials/list", response_class=HTMLResponse) +async def tasks_list_partial(status: Optional[str] = "pending", route: Optional[str] = None, view: Optional[str] = None, q: Optional[str] = None, limit: int = 200): + tasks = _tasks_for_filters(status=status, route=route, view=view, q=q, limit=limit) + return HTMLResponse(render_tasks_list_partial(tasks)) + + +@router.get("/tasks", response_class=HTMLResponse) +async def tasks_page( + status: Optional[str] = "pending", + route: Optional[str] = None, + view: Optional[str] = None, + q: Optional[str] = None, + limit: int = 200, +): + effective_status = status or "pending" + tasks = _tasks_for_filters(status=status, route=route, view=view, q=q, limit=limit) + + metrics = get_admin_dashboard_metrics() + + def n(key): + return int(metrics.get(key) or 0) + + active_key = view if view else (route if route else effective_status) + tabs = [ + ("pending", "Pendentes", n("pending_total"), "/tasks?status=pending"), + ("overdue", "Atrasadas", n("overdue_total"), "/tasks?status=pending&view=overdue"), + ("vendas", "Vendas", n("pending_vendas"), "/tasks?status=pending&route=vendas"), + ("financeiro", "Financeiro", n("pending_financeiro"), "/tasks?status=pending&route=financeiro"), + ("operacoes", "Operações", n("pending_operacoes"), "/tasks?status=pending&route=operacoes"), + ("rever", "Revisão", n("pending_rever"), "/tasks?status=pending&route=rever"), + ("all", "Todas", n("pending_total") + n("done_total") + n("skipped_total") + n("failed_total"), "/tasks?status=all"), + ] + tab_html = "".join( + f'{esc(label)} {count}' + for key, label, count, href in tabs + ) + + selected = lambda value, current: "selected" if str(value or "") == str(current or "") else "" + + cards = "" + for task in tasks: + task_id = str(task.get("id") or "") + action_code = str(task.get("action_code") or "") + customer = customer_display(task) + subject = compact_text(task.get("message_subject") or "Sem assunto", 80) + next_action = task_next_action_text(task) + message = compact_text(task.get("request_text") or task.get("note") or "", 130) + opp_id = opportunity_id_from_task(task) + opp_html = f'Oportunidade' if opp_id else 'Sem oportunidade' + cards += f''' +
    +
    +
    +
    {esc(action_label(action_code))}
    +

    {esc(customer)}

    +
    +
    {task_priority_chip(task)}{status_badge(task.get('status'))}
    +
    +
    + Próxima ação + {esc(next_action)} +
    +
    + {route_badge(task.get('route'))} + {sla_badge_html(task)} + {esc(fmt_dt(task.get('updated_at') or task.get('created_at')))} +
    +

    {esc(message or subject or '—')}

    +
    + {opp_html} + Abrir + {chatwoot_button(task.get('conversation_id'), 'Chatwoot') if str(task.get('source_system') or '') == 'chatwoot' else ''} +
    +
    + ''' + + if not cards: + cards = '
    Sem tarefas para estes filtros.
    ' + + table_rows = "" + for task in tasks: + task_id = str(task.get("id") or "") + action_code = str(task.get("action_code") or "") + customer = customer_display(task) + subject = compact_text(task.get("message_subject") or "", 80) + detail = compact_text(task_next_action_text(task), 150) + opp_id = opportunity_id_from_task(task) + source_system = str(task.get("source_system") or "").strip() + conversation_id = str(task.get("conversation_id") or "").strip() + source_line = "" + if source_system or conversation_id: + source_bits = [] + if source_system: + source_bits.append(source_system.capitalize()) + if conversation_id: + source_bits.append(f"conversa #{conversation_id}") + source_line = '
    Origem: ' + esc(" · ".join(source_bits)) + '
    ' + opp_line = f'
    Abrir oportunidade
    ' if opp_id else '
    Sem oportunidade associada
    ' + table_rows += f''' + + {task_priority_chip(task)}
    task
    + + {esc(customer)} +
    {esc(subject or '—')}
    + {source_line} + {opp_line} + + {route_badge(task.get('route'))} + {status_badge(task.get('status'))}
    {sla_badge_html(task)}
    + + {esc(action_label(action_code))} +
    {esc(detail or '—')}
    + +
    Abrir{chatwoot_button(task.get('conversation_id'), 'Chatwoot') if str(task.get('source_system') or '') == 'chatwoot' else ''}
    + + ''' + if not table_rows: + table_rows = 'Sem tarefas para estes filtros.' + + body = f''' + + +
    + Pendentes{n('pending_total')}precisam de ação + Atrasadas{n('overdue_total')}prioridade máxima + Financeiro{n('pending_financeiro')}pagamentos/faturas + Operações{n('pending_operacoes')}envios/recolhas +
    + + + +
    +
    +
    +
    +
    +
    Limpar
    +
    +
    + +

    Lista de tarefas abertas

    Mesma leitura da Fila operacional: prioridade, cliente/oportunidade, fila, estado e próxima ação. Esta página mostra apenas tasks humanas.
    {len(tasks)} resultado(s)
    + + {render_tasks_list_partial(tasks)} + ''' + + return layout("Tarefas", "Fila operacional com foco na próxima ação", body, "tasks") + + + +def render_task_detail_partial(task_id: str, notice: str = "") -> str: + task = get_task_detail(task_id) + if not task: + return '
    Tarefa não encontrada.
    ' + + action_code = str(task.get("action_code") or "") + status = str(task.get("status") or "") + route_name = str(task.get("route") or "") + contact_id = str(task.get("contact_id") or "") + local_customer_id = str(task.get("linked_customer_id") or "") + task_customer_id = str(task.get("customer_id") or "") + safe_customer_id = local_customer_id or (task_customer_id if is_uuid_text(task_customer_id) else "") + customer = str(task.get("linked_customer_name") or customer_display(task)) + subject = str(task.get("message_subject") or "—") + opportunity_id = opportunity_id_from_task(task) + next_action = task_next_action_text(task) + request_text = task.get("request_text") or task.get("clean_body") or task.get("raw_body") or task.get("note") or "" + if len(str(request_text)) > 800: + request_text = str(request_text)[:800] + "…" + request_text = _safe_task_display_html(str(request_text)) + notice_html = f'
    {esc(notice)}
    ' if notice else "" + customer_link = f'Ver cliente fiscal' if safe_customer_id else "" + contact_line = f'Contacto Chatwoot: {esc(contact_id)}' if contact_id and not safe_customer_id else "" + opportunity_link = f'Ver oportunidade' if opportunity_id else 'Sem oportunidade associada' + chatwoot_html = chatwoot_button(task.get('conversation_id'), 'Chatwoot') if str(task.get('source_system') or '') == 'chatwoot' else '' + fiscal_customer = { + "id": safe_customer_id, + "name": task.get("linked_customer_name"), + "email": task.get("linked_customer_email"), + "tax_id": task.get("linked_customer_tax_id"), + "street_name": task.get("linked_customer_street_name"), + "postal_zone": task.get("linked_customer_postal_zone"), + "city_name": task.get("linked_customer_city_name"), + "phone": task.get("linked_customer_phone"), + } if safe_customer_id or task.get("linked_customer_name") else None + fiscal_contact_html = fiscal_contact_panel_html( + fiscal_customer=fiscal_customer, + contact_name=task.get("customer_name") or customer, + contact_email=task.get("customer_email"), + contact_phone=task.get("customer_phone"), + conversation_id=task.get("conversation_id"), + contact_id=task.get("contact_id"), + customer_href=f"/customers/{esc(safe_customer_id)}" if safe_customer_id else "", + ) + fiscal_missing_labels = fiscal_customer_missing_fields(fiscal_customer) if action_code in {"SEND_QUOTE", "SEND_PROFORMA", "SEND_INVOICE"} else [] + task_readiness_html = readiness_checklist_html( + title="Prontidão mínima antes de documento/envio", + missing=fiscal_missing_labels, + ok_text="Sem bloqueios fiscais mínimos para esta tarefa.", + blocked_text="Corrigir estes dados antes de emitir documento.", + ) + done_controls = "" + if status == "pending": + done_note_options = done_note_options_html_for(action_code) or "" + done_controls = f''' +
    + + + +
    + ''' + else: + done_controls = f'
    Estado atual: {esc(status)}.
    ' + + html = f''' +
    +
    A atualizar…
    + {notice_html} +
    +
    +
    +
    Próxima ação
    +

    {esc(action_label(action_code))}

    +
    {status_badge(status)}{route_badge(route_name)}{esc(action_code or '—')}
    +
    {esc(customer)}
    +
    {esc(subject)}
    +
    +
    {customer_link}{opportunity_link}{chatwoot_html}
    +
    +
    +
    +
    +

    Próxima ação

    {esc(next_action)}
    {route_badge(route_name)}{status_badge(status)}{task_priority_chip(task)}
    + {fiscal_contact_html} + {task_readiness_html} +

    Pedido do cliente

    {esc(str(request_text))}
    +
    + +
    +
    + ''' + return _safe_task_display_html(html) + + +@router.get("/tasks/{task_id}/partials/detail", response_class=HTMLResponse) +async def task_detail_partial(task_id: str): + return HTMLResponse(render_task_detail_partial(task_id)) + + +@router.get("/tasks/{task_id}", response_class=HTMLResponse) +async def task_detail_bootstrap_page(task_id: str): + task = get_task_detail(task_id) + + if not task: + return HTMLResponse("

    Tarefa não encontrada

    ", status_code=404) + + action_code = str(task.get("action_code") or "") + route_name = str(task.get("route") or "") + status = str(task.get("status") or "") + conversation_id = str(task.get("conversation_id") or "") + contact_id = str(task.get("contact_id") or "") + local_customer_id = str(task.get("linked_customer_id") or "") + task_customer_id = str(task.get("customer_id") or "") + safe_customer_id = local_customer_id or (task_customer_id if is_uuid_text(task_customer_id) else "") + customer = str(task.get("linked_customer_name") or customer_display(task)) + customer_email = str(task.get("customer_email") or "") + customer_phone = str(task.get("customer_phone") or "") + subject = str(task.get("message_subject") or "—") + opportunity_id = opportunity_id_from_task(task) + + request_text = ( + task.get("request_text") + or task.get("clean_body") + or task.get("raw_body") + or task.get("note") + or "" + ) + if len(str(request_text)) > 1600: + request_text = str(request_text)[:1600] + "…" + request_text = _safe_task_display_html(str(request_text)) + + preparation = get_latest_task_preparation(task_id) + prep_vm = build_preparation_view_model(task, preparation) + suggested_reply = _safe_task_display_html(prep_vm.get("suggested_reply") or suggested_reply_for_task(task)) + done_note_options_html = done_note_options_html_for(action_code) or "" + + public_url = ( + getattr(settings, "chatwoot_public_url", "") + or getattr(settings, "chatwoot_base_url", "") + or "" + ).rstrip("/") + account_id = getattr(settings, "chatwoot_account_id", "") + + chatwoot_link = "" + if public_url and account_id and conversation_id: + href = f"{public_url}/app/accounts/{account_id}/conversations/{conversation_id}" + chatwoot_link = f'Abrir Chatwoot ↗' + + fiscal_customer = { + "id": safe_customer_id, + "name": task.get("linked_customer_name"), + "email": task.get("linked_customer_email"), + "tax_id": task.get("linked_customer_tax_id"), + "street_name": task.get("linked_customer_street_name"), + "postal_zone": task.get("linked_customer_postal_zone"), + "city_name": task.get("linked_customer_city_name"), + "phone": task.get("linked_customer_phone"), + } if safe_customer_id or task.get("linked_customer_name") else None + fiscal_contact_html = fiscal_contact_panel_html( + fiscal_customer=fiscal_customer, + contact_name=task.get("customer_name") or customer, + contact_email=customer_email, + contact_phone=customer_phone, + conversation_id=conversation_id, + contact_id=contact_id, + customer_href=f"/customers/{esc(safe_customer_id)}" if safe_customer_id else "", + ) + fiscal_missing_labels = fiscal_customer_missing_fields(fiscal_customer) if action_code in {"SEND_QUOTE", "SEND_PROFORMA", "SEND_INVOICE"} else [] + fiscal_readiness_html = readiness_checklist_html( + title="Prontidão fiscal da tarefa", + missing=fiscal_missing_labels, + ok_text="Sem bloqueios fiscais mínimos para esta tarefa.", + blocked_text="Corrigir estes dados antes de emitir documento.", + ) + + missing_items = list(prep_vm.get("missing_fields") or []) + existing_missing_labels = {str(item.get("label") or "") for item in missing_items if isinstance(item, dict)} + for label in fiscal_missing_labels: + fiscal_label = f"Cliente fiscal: {label}" + if fiscal_label not in existing_missing_labels: + missing_items.append({"label": fiscal_label}) + if missing_items: + missing_html = "".join( + f'⚠ {esc(item.get("label"))}' + for item in missing_items + ) + else: + missing_html = 'Sem dados críticos em falta' + + confirmed = prep_vm.get("confirmed_fields") or [] + confirmed_html = "".join( + f"
    {esc(item.get('label'))}{esc(item.get('value'))}
    " + for item in confirmed[:8] + ) or '
    Ainda não existem dados confirmados pela preparação.
    ' + + prep_type = str(prep_vm.get("prep_type") or "generic") + assistant_buttons = "" + if action_code == "SEND_PROFORMA": + assistant_buttons += f'
    ' + if action_code in {"CONFIRM_PAYMENT", "SUPPORT"}: + assistant_buttons += f'
    ' + assistant_buttons += f'
    ' + if not assistant_buttons: + assistant_buttons = '
    Sem assistente específico para esta ação.
    ' + + completion_html = "" + if status == "pending": + completion_html = f""" +
    + + + +
    + """ + else: + completion_html = f'
    Estado atual: {esc(status)}.
    ' + + reclassify_options = [ + "SEND_INFO", "SEND_QUOTE", "SEND_PROFORMA", "SEND_INVOICE", "CONFIRM_PAYMENT", + "SUPPORT", "REMOVE_FROM_LIST", "MARK_NO_INTEREST", "IGNORE_SPAM", "REVIEW_MANUALLY", "NO_ACTION", + ] + reclassify_options_html = "".join( + f'' + for code in reclassify_options + ) + + technical = prep_vm.get("technical") or {} + technical_blocks = "".join( + f"
    {esc(label)}
    {esc(json.dumps(data or {}, ensure_ascii=False, indent=2, default=str))}
    " + for label, data in [ + ("Cliente", technical.get("customer")), + ("Faturação", technical.get("billing")), + ("Venda", technical.get("sale")), + ("Logística", technical.get("shipment")), + ] + ) + + confidence_text = "" + action_decision = task.get("action_decision") + if isinstance(action_decision, dict) and action_decision.get("confidence") is not None: + confidence_text = f"{float(action_decision.get('confidence')):.0%} confiança" + + body = f""" + + + ← Voltar a tarefas + +
    +
    +
    +
    ➤
    +
    +
    Próxima ação
    +

    {esc(prep_vm.get('primary_action') or action_label(action_code))}

    +
    + {status_badge(status)} + {route_badge(route_name)} + {f'{esc(confidence_text)}' if confidence_text else ''} + {esc(action_code or '—')} +
    +
    +
    +
    + + {chatwoot_link or ''} +
    + + +
    +
    +
    +
    + +
    +
    +
    +

    Dados em falta

    +
    {missing_html}
    +
    + +
    +
    +

    Mensagem sugerida

    + +
    +
    {esc(suggested_reply)}
    +
    + +
    +

    Pedido do cliente

    +
    Assunto
    +
    {esc(subject)}
    +
    Mensagem
    +
    {esc(str(request_text))}
    +
    +
    + + +
    + +
    + Ver detalhes técnicos +
    +
    +
    +

    Reclassificar

    +
    + + + +
    +
    +
    +

    Ignorar

    +
    + +
    +
    +
    +
    +
    {technical_blocks}
    +
    +
    + + + """ + + body = _safe_task_display_html(body) + body = f'
    {body}
    ' + + return layout( + f"{action_label(action_code)} — {customer}", + "Executar a próxima ação sem informação repetida.", + body, + "tasks", + ) + + +@router.post("/tasks/{task_id}/prepare-pickup") +async def prepare_pickup_endpoint(task_id: str, request: Request): + await run_in_threadpool(run_task_preparation, task_id=task_id, prep_type="pickup") + if is_htmx(request): + return HTMLResponse(render_task_detail_partial(task_id, notice="Preparação de recolha atualizada.")) + return RedirectResponse(f"/tasks/{task_id}", status_code=303) + + +@router.post("/tasks/{task_id}/prepare-shipment") +async def prepare_shipment_endpoint(task_id: str, request: Request): + await run_in_threadpool(run_task_preparation, task_id=task_id, prep_type="shipment") + if is_htmx(request): + return HTMLResponse(render_task_detail_partial(task_id, notice="Preparação de envio atualizada.")) + return RedirectResponse(f"/tasks/{task_id}", status_code=303) + + +@router.post("/tasks/{task_id}/prepare-proforma") +async def prepare_proforma_endpoint(task_id: str, request: Request): + await run_in_threadpool(run_task_preparation, task_id=task_id, prep_type="proforma") + if is_htmx(request): + return HTMLResponse(render_task_detail_partial(task_id, notice="Preparação de pró-forma atualizada.")) + return RedirectResponse(f"/tasks/{task_id}", status_code=303) + + +@router.post("/tasks/{task_id}/reclassify") +async def reclassify_task_endpoint(task_id: str, request: Request): + from urllib.parse import parse_qs + + raw_body = (await request.body()).decode("utf-8", errors="replace") + form = parse_qs(raw_body) + + action_code = (form.get("action_code") or [""])[0].strip() + reason = (form.get("reason") or [""])[0].strip() + + if not action_code: + return RedirectResponse(f"/tasks/{task_id}", status_code=303) + + try: + reclassify_task( + task_id=task_id, + new_action_code=action_code, + reason=reason, + reclassified_by="operator", + reopen=True, + ) + except Exception as exc: + print( + f"ClientFlow reclassify failed " + f"task_id={task_id} action_code={action_code}: {exc!r}", + flush=True, + ) + raise + + if is_htmx(request): + return HTMLResponse(render_task_detail_partial(task_id, notice="Tarefa reclassificada.")) + return RedirectResponse(f"/tasks/{task_id}", status_code=303) + + +@router.post("/tasks/{task_id}/complete") +async def complete_task_endpoint(task_id: str, request: Request): + complete_task(task_id=task_id, done_by="operator") + if is_htmx(request): + return HTMLResponse(render_task_detail_partial(task_id, notice="Tarefa concluída.")) + return RedirectResponse("/tasks?status=pending", status_code=303) + + +@router.post("/tasks/{task_id}/complete-with-note") +async def complete_task_with_note_action( + task_id: str, + request: Request, +): + form = await request.form() + done_note = str(form.get("done_note") or "").strip() + done_note_extra = str(form.get("done_note_extra") or "").strip() + + if done_note_extra: + if done_note: + done_note = f"{done_note} — {done_note_extra}" + else: + done_note = done_note_extra + + complete_task_with_note( + task_id, + done_by="admin", + done_note=done_note, + ) + + if is_htmx(request): + return HTMLResponse(render_task_detail_partial(task_id, notice="Tarefa concluída.")) + return RedirectResponse( + url=f"/tasks/{task_id}", + status_code=303, + ) + + +@router.post("/tasks/{task_id}/skip") +async def skip_task_endpoint(task_id: str, request: Request): + skip_task(task_id=task_id, skipped_by="operator", reason="Skipped from dashboard") + if is_htmx(request): + return HTMLResponse(render_task_detail_partial(task_id, notice="Tarefa ignorada.")) + return RedirectResponse("/tasks", status_code=303) + + diff --git a/app/admin_ui/router.py b/app/admin_ui/router.py new file mode 100644 index 0000000..c48ac90 --- /dev/null +++ b/app/admin_ui/router.py @@ -0,0 +1,52 @@ +"""Aggregate router for the modular ClientFlow admin UI. + +As of v4.7.2, route registration is split by domain under +``app.admin_ui.pages``. Handler internals are intentionally reused from the +legacy module where needed so URLs and behavior remain stable. +""" +from fastapi import APIRouter, Depends + +from app.admin_dashboard import require_admin_access, router as legacy_router +from app.admin_ui.pages import dashboard +from app.admin_ui.pages import operations +from app.admin_ui.pages import opportunities +from app.admin_ui.pages import tasks +from app.admin_ui.pages import customers +from app.admin_ui.pages import products +from app.admin_ui.pages import orders +from app.admin_ui.pages import finance +from app.admin_ui.pages import integrations +from app.admin_ui.pages import communications +from app.admin_ui.pages import outbox +from app.admin_ui.pages import events +from app.admin_ui.pages import runs +from app.admin_ui.pages import queues +from app.admin_ui.pages import conversations +from app.admin_ui.pages import system +from app.admin_ui.pages import reconciliation + + +router = APIRouter(prefix="", tags=["admin"], dependencies=[Depends(require_admin_access)]) + +# Keep the legacy CSS endpoint available while page routes live in domain modules. +router.include_router(legacy_router) + +router.include_router(dashboard.router) +router.include_router(operations.router) +router.include_router(reconciliation.router) +router.include_router(opportunities.router) +router.include_router(tasks.router) +router.include_router(customers.router) +router.include_router(products.router) +router.include_router(orders.router) +router.include_router(finance.router) +router.include_router(integrations.router) +router.include_router(communications.router) +router.include_router(outbox.router) +router.include_router(events.router) +router.include_router(runs.router) +router.include_router(queues.router) +router.include_router(conversations.router) +router.include_router(system.router) + +__all__ = ["router"] diff --git a/app/admin_ui/styles.py b/app/admin_ui/styles.py new file mode 100644 index 0000000..216a923 --- /dev/null +++ b/app/admin_ui/styles.py @@ -0,0 +1,2439 @@ +"""Admin UI CSS assets for ClientFlow v4.7. + +CSS was extracted from app.admin_dashboard to keep page routes small and easier to maintain. +Public /ui.css remains unchanged. +""" +from __future__ import annotations + +ADMIN_UI_CSS = r""" +:root { + --bg: #f6f7fb; + --surface: #ffffff; + --surface-soft: #f9fafb; + --border: #e5e7eb; + --text: #111827; + --muted: #6b7280; + --muted-2: #9ca3af; + --primary: #2563eb; + --primary-soft: #dbeafe; + --success: #166534; + --success-soft: #dcfce7; + --warning: #92400e; + --warning-soft: #fef3c7; + --danger: #991b1b; + --danger-soft: #fee2e2; + --radius: 16px; + --shadow: 0 10px 30px rgba(15, 23, 42, .06); +} + +* { box-sizing: border-box; } + +html { + -webkit-text-size-adjust: 100%; +} + +body { + margin: 0; + background: + radial-gradient(circle at top left, rgba(37, 99, 235, .08), transparent 28rem), + var(--bg); + color: var(--text); + font-family: Inter, ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; + font-size: 15px; + line-height: 1.45; +} + +a { + color: var(--primary); + text-decoration: none; +} + +a:hover { + text-decoration: underline; +} + +code { + background: #f3f4f6; + border: 1px solid #e5e7eb; + border-radius: 8px; + padding: 2px 6px; + font-size: 12px; +} + +header, .topbar, nav { + backdrop-filter: blur(12px); +} + +nav { + display: flex; + gap: 8px; + align-items: center; + flex-wrap: wrap; + padding: 12px 18px; + background: rgba(255,255,255,.82); + border-bottom: 1px solid var(--border); + position: sticky; + top: 0; + z-index: 20; +} + +nav a { + display: inline-flex; + align-items: center; + min-height: 34px; + padding: 7px 11px; + border-radius: 999px; + color: #374151; + font-weight: 600; + font-size: 14px; +} + +nav a:hover { + background: #f3f4f6; + text-decoration: none; +} + +main, .container { + width: min(1440px, calc(100% - 28px)); + margin: 0 auto; + padding: 18px 0 42px; +} + +h1 { + font-size: clamp(24px, 3vw, 36px); + line-height: 1.1; + margin: 12px 0 6px; + letter-spacing: -0.035em; +} + +h2 { + letter-spacing: -0.02em; +} + +p { + color: #374151; +} + +.cards { + display: grid; + grid-template-columns: repeat(4, minmax(170px, 1fr)); + gap: 14px; + margin: 16px 0; +} + +.card, .panel, .customer-panel, .system-panel, .task-card, article.message-card { + background: rgba(255,255,255,.92); + border: 1px solid rgba(229,231,235,.95); + border-radius: var(--radius); + box-shadow: var(--shadow); +} + +.card { + padding: 16px; + min-height: 112px; +} + +.card span { + color: var(--muted); + display: block; + font-size: 13px; + font-weight: 600; +} + +.card strong { + display: block; + font-size: 30px; + line-height: 1; + margin: 10px 0; + letter-spacing: -0.04em; +} + +.card a { + display: inline-flex; + margin-top: 4px; + font-size: 13px; + font-weight: 700; +} + +button, .button, input[type="submit"] { + border: 1px solid #bfdbfe; + background: var(--primary); + color: white; + border-radius: 12px; + padding: 9px 13px; + font-weight: 750; + cursor: pointer; + min-height: 38px; +} + +button:hover, .button:hover, input[type="submit"]:hover { + filter: brightness(.98); +} + +input, select, textarea { + font: inherit; +} + +input[type="search"], input[type="text"], select, textarea { + max-width: 100%; +} + +table { + width: 100%; + border-collapse: separate; + border-spacing: 0; + overflow: hidden; + font-size: 14px; +} + +th { + text-align: left; + color: #4b5563; + background: #f9fafb; + font-size: 12px; + text-transform: uppercase; + letter-spacing: .04em; +} + +th, td { + padding: 10px 12px; + border-bottom: 1px solid #eef0f3; + vertical-align: top; +} + +tr:last-child td { + border-bottom: 0; +} + +.table-wrap { + width: 100%; + overflow-x: auto; + border-radius: var(--radius); + border: 1px solid var(--border); + background: white; +} + +.kanban-toolbar { + display: flex; + justify-content: space-between; + gap: 12px; + align-items: center; + flex-wrap: wrap; + background: rgba(255,255,255,.88); + border: 1px solid var(--border); + border-radius: 18px; + padding: 12px; + margin: 16px 0; + box-shadow: var(--shadow); +} + +.kanban-toolbar form { + flex: 1 1 380px; +} + +.kanban-toolbar input[type="search"] { + width: min(460px, 100%); +} + +.kanban-board, .kanban { + display: grid; + grid-template-columns: repeat(5, minmax(240px, 1fr)); + gap: 14px; + align-items: start; +} + +.kanban-column { + background: rgba(255,255,255,.66); + border: 1px solid #e5e7eb; + border-radius: 18px; + padding: 12px; + min-width: 0; +} + +.kanban-column h2, .kanban-column h3 { + margin: 0 0 10px; + font-size: 15px; + color: #111827; + display: flex; + align-items: center; + justify-content: space-between; +} + +.task-card { + padding: 13px; + margin-bottom: 12px; + transition: transform .12s ease, box-shadow .12s ease, border-color .12s ease; +} + +.task-card:hover { + transform: translateY(-1px); + border-color: #c7d2fe; + box-shadow: 0 14px 36px rgba(15, 23, 42, .10); +} + +.customer-name { + font-weight: 800; + font-size: 15px; + margin-bottom: 4px; +} + +.customer-name a { + color: #111827; +} + +.task-card small, +.message-meta, +.muted { + color: var(--muted); +} + +.task-card p { + margin: 7px 0; +} + +.task-card .request-text, +.request-text { + display: -webkit-box; + -webkit-line-clamp: 4; + -webkit-box-orient: vertical; + overflow: hidden; + color: #374151; +} + +.btn-chatwoot, +.btn-more, +.btn-external-opportunity, +.detail-button, +.task-card a[class^="btn"] { + display: inline-flex; + align-items: center; + justify-content: center; + gap: 6px; + min-height: 34px; + border-radius: 11px; + padding: 7px 10px; + border: 1px solid var(--border); + background: #f9fafb; + color: #374151; + font-size: 13px; + font-weight: 750; + margin: 4px 4px 0 0; +} + +.btn-chatwoot:hover, +.btn-more:hover, +.btn-external-opportunity:hover, +.detail-button:hover, +.task-card a[class^="btn"]:hover { + text-decoration: none; + filter: brightness(.99); +} + +.btn-chatwoot { + background: var(--primary-soft) !important; + border-color: #bfdbfe !important; + color: #1d4ed8 !important; +} + +.btn-more { + background: var(--warning-soft) !important; + border-color: #fde68a !important; + color: var(--warning) !important; +} + +.btn-external-opportunity, +.detail-button-external, +.detail-button-opportunity { + background: var(--success-soft) !important; + border-color: #86efac !important; + color: var(--success) !important; +} + +.status-badge, +.route-badge, +.sla-badge, +.system-pill { + display: inline-flex; + align-items: center; + border-radius: 999px; + padding: 4px 8px; + font-size: 12px; + font-weight: 800; + line-height: 1; +} + +.sla-overdue { + background: var(--danger-soft); + color: var(--danger); + border: 1px solid #fecaca; +} + +.sla-ok { + background: var(--success-soft); + color: var(--success); + border: 1px solid #bbf7d0; +} + +.detail-grid, .customer-grid, .admin-grid { + gap: 16px; +} + +.customer-panel, .system-panel, .panel { + padding: 16px; +} + +.customer-kv { + display: grid; + grid-template-columns: 150px 1fr; + gap: 8px 12px; +} + +.customer-kv strong { + color: var(--muted); +} + +.message-card { + padding: 12px; +} + +.message-body { + white-space: pre-wrap; + overflow-wrap: anywhere; +} + +pre, .system-pre { + max-width: 100%; + overflow-x: auto; +} + +.empty-column { + color: var(--muted-2); + text-align: center; + padding: 24px 8px; + border: 1px dashed #d1d5db; + border-radius: 14px; + background: rgba(255,255,255,.58); +} + +hr { + border: 0; + border-top: 1px solid var(--border); + margin: 18px 0; +} + +@media (max-width: 1200px) { + .cards { + grid-template-columns: repeat(2, minmax(170px, 1fr)); + } + + .kanban-board, .kanban { + grid-template-columns: repeat(2, minmax(240px, 1fr)); + } +} + +@media (max-width: 760px) { + body { + font-size: 15px; + } + + nav { + padding: 10px; + gap: 6px; + overflow-x: auto; + flex-wrap: nowrap; + } + + nav a { + white-space: nowrap; + } + + main, .container { + width: min(100% - 18px, 100%); + padding-top: 10px; + } + + .cards, + .system-grid, + .customer-grid, + .admin-grid, + .detail-grid, + .kanban-board, + .kanban { + grid-template-columns: 1fr !important; + } + + .card { + min-height: auto; + } + + .kanban-toolbar { + align-items: stretch; + } + + .kanban-toolbar form { + width: 100%; + flex-direction: column; + align-items: stretch !important; + } + + .kanban-toolbar input[type="search"], + .kanban-toolbar button, + .kanban-toolbar a { + width: 100%; + } + + .kanban-column { + padding: 10px; + } + + .task-card { + padding: 14px; + } + + .task-card a[class^="btn"], + .btn-chatwoot, + .btn-more, + .btn-external-opportunity, + .detail-button, + button { + width: 100%; + margin-right: 0; + margin-top: 8px; + } + + .customer-kv { + grid-template-columns: 1fr; + } + + table { + font-size: 13px; + } + + th, td { + padding: 9px 10px; + } + + .table-wrap, + .customer-panel, + .system-panel, + .panel { + overflow-x: auto; + } + + textarea { + min-height: 160px; + } +} + +@media (max-width: 430px) { + h1 { + font-size: 24px; + } + + .card strong { + font-size: 26px; + } + + .task-card .request-text, + .request-text { + -webkit-line-clamp: 3; + } +} + + +/* ========================= + ClientFlow UI v2.1 Kanban + ========================= */ + +.kanban-toolbar { + position: sticky; + top: 62px; + z-index: 12; +} + +.kanban-toolbar > div, +.kanban-toolbar form { + min-width: 0; +} + +.kanban-toolbar a { + display: inline-flex; + align-items: center; + justify-content: center; + min-height: 36px; + padding: 8px 11px; + border: 1px solid var(--border); + border-radius: 999px; + background: white; + color: #374151; + font-weight: 750; + font-size: 13px; +} + +.kanban-toolbar a:hover { + text-decoration: none; + background: #f3f4f6; +} + +.kanban-column { + max-height: calc(100vh - 190px); + overflow-y: auto; + scrollbar-width: thin; +} + +.kanban-column h2, +.kanban-column h3 { + position: sticky; + top: 0; + z-index: 2; + background: rgba(255,255,255,.92); + backdrop-filter: blur(8px); + padding: 8px 4px; + border-radius: 12px; +} + +.task-card { + display: flex; + flex-direction: column; + gap: 8px; +} + +.task-card .customer-name { + display: flex; + align-items: center; + justify-content: space-between; + gap: 8px; + line-height: 1.2; +} + +.task-card .customer-name a { + display: inline-block; + max-width: 100%; + overflow: hidden; + text-overflow: ellipsis; +} + +.task-card p { + margin: 0; +} + +.task-card small { + line-height: 1.35; +} + +.task-card-footer, +.task-footer, +.card-footer { + display: flex; + flex-wrap: wrap; + gap: 6px; + align-items: center; +} + +.task-card form { + margin: 0; +} + +.task-card button { + min-height: 34px; + padding: 7px 10px; + font-size: 13px; +} + +.task-card .status-badge, +.task-card .route-badge, +.task-card .sla-badge { + margin: 2px 2px 2px 0; +} + +.task-card-actions, +.task-actions { + display: flex; + flex-wrap: wrap; + gap: 6px; + margin-top: 4px; +} + +.task-card-actions a, +.task-actions a { + flex: 1 1 auto; +} + +.request-text { + background: #f9fafb; + border: 1px solid #eef0f3; + border-radius: 12px; + padding: 9px 10px; + font-size: 13px; +} + +.kanban-column:nth-child(1) { + border-top: 4px solid #60a5fa; +} + +.kanban-column:nth-child(2) { + border-top: 4px solid #f59e0b; +} + +.kanban-column:nth-child(3) { + border-top: 4px solid #34d399; +} + +.kanban-column:nth-child(4) { + border-top: 4px solid #a78bfa; +} + +.kanban-column:nth-child(5) { + border-top: 4px solid #f87171; +} + +/* Tabelas menos pesadas */ +.panel table, +.customer-panel table, +.system-panel table { + font-size: 13px; +} + +.panel th, +.customer-panel th, +.system-panel th { + white-space: nowrap; +} + +.panel td, +.customer-panel td, +.system-panel td { + color: #374151; +} + +/* Detalhe task mais respirável */ +.detail-grid { + display: grid; + grid-template-columns: minmax(0, 1.2fr) minmax(320px, .8fr); + gap: 16px; + align-items: start; +} + +.detail-panel, +.task-detail-panel { + background: white; + border: 1px solid var(--border); + border-radius: var(--radius); + box-shadow: var(--shadow); + padding: 16px; +} + +/* Mobile: Kanban vira lista simples */ +@media (max-width: 760px) { + .kanban-toolbar { + position: static; + top: auto; + } + + .kanban-column { + max-height: none; + overflow: visible; + } + + .kanban-column h2, + .kanban-column h3 { + position: static; + font-size: 17px; + padding: 10px 4px; + } + + .task-card { + gap: 10px; + } + + .request-text { + font-size: 14px; + } + + .task-card .customer-name { + font-size: 16px; + } + + .task-card-actions, + .task-actions { + flex-direction: column; + } + + .task-card-actions a, + .task-actions a, + .task-card form, + .task-card button { + width: 100%; + } + + .panel table, + .customer-panel table, + .system-panel table { + min-width: 680px; + } + + .detail-grid { + grid-template-columns: 1fr; + } +} + + +/* ========================= + ClientFlow UI v2.2 Polish + ========================= */ + +.subtitle { + max-width: 820px; + color: var(--muted); + margin-top: 0; +} + +main > p:first-of-type { + margin-top: 8px; +} + +/* Melhor separação visual entre blocos */ +.home-panel, +.customer-v2-panel, +.task-v2-panel, +.system-panel, +.customer-panel, +.panel { + position: relative; +} + +.home-panel::before, +.customer-v2-panel::before, +.task-v2-panel::before, +.system-panel::before { + content: ""; + position: absolute; + inset: 0 0 auto 0; + height: 3px; + border-radius: 18px 18px 0 0; + background: linear-gradient(90deg, rgba(37,99,235,.65), rgba(16,185,129,.55)); +} + +/* Campos/formulários mais bonitos */ +form label { + display: block; + margin: 12px 0 6px; + color: #374151; +} + +select, +textarea, +input[type="search"], +input[type="text"] { + width: 100%; + border: 1px solid #d1d5db; + border-radius: 12px; + padding: 10px 12px; + background: white; + color: var(--text); + outline: none; +} + +select:focus, +textarea:focus, +input[type="search"]:focus, +input[type="text"]:focus { + border-color: #93c5fd; + box-shadow: 0 0 0 4px rgba(147,197,253,.25); +} + +/* Botões secundários */ +.secondary-button, +button.secondary-button { + background: #f9fafb !important; + color: #374151 !important; + border-color: #d1d5db !important; +} + +.danger-button, +button.danger-button { + background: #fee2e2 !important; + color: #991b1b !important; + border-color: #fecaca !important; +} + +/* Cards com menos texto visível */ +.home-task-card p, +.customer-task-card p, +.mini-history-item p, +.customer-message-card p { + display: -webkit-box; + -webkit-line-clamp: 5; + -webkit-box-orient: vertical; + overflow: hidden; +} + +/* Debug sempre discreto */ +details { + transition: all .15s ease; +} + +details summary { + list-style: none; +} + +details summary::-webkit-details-marker { + display: none; +} + +details summary::before { + content: "▸"; + display: inline-block; + margin-right: 8px; + transition: transform .15s ease; +} + +details[open] summary::before { + transform: rotate(90deg); +} + +/* Tabelas técnicas mais controladas */ +table code { + word-break: break-word; +} + +td { + max-width: 420px; + overflow-wrap: anywhere; +} + +/* Navegação mais limpa */ +nav::before { + content: "ClientFlow"; + font-weight: 900; + color: #111827; + margin-right: 6px; + letter-spacing: -0.03em; +} + +nav a[href="/"], +nav a[href="/tasks"], +nav a[href="/system"] { + background: #f9fafb; + border: 1px solid #e5e7eb; +} + +/* Página sistema menos agressiva */ +.system-card strong .system-pill { + font-size: 13px; + padding: 7px 10px; +} + +.system-pre { + max-height: 320px; +} + +/* Melhor impressão visual no mobile */ +@media (max-width: 760px) { + nav::before { + display: none; + } + + h1 { + margin-top: 8px; + } + + .subtitle { + font-size: 14px; + } + + .home-kpi, + .customer-summary-card, + .card, + .system-card { + padding: 14px; + } + + .home-kpi strong, + .customer-summary-card strong, + .card strong { + font-size: 28px; + } + + .home-panel, + .customer-v2-panel, + .task-v2-panel, + .system-panel, + .customer-panel, + .panel { + padding: 14px; + border-radius: 16px; + } + + /* Tabelas ficam legíveis sem rebentar a página */ + table { + min-width: 640px; + } + + .system-panel, + .customer-v2-panel, + .task-v2-panel, + .home-panel, + .panel { + overflow-x: auto; + } + + /* Ações sempre grandes no telemóvel */ + .home-task-actions, + .customer-card-actions, + .task-actions-row { + flex-direction: column; + } + + .home-task-actions a, + .customer-card-actions a, + .task-actions-row form, + .task-actions-row button, + .detail-button { + width: 100%; + } + + textarea.suggested-reply-box, + #suggested-reply { + min-height: 260px; + } +} + +/* Ecrãs muito pequenos */ +@media (max-width: 420px) { + main, .container { + width: calc(100% - 12px); + } + + .home-task-card, + .customer-task-card, + .customer-message-card, + .mini-history-item, + .task-card { + border-radius: 14px; + } + + .home-kpi strong, + .customer-summary-card strong, + .card strong { + font-size: 24px; + } +} + + +/* ============================== + ClientFlow UI v2.3 Kanban Clean + ============================== */ + +/* Kanban menos pesado visualmente */ +.kanban-board, +.kanban { + gap: 12px; +} + +.kanban-column { + background: rgba(255,255,255,.72); + padding: 10px; +} + +.kanban-column h2, +.kanban-column h3 { + font-size: 14px; + margin-bottom: 8px; +} + +/* Card mais compacto e legível */ +.task-card { + padding: 12px; + gap: 7px; + border-radius: 16px; +} + +.task-card .customer-name { + font-size: 15px; + margin: 0; +} + +.task-card .customer-name a { + color: #111827; + font-weight: 900; +} + +/* Reduz excesso de texto no card */ +.task-card p { + font-size: 13px; + color: #4b5563; +} + +.task-card small { + font-size: 12px; +} + +/* Pedido resumido no Kanban */ +.task-card .request-text, +.request-text { + -webkit-line-clamp: 3; + font-size: 13px; + line-height: 1.35; + padding: 8px 9px; + margin-top: 2px; +} + +/* Menos destaque para informação técnica dentro dos cards */ +.task-card code { + font-size: 11px; + color: #6b7280; + background: #f9fafb; + border-color: #eef0f3; +} + +/* Badges mais pequenos no Kanban */ +.task-card .status-badge, +.task-card .route-badge, +.task-card .sla-badge { + font-size: 11px; + padding: 4px 7px; +} + +/* Ações do card em grelha limpa */ +.task-card .btn-chatwoot, +.task-card .btn-more, +.task-card .btn-external-opportunity, +.task-card a[class^="btn"] { + min-height: 32px; + padding: 7px 9px; + font-size: 12px; + flex: 1 1 42%; + margin: 0; +} + +/* Evita cartões gigantes quando há muitos botões */ +.task-card a[class^="btn"]:only-child { + flex-basis: 100%; +} + +/* Botão Feito/Ignorar mais discreto dentro do Kanban */ +.task-card form button { + min-height: 32px; + padding: 7px 9px; + font-size: 12px; +} + +/* Toolbar menos alta */ +.kanban-toolbar { + padding: 10px; + margin: 12px 0; +} + +.kanban-toolbar input[type="search"] { + min-height: 38px; +} + +.kanban-toolbar button, +.kanban-toolbar a { + min-height: 36px; +} + +/* Destacar só o que precisa de atenção */ +.task-card:has(.sla-overdue) { + border-color: #fecaca; + background: #fffafa; +} + +.task-card:has(.sla-overdue)::before { + content: "Atenção"; + display: inline-flex; + align-self: flex-start; + background: #fee2e2; + color: #991b1b; + border: 1px solid #fecaca; + border-radius: 999px; + padding: 3px 7px; + font-size: 11px; + font-weight: 900; +} + +/* No desktop, colunas mais suaves */ +@media (min-width: 1201px) { + .kanban-board, + .kanban { + grid-template-columns: repeat(5, minmax(220px, 1fr)); + } +} + +/* Em tablet, duas colunas continuam boas */ +@media (max-width: 1200px) and (min-width: 761px) { + .kanban-board, + .kanban { + grid-template-columns: repeat(2, minmax(260px, 1fr)); + } +} + +/* Mobile: lista operacional */ +@media (max-width: 760px) { + .kanban-toolbar { + gap: 10px; + } + + .kanban-toolbar > div { + width: 100%; + } + + .kanban-toolbar div a { + width: 100%; + margin-top: 6px; + } + + .kanban-column { + border-radius: 16px; + padding: 10px; + margin-bottom: 12px; + } + + .kanban-column h2, + .kanban-column h3 { + font-size: 18px; + padding: 8px 2px; + } + + .task-card { + padding: 14px; + border-radius: 16px; + } + + .task-card .customer-name { + font-size: 17px; + } + + .task-card .request-text, + .request-text { + -webkit-line-clamp: 4; + font-size: 14px; + } + + .task-card .btn-chatwoot, + .task-card .btn-more, + .task-card .btn-external-opportunity, + .task-card a[class^="btn"], + .task-card form button { + width: 100%; + flex-basis: 100%; + font-size: 14px; + min-height: 42px; + } +} + +/* Reduz ruído visual nas páginas em geral */ +.status-badge, +.route-badge, +.sla-badge { + white-space: nowrap; +} + +.home-mini-item, +.mini-history-item, +.customer-message-card, +.customer-task-card { + box-shadow: none; +} + +/* Melhor leitura das mensagens longas */ +.customer-message-card p, +.message-body, +.request-box { + line-height: 1.5; +} + +/* Evita páginas com aparência demasiado técnica */ +details.debug-details, +details.customer-debug, +details.home-debug { + opacity: .96; +} + + +/* CLIENTFLOW_TASK_DETAIL_NO_SIDEBAR */ +.task-v2-layout { + display: grid !important; + grid-template-columns: minmax(0, 1fr) !important; + gap: 18px !important; +} + +.task-v2-layout > main, +.task-v2-layout > aside { + width: 100% !important; + max-width: 100% !important; + min-width: 0 !important; +} + +.task-v2-layout > main { + order: 1; +} + +.task-v2-layout > aside { + order: 2; + position: static !important; + top: auto !important; + align-self: stretch !important; +} + +.task-v2-panel { + max-width: 100% !important; + overflow-wrap: anywhere; +} + +.task-v2-panel .cards, +.task-v2-panel .card-grid { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(220px, 1fr)) !important; + gap: 12px; +} + +.task-v2-panel .card { + min-width: 0 !important; +} + +.task-actions { + display: flex !important; + flex-wrap: wrap !important; + gap: 10px !important; +} + +.task-actions form, +.task-actions button { + min-width: 0 !important; +} + +@media (max-width: 1300px) { + .task-v2-layout { + grid-template-columns: 1fr !important; + } + + .task-actions { + flex-direction: column !important; + } + + .task-actions form, + .task-actions button { + width: 100% !important; + } +} + +@media (max-width: 760px) { + .task-v2-panel { + padding: 14px !important; + border-radius: 16px !important; + } + + .task-v2-panel .cards, + .task-v2-panel .card-grid { + grid-template-columns: 1fr !important; + } +} + + +/* CLIENTFLOW_TASK_DETAIL_WIDTH_FIX_V2 */ +html, +body { + max-width: 100%; + overflow-x: hidden; +} + +main { + max-width: 1180px !important; + width: min(1180px, calc(100vw - 28px)) !important; + margin-left: auto !important; + margin-right: auto !important; + box-sizing: border-box !important; +} + +.task-v2-layout, +.task-detail-layout, +.task-detail-grid, +.detail-grid { + display: grid !important; + grid-template-columns: 1fr !important; + width: 100% !important; + max-width: 100% !important; + min-width: 0 !important; + gap: 16px !important; +} + +.task-v2-layout > *, +.task-detail-layout > *, +.task-detail-grid > *, +.detail-grid > *, +.task-v2-layout main, +.task-v2-layout aside, +.task-detail-layout main, +.task-detail-layout aside, +.task-detail-grid main, +.task-detail-grid aside, +.detail-grid main, +.detail-grid aside { + width: 100% !important; + max-width: 100% !important; + min-width: 0 !important; + box-sizing: border-box !important; +} + +aside { + position: static !important; + width: 100% !important; + max-width: 100% !important; + min-width: 0 !important; +} + +.task-v2-panel, +.panel, +.card, +.cards, +.task-actions, +form, +textarea, +select, +input, +button { + max-width: 100% !important; + min-width: 0 !important; + box-sizing: border-box !important; +} + +.task-v2-panel, +.panel, +.card { + overflow-wrap: anywhere !important; + word-break: normal !important; +} + +.cards { + display: grid !important; + grid-template-columns: repeat(auto-fit, minmax(180px, 1fr)) !important; + gap: 10px !important; +} + +.task-actions { + display: grid !important; + grid-template-columns: 1fr !important; + gap: 10px !important; +} + +.task-actions form, +.task-actions button, +.task-actions .btn, +.task-v2-panel button, +.task-v2-panel .btn, +.task-v2-panel .primary, +.task-v2-panel .secondary, +.task-v2-panel .secondary-button, +.task-v2-panel .danger-button { + width: 100% !important; + white-space: normal !important; +} + +pre, +code, +table { + max-width: 100% !important; + overflow-x: auto !important; + white-space: pre-wrap !important; +} + +@media (max-width: 1400px) { + main { + width: min(1040px, calc(100vw - 24px)) !important; + } + + .task-v2-layout, + .task-detail-layout, + .task-detail-grid, + .detail-grid { + grid-template-columns: 1fr !important; + } +} + +@media (max-width: 900px) { + main { + width: calc(100vw - 18px) !important; + padding-left: 0 !important; + padding-right: 0 !important; + } + + .topbar { + align-items: flex-start !important; + flex-direction: column !important; + } + + .nav-links { + width: 100% !important; + justify-content: flex-start !important; + } + + .task-v2-panel, + .panel { + padding: 12px !important; + border-radius: 14px !important; + } + + .cards { + grid-template-columns: 1fr !important; + } +} + + +/* CLIENTFLOW_PREP_READABLE_V1 */ +.prep-readable { + display: grid; + gap: 12px; +} + +.prep-readable .prep-block { + background: #f8fafc; + border: 1px solid #e2e8f0; + border-radius: 14px; + padding: 12px; +} + +.prep-readable .prep-block h3 { + margin: 0 0 10px; + font-size: 0.95rem; + color: #111827; +} + +.prep-readable dl { + display: grid; + grid-template-columns: 130px minmax(0, 1fr); + gap: 6px 12px; + margin: 0; +} + +.prep-readable dt { + font-weight: 800; + color: #64748b; +} + +.prep-readable dd { + margin: 0; + color: #111827; + overflow-wrap: anywhere; +} + +.prep-readable .prep-missing { + background: #fff7ed; + border-color: #fed7aa; +} + +.prep-readable .prep-reply { + background: #eff6ff; + border-color: #bfdbfe; +} + +@media (max-width: 900px) { + .prep-readable dl { + grid-template-columns: 1fr; + } + + .prep-readable dt { + margin-top: 6px; + } +} + + +/* CLIENTFLOW_PREP_COMPONENTS_V1 */ +.prep-components-shell { + display: grid; + gap: 14px; +} + +.prep-components-header { + background: #111827; + color: white; + border-radius: 18px; + padding: 16px; + display: flex; + align-items: center; + justify-content: space-between; + gap: 12px; +} + +.prep-components-header span { + display: block; + color: #bfdbfe; + font-size: .76rem; + font-weight: 900; + text-transform: uppercase; + letter-spacing: .06em; +} + +.prep-components-header h2 { + margin: 3px 0 0; + font-size: 1.25rem; +} + +.prep-components-confidence { + background: rgba(255,255,255,.12); + border: 1px solid rgba(255,255,255,.22); + border-radius: 14px; + padding: 8px 12px; + min-width: 82px; + text-align: center; +} + +.prep-components-confidence strong { + display: block; + font-size: 1.05rem; +} + +.prep-components-confidence small { + color: #dbeafe; + font-weight: 800; +} + +.prep-component-card { + background: white; + border: 1px solid #e5e7eb; + border-radius: 18px; + padding: 15px; + box-shadow: 0 8px 20px rgba(15, 23, 42, .045); +} + +.prep-component-title { + display: flex; + align-items: center; + gap: 9px; + margin-bottom: 14px; +} + +.prep-component-title span { + width: 32px; + height: 32px; + border-radius: 12px; + display: inline-flex; + align-items: center; + justify-content: center; + background: #f8fafc; + border: 1px solid #e2e8f0; +} + +.prep-component-title h3 { + margin: 0; + color: #0f172a; + font-size: 1.02rem; +} + +.prep-component-fields { + display: grid; + gap: 10px; +} + +.prep-component-fields > div { + background: #f8fafc; + border: 1px solid #e2e8f0; + border-radius: 14px; + padding: 11px 12px; +} + +.prep-component-fields label { + display: block; + color: #64748b; + font-size: .74rem; + font-weight: 900; + text-transform: uppercase; + letter-spacing: .04em; + margin-bottom: 5px; +} + +.prep-component-fields p { + margin: 0; + color: #111827; + line-height: 1.42; + overflow-wrap: anywhere; + font-weight: 650; +} + +.prep-component-warning { + background: #fffbeb; + border-color: #fde68a; +} + +.prep-component-reply { + background: #eff6ff; + border-color: #bfdbfe; +} + +.prep-component-reply > p { + margin: 0; + color: #1e3a8a; + line-height: 1.5; + font-weight: 650; +} + +.prep-component-chips { + display: flex; + flex-wrap: wrap; + gap: 8px; +} + +.prep-component-chip { + display: inline-flex; + align-items: center; + border-radius: 999px; + padding: 7px 10px; + font-size: .82rem; + font-weight: 850; +} + +.prep-component-chip.missing { + background: #fff7ed; + color: #9a3412; + border: 1px solid #fed7aa; +} + +.prep-component-chip.ok { + background: #ecfdf5; + color: #047857; + border: 1px solid #bbf7d0; +} + +.prep-component-meta { + display: flex; + flex-wrap: wrap; + gap: 8px; + color: #64748b; + font-size: .75rem; +} + +.prep-component-meta span { + background: #f1f5f9; + border-radius: 999px; + padding: 5px 8px; +} + +@media (max-width: 760px) { + .prep-components-header { + align-items: flex-start; + flex-direction: column; + } + + .prep-components-confidence { + width: 100%; + } +} + + +/* CLIENTFLOW_PREP_COMPONENTS_POLISH_V2 */ +.prep-components-shell { + background: #f8fafc; + border: 1px solid #e2e8f0; + border-radius: 22px; + padding: 14px; +} + +.prep-components-header { + box-shadow: 0 10px 28px rgba(15, 23, 42, .14); +} + +.prep-component-card { + border: 1px solid #e2e8f0; + box-shadow: 0 8px 22px rgba(15, 23, 42, .055); + transition: transform .12s ease, box-shadow .12s ease; +} + +.prep-component-card:hover { + transform: translateY(-1px); + box-shadow: 0 12px 30px rgba(15, 23, 42, .075); +} + +.prep-component-title { + padding-bottom: 10px; + border-bottom: 1px solid #eef2f7; +} + +.prep-component-title h3 { + font-size: 1.05rem; + font-weight: 900; +} + +.prep-component-fields { + gap: 8px; +} + +.prep-component-fields > div { + background: #ffffff; + border: 1px solid #e8eef6; + border-radius: 12px; + padding: 10px 11px; +} + +.prep-component-fields label { + color: #64748b; + font-size: .7rem; +} + +.prep-component-fields p { + font-size: .96rem; + color: #0f172a; +} + +.prep-component-fields p:empty::after { + content: "—"; + color: #94a3b8; +} + +.prep-component-warning { + background: linear-gradient(180deg, #fffbeb, #fff7ed); +} + +.prep-component-warning .prep-component-title span { + background: #fff7ed; + border-color: #fed7aa; +} + +.prep-component-reply { + background: linear-gradient(180deg, #eff6ff, #f8fafc); +} + +.prep-component-reply .prep-component-title span { + background: #dbeafe; + border-color: #bfdbfe; +} + +.prep-component-reply > p { + font-size: .98rem; + padding: 12px; + background: white; + border: 1px solid #bfdbfe; + border-radius: 14px; +} + +.prep-component-chip { + box-shadow: 0 2px 8px rgba(15, 23, 42, .045); +} + +.prep-component-meta { + opacity: .68; + font-size: .7rem; + padding: 2px 4px; +} + +.prep-component-meta:hover { + opacity: 1; +} + +/* Em ecrãs médios, manter tudo confortável em 1 coluna */ +@media (max-width: 1200px) { + .prep-components-shell { + padding: 12px; + } + + .prep-component-card { + padding: 14px; + } + + .prep-component-fields { + grid-template-columns: 1fr !important; + } +} + +/* Em mobile/laptop pequeno, reduzir ruído visual */ +@media (max-width: 760px) { + .prep-components-header { + border-radius: 16px; + padding: 14px; + } + + .prep-component-card { + border-radius: 16px; + padding: 13px; + } + + .prep-component-title { + margin-bottom: 10px; + } + + .prep-component-fields > div { + padding: 9px 10px; + } + + .prep-component-meta { + display: none; + } +} + + +/* CLIENTFLOW_PREP_PRIORITY_ORDER_V1 */ +.prep-priority-zone { + display: grid; + gap: 12px; + padding: 12px; + background: linear-gradient(180deg, #f8fafc, #eef2ff); + border: 1px solid #dbeafe; + border-radius: 20px; +} + +.prep-priority-zone .prep-component-card { + box-shadow: 0 10px 26px rgba(15, 23, 42, .07); +} + +.prep-priority-zone .prep-component-warning { + border-left: 5px solid #f59e0b; +} + +.prep-priority-zone .prep-component-reply { + border-left: 5px solid #2563eb; +} + +.prep-component-card:not(.prep-component-warning):not(.prep-component-reply) { + border-left: 4px solid #e2e8f0; +} + +.prep-component-card:nth-of-type(2) { + margin-top: 2px; +} + +@media (max-width: 760px) { + .prep-priority-zone { + padding: 10px; + border-radius: 16px; + } +} + + +/* CLIENTFLOW_TEXTAREA_COMPONENT_FIX_V1 */ +.task-v2-panel textarea, +.panel textarea, +.prep-components-shell textarea, +.prep-component-card textarea, +.prep-component-reply textarea, +.prep-component-card input, +.prep-component-card select { + display: block !important; + width: 100% !important; + max-width: 100% !important; + min-width: 0 !important; + box-sizing: border-box !important; + resize: vertical !important; + overflow: auto !important; + border: 1px solid #dbe3ef; + border-radius: 14px; + padding: 11px 12px; + font: inherit; +} + +.task-v2-panel form, +.panel form, +.prep-component-card form, +.prep-component-reply form { + width: 100% !important; + max-width: 100% !important; + min-width: 0 !important; + box-sizing: border-box !important; +} + +.prep-component-card, +.prep-component-reply, +.prep-component-fields > div { + overflow: hidden !important; +} + +.prep-component-reply p, +.prep-component-reply textarea, +.prep-component-fields p { + overflow-wrap: anywhere !important; + word-break: normal !important; +} + +@media (max-width: 900px) { + .task-v2-panel textarea, + .panel textarea, + .prep-components-shell textarea, + .prep-component-card textarea { + min-height: 90px; + } +} + + +/* CLIENTFLOW_TASK_DETAIL_SINGLE_COLUMN_CSS_ONLY_V2 */ +.task-v2-layout { + display: grid !important; + grid-template-columns: minmax(0, 1fr) !important; + gap: 18px !important; + max-width: 1120px !important; + width: 100% !important; + margin-left: auto !important; + margin-right: auto !important; +} + +.task-v2-layout > main, +.task-v2-layout > aside { + grid-column: 1 / -1 !important; + width: 100% !important; + max-width: 100% !important; + min-width: 0 !important; + box-sizing: border-box !important; +} + +.task-v2-layout > aside { + position: static !important; +} + +.task-v2-panel, +.panel, +.prep-components-shell, +.prep-component-card, +.prep-component-fields, +.prep-component-fields > div, +.task-actions, +form, +textarea, +input, +select, +button { + max-width: 100% !important; + min-width: 0 !important; + box-sizing: border-box !important; +} + +.task-v2-panel, +.panel, +.prep-component-card { + overflow-wrap: anywhere !important; +} + +textarea { + width: 100% !important; + resize: vertical !important; +} + +.task-actions { + display: grid !important; + grid-template-columns: 1fr !important; + gap: 10px !important; +} + +.task-actions form, +.task-actions button { + width: 100% !important; +} + +@media (max-width: 1200px) { + .task-v2-layout { + max-width: 100% !important; + } +} + + +/* CLIENTFLOW_RECLASSIFY_DROPDOWN_FIX_V1 */ +.reclassify-box, +.reclassify-form { + display: grid; + gap: 10px; + width: 100%; + max-width: 100%; + overflow: visible !important; + position: relative; + z-index: 5; +} + +.reclassify-form select, +.reclassify-form textarea, +.reclassify-form button { + width: 100% !important; + max-width: 100% !important; + box-sizing: border-box !important; +} + +.reclassify-form select { + display: block !important; + appearance: auto !important; + -webkit-appearance: menulist !important; + pointer-events: auto !important; + position: relative; + z-index: 10; + min-height: 42px; + background: white; + cursor: pointer; +} + +.task-v2-panel:has(.reclassify-box) { + overflow: visible !important; +} + + +/* CLIENTFLOW_ASSISTANTS_SHORT_PREP_RESULT_V1 */ +.task-v2-panel:has(> h2:first-child) { + max-width: 100%; +} + +.task-v2-panel h2 + .muted { + margin-top: -4px; + margin-bottom: 12px; +} + +.task-v2-panel .task-actions { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(170px, 1fr)); + gap: 10px; +} + +.task-v2-panel .task-actions form { + margin: 0; +} + +.task-v2-panel .task-actions button, +.task-v2-panel .btn { + width: 100%; + white-space: normal; +} + +.task-preparation-result-main { + background: #f8fafc; + border-color: #dbeafe; +} + +.task-preparation-result-main > .prep-components-shell { + margin-top: 10px; +} + +@media (max-width: 900px) { + .task-v2-panel .task-actions { + grid-template-columns: 1fr; + } +} +""" + +ADMIN_UI_V451_CSS = r""" +:root{ + --cf-bg:#f5f7fb; + --cf-panel:#ffffff; + --cf-panel-2:#f9fbff; + --cf-text:#142033; + --cf-muted:#6b778d; + --cf-line:#e7edf6; + --cf-primary:#2563eb; + --cf-primary-2:#dbeafe; + --cf-teal:#0f766e; + --cf-green:#16a34a; + --cf-green-2:#dcfce7; + --cf-orange:#f97316; + --cf-orange-2:#ffedd5; + --cf-red:#dc2626; + --cf-red-2:#fee2e2; + --cf-purple:#7c3aed; + --cf-purple-2:#ede9fe; + --cf-shadow:0 10px 26px rgba(20,32,51,.06); + --cf-shadow-strong:0 18px 45px rgba(20,32,51,.10); + --cf-radius:16px; + --cf-radius-sm:11px; + --cf-sidebar-width:284px; +} +*{box-sizing:border-box} +html{-webkit-text-size-adjust:100%;scroll-behavior:smooth} +body{ + margin:0; + background: + radial-gradient(circle at top left,rgba(37,99,235,.08),transparent 32rem), + var(--cf-bg); + color:var(--cf-text); + font-family:Inter,ui-sans-serif,system-ui,-apple-system,BlinkMacSystemFont,"Segoe UI",sans-serif; + font-size:.925rem; + line-height:1.45; +} +a{color:inherit;text-decoration:none} +a:hover{text-decoration:none;color:var(--cf-primary)} +code{color:#334155;background:#f1f5f9;border:1px solid #e2e8f0;border-radius:.45rem;padding:.1rem .35rem;font-size:.82em} +pre{background:#0f172a;color:#e2e8f0;padding:1rem;border-radius:1rem;overflow-x:auto;white-space:pre-wrap} + +.cf-app{min-height:100vh;display:grid;grid-template-columns:var(--cf-sidebar-width) minmax(0,1fr)} +.cf-sidebar{ + position:sticky;top:0;height:100vh;max-height:100vh;background:linear-gradient(180deg,#fff 0,#f8fbff 100%); + border-right:1px solid var(--cf-line);padding:24px 18px;display:flex;flex-direction:column;gap:18px;z-index:30; + overflow-y:auto;overflow-x:hidden;overscroll-behavior:contain;scrollbar-gutter:stable; +} +.cf-brand{display:flex;gap:12px;align-items:center;padding:0 8px 10px;color:var(--cf-text)} +.cf-brand:hover{color:var(--cf-text)} +.cf-brand-mark{width:36px;height:36px;border-radius:12px;background:linear-gradient(135deg,#2563eb,#12b8a6);display:grid;place-items:center;color:#fff;font-weight:900;box-shadow:0 10px 20px rgba(37,99,235,.25);font-size:.78rem;letter-spacing:-.04em} +.cf-brand-text strong{display:block;font-size:19px;letter-spacing:-.03em;line-height:1.05}.cf-brand-text small{display:block;color:var(--cf-muted);font-size:11px;margin-top:2px;font-weight:700} +.cf-nav-title{font-size:11px;text-transform:uppercase;letter-spacing:.08em;color:#94a3b8;margin:12px 10px 4px;font-weight:900} +.cf-nav{display:flex;flex-direction:column;gap:5px;backdrop-filter:none;flex:0 0 auto} +.cf-nav-link{display:flex;align-items:center;gap:11px;padding:10px 12px;border-radius:12px;color:#475569;font-weight:750;min-height:42px;transition:background .14s ease,color .14s ease,transform .14s ease} +.cf-nav-link:hover{background:#f1f5ff;color:var(--cf-primary);transform:translateX(1px)} +.cf-nav-link.active{background:var(--cf-primary-2);color:#1d4ed8} +.cf-nav-icon{width:22px;height:22px;border-radius:8px;display:grid;place-items:center;background:#eef2ff;color:var(--cf-primary);font-size:13px;line-height:1;flex:0 0 auto} +.cf-nav-link.active .cf-nav-icon{background:#fff;box-shadow:0 4px 10px rgba(37,99,235,.08)} +.cf-sidebar-footer{margin-top:auto;background:#f8fafc;border:1px solid var(--cf-line);border-radius:16px;padding:12px;display:grid;gap:10px;flex:0 0 auto}.cf-company{font-size:12px;color:var(--cf-muted);font-weight:700}.cf-company strong{display:block;color:#334155;font-size:13px;margin-top:4px}.cf-user-mini{display:flex;gap:10px;align-items:center}.cf-avatar{width:34px;height:34px;border-radius:50%;display:grid;place-items:center;background:linear-gradient(135deg,#2563eb,#14b8a6);color:white;font-weight:900;font-size:.8rem}.cf-user-mini strong{display:block;font-size:.85rem;line-height:1.05}.cf-user-mini span{display:block;font-size:.72rem;color:var(--cf-muted);font-weight:700} + +.cf-content{min-width:0}.cf-topbar{height:70px;background:rgba(255,255,255,.80);backdrop-filter:blur(12px);border-bottom:1px solid var(--cf-line);display:flex;align-items:center;justify-content:space-between;padding:0 30px;position:sticky;top:0;z-index:20;gap:16px}.cf-topbar-left{display:flex;align-items:center;gap:14px;min-width:0;flex:1}.cf-menu-dot{border:0;background:transparent;color:#64748b;font-size:1.1rem;padding:7px;border-radius:10px}.cf-global-search{display:flex;align-items:center;gap:10px;background:#f8fafc;border:1px solid var(--cf-line);border-radius:999px;padding:9px 14px;width:min(520px,45vw);color:var(--cf-muted)}.cf-global-search input{border:0;background:transparent;outline:0;width:100%;color:var(--cf-text);min-width:0}.cf-topbar-actions{display:flex;align-items:center;gap:10px}.cf-pill{display:inline-flex;align-items:center;gap:7px;border:1px solid var(--cf-line);background:#fff;border-radius:999px;padding:8px 11px;color:#475569;font-weight:750;white-space:nowrap}.cf-pill:hover{border-color:#bfd1ff;background:#fff;color:var(--cf-primary)}.cf-icon-btn{width:36px;height:36px;border:1px solid var(--cf-line);background:#fff;color:#475569;display:grid;place-items:center;border-radius:50%;font-weight:800;padding:0}.cf-icon-btn:hover{border-color:#bfd1ff;color:var(--cf-primary)} +.cf-page{width:min(1440px,calc(100% - 60px));margin:0 auto;padding:28px 0 54px}.cf-page-title{display:flex;justify-content:space-between;gap:20px;align-items:flex-start;margin-bottom:20px}.cf-page-title h1{margin:0;font-size:clamp(1.75rem,2.7vw,2.35rem);letter-spacing:-.04em;font-weight:900}.cf-page-title p{margin:5px 0 0;color:var(--cf-muted);font-size:.98rem}.cf-page-title::after{content:"v4.7 Operations";display:inline-flex;align-items:center;border:1px solid var(--cf-line);background:#fff;border-radius:999px;padding:7px 11px;color:#64748b;font-weight:800;font-size:.75rem;white-space:nowrap} + +.cf-card,.card.cf-card,.card,.panel,.customer-panel,.system-panel,.task-card,article.message-card{background:var(--cf-panel);border:1px solid var(--cf-line);border-radius:var(--cf-radius);box-shadow:var(--cf-shadow)} +.card{overflow:hidden}.card-body{border-radius:inherit}.cf-soft-box{background:#f8fafc;border:1px solid var(--cf-line);border-radius:13px;padding:1rem}.cf-empty{padding:2rem;text-align:center;color:#94a3b8;border:1px dashed #cbd5e1;border-radius:1rem;background:#fff}.cf-muted,.muted{color:var(--cf-muted)}.small,.text-secondary{color:var(--cf-muted)!important}.fw-bold{font-weight:850!important}.border-bottom{border-bottom-color:var(--cf-line)!important}.border-top{border-top-color:var(--cf-line)!important}.bg-light{background:#f8fafc!important} +.cf-kpi-grid{display:grid;grid-template-columns:repeat(4,minmax(170px,1fr));gap:16px;margin-bottom:18px}.cf-kpi{padding:18px;min-height:132px;position:relative;overflow:hidden;transition:transform .14s ease,box-shadow .14s ease,border-color .14s ease}.cf-kpi:hover{transform:translateY(-2px);box-shadow:var(--cf-shadow-strong);border-color:#cfe0ff;color:inherit}.cf-kpi::after{display:none!important;content:none!important}.cf-kpi-icon{position:absolute;right:16px;top:16px;width:44px;height:44px;border-radius:14px;background:var(--cf-primary-2);color:var(--cf-primary);display:grid;place-items:center;opacity:.98;font-size:1.12rem;line-height:1}.cf-kpi-icon i{line-height:1}.cf-kpi-tone-green .cf-kpi-icon{background:var(--cf-green-2);color:var(--cf-green)}.cf-kpi-tone-orange .cf-kpi-icon{background:var(--cf-orange-2);color:var(--cf-orange)}.cf-kpi-tone-red .cf-kpi-icon{background:var(--cf-red-2);color:var(--cf-red)}.cf-kpi-tone-purple .cf-kpi-icon{background:var(--cf-purple-2);color:var(--cf-purple)}.cf-kpi span:not(.cf-kpi-icon){display:block;color:#4b5563;font-weight:850;font-size:.82rem;max-width:calc(100% - 58px)}.cf-kpi strong{display:block;font-size:1.85rem;line-height:1;letter-spacing:-.05em;margin:.75rem 0 .3rem;color:#0f172a}.cf-kpi small{color:var(--cf-green);font-weight:850}.cf-kpi-tone-orange small{color:var(--cf-orange)}.cf-kpi-tone-red small{color:var(--cf-red)}.cf-kpi-tone-purple small{color:var(--cf-purple)}.cf-section-title{font-size:1.05rem;font-weight:900;margin:0;letter-spacing:-.015em}.cf-row-link{color:var(--cf-primary);font-weight:900} + +.alert{border-radius:var(--cf-radius)!important;border:1px solid #bfdbfe!important;box-shadow:var(--cf-shadow);background:#eff6ff!important;color:#1e40af!important}.alert-danger{background:#fff7f7!important;color:#991b1b!important;border-color:#fecaca!important}.alert-warning{background:#fff7ed!important;color:#9a3412!important;border-color:#fed7aa!important}.alert-info{background:#eef9ff!important;color:#075985!important;border-color:#bae6fd!important}.alert-success{background:#f0fdf4!important;color:#166534!important;border-color:#bbf7d0!important} +.btn,button,.button,input[type="submit"]{border-radius:11px!important;font-weight:800!important;box-shadow:0 2px 6px rgba(15,23,42,.03);transition:box-shadow .14s ease,border-color .14s ease,background .14s ease,transform .14s ease}.btn:hover,button:hover,.button:hover,input[type="submit"]:hover{box-shadow:0 8px 18px rgba(37,99,235,.08);transform:translateY(-1px)}.btn-primary,.btn.btn-primary{background:var(--cf-primary)!important;border-color:var(--cf-primary)!important;color:#fff!important}.btn-outline-primary{border-color:#bfd1ff!important;color:#1d4ed8!important;background:#fff!important}.btn-outline-secondary{border-color:var(--cf-line)!important;color:#475569!important;background:#fff!important}.btn-success{background:var(--cf-green)!important;border-color:var(--cf-green)!important}.btn-warning{background:var(--cf-orange-2)!important;border-color:#fed7aa!important;color:#9a3412!important}.btn-danger,.btn-outline-danger{background:var(--cf-red-2)!important;border-color:#fecaca!important;color:#991b1b!important}.btn-sm{border-radius:9px!important;padding:.38rem .6rem!important;font-size:.8rem!important} +.form-control,.form-select,textarea,input[type="search"],input[type="text"],select{border:1px solid var(--cf-line)!important;border-radius:11px!important;padding:.62rem .72rem;background:#fff;color:var(--cf-text);outline:0}.form-control:focus,.form-select:focus,textarea:focus,input:focus,select:focus{border-color:#93c5fd!important;box-shadow:0 0 0 4px #dbeafe!important}.filters,.cf-filter-card{background:#fff;border:1px solid var(--cf-line);border-radius:var(--cf-radius);padding:1rem;margin:1rem 0;box-shadow:var(--cf-shadow)} + +.cf-table-wrap{overflow-x:auto;border-radius:var(--cf-radius);border:0;background:transparent}.cf-table{width:100%;border-collapse:separate!important;border-spacing:0 9px!important;margin:0}.cf-table thead th{font-size:11px;text-transform:uppercase;letter-spacing:.06em;text-align:left;color:#94a3b8!important;font-weight:900;padding:0 12px 2px!important;border:0!important;background:transparent!important}.cf-table tbody td{background:#fff;border-top:1px solid var(--cf-line);border-bottom:1px solid var(--cf-line);padding:12px!important;vertical-align:middle!important}.cf-table tbody tr td:first-child{border-left:1px solid var(--cf-line);border-radius:12px 0 0 12px}.cf-table tbody tr td:last-child{border-right:1px solid var(--cf-line);border-radius:0 12px 12px 0}.cf-table tbody tr:hover td{border-color:#cfe0ff;background:#fcfdff} +table{width:100%}th,td{vertical-align:top}.table{--bs-table-bg:transparent;--bs-table-border-color:transparent}.table>:not(caption)>*>*{background:transparent} + +.cf-chip,.status-badge,.route-badge,.sla-badge{display:inline-flex;align-items:center;gap:.35rem;border-radius:999px;padding:.28rem .55rem;font-size:.74rem;font-weight:850;border:1px solid transparent;white-space:nowrap;line-height:1.1}.cf-chip-blue,.route-vendas{background:var(--cf-primary-2);color:#1d4ed8;border-color:#bfdbfe}.cf-chip-green,.status-done,.route-financeiro,.sla-ok{background:var(--cf-green-2);color:#15803d;border-color:#bbf7d0}.cf-chip-orange,.status-pending,.route-operacoes{background:var(--cf-orange-2);color:#c2410c;border-color:#fed7aa}.cf-chip-purple,.route-suporte{background:var(--cf-purple-2);color:#6d28d9;border-color:#ddd6fe}.cf-chip-red,.status-failed,.sla-overdue{background:var(--cf-red-2);color:#b91c1c;border-color:#fecaca}.cf-chip-gray,.status-skipped,.route-rever,.status-badge,.route-badge{background:#f1f5f9;color:#475569;border-color:#e2e8f0}.status-pending{background:var(--cf-orange-2)!important;color:#c2410c!important;border-color:#fed7aa!important}.status-done{background:var(--cf-green-2)!important;color:#15803d!important;border-color:#bbf7d0!important}.status-failed{background:var(--cf-red-2)!important;color:#b91c1c!important;border-color:#fecaca!important} + +.cards{display:grid;grid-template-columns:repeat(4,minmax(170px,1fr));gap:16px;margin:16px 0}.card:not(.cf-card){padding:16px}.card span{color:var(--cf-muted)}.card strong{letter-spacing:-.04em}.kanban-toolbar{display:flex;justify-content:space-between;gap:12px;align-items:center;flex-wrap:wrap;background:rgba(255,255,255,.92);border:1px solid var(--cf-line);border-radius:18px;padding:12px;margin:16px 0;box-shadow:var(--cf-shadow);position:sticky;top:82px;z-index:12}.kanban-board,.kanban,.cf-kanban{display:grid;grid-template-columns:repeat(5,minmax(240px,1fr));gap:14px;align-items:start;overflow-x:auto;padding-bottom:.3rem}.kanban-column,.cf-kanban-col{background:#f8fafc;border:1px solid var(--cf-line);border-radius:16px;padding:12px;min-width:225px}.task-card,.cf-mini-card{background:#fff;border:1px solid var(--cf-line);border-radius:14px;padding:12px;margin-bottom:10px;box-shadow:0 4px 12px rgba(15,23,42,.04);transition:transform .12s ease,border-color .12s ease,box-shadow .12s ease}.task-card:hover,.cf-mini-card:hover{transform:translateY(-1px);border-color:#aac2ff;box-shadow:0 10px 26px rgba(15,23,42,.08)}.customer-name,.cf-mini-title{font-weight:900;color:#0f172a}.request-text,.cf-copy-box{white-space:pre-wrap;background:#f8fafc;border:1px solid var(--cf-line);border-radius:12px;padding:10px;color:#334155}.task-card-actions,.task-actions,.task-card-footer,.task-footer,.card-footer{display:flex;flex-wrap:wrap;gap:6px;align-items:center}.btn-chatwoot,.btn-more,.btn-external-opportunity,.detail-button,.task-card a[class^="btn"]{display:inline-flex;align-items:center;justify-content:center;gap:6px;min-height:34px;border-radius:11px;padding:7px 10px;border:1px solid var(--cf-line);background:#fff;color:#374151;font-size:13px;font-weight:800;margin:4px 4px 0 0}.btn-chatwoot{background:var(--cf-primary-2)!important;border-color:#bfdbfe!important;color:#1d4ed8!important}.btn-more{background:var(--cf-orange-2)!important;border-color:#fed7aa!important;color:#9a3412!important}.btn-external-opportunity,.detail-button-external,.detail-button-opportunity{background:var(--cf-green-2)!important;border-color:#bbf7d0!important;color:#15803d!important} + +.cf-two-col{display:grid;grid-template-columns:minmax(0,1fr) 360px;gap:18px;align-items:start}.cf-side{position:sticky;top:86px;display:grid;gap:16px}.cf-action-focus{background:#f8fafc;border:1px solid var(--cf-line);border-left:4px solid var(--cf-primary);border-radius:13px;padding:1rem}.cf-quick-actions{display:grid;grid-template-columns:1fr;gap:.5rem}.cf-stage-progress{display:grid;grid-template-columns:repeat(7,1fr);gap:.35rem;align-items:start}.cf-step{position:relative;display:grid;gap:.45rem;justify-items:center;text-align:center;color:#94a3b8;font-weight:800}.cf-step::before{content:"";position:absolute;top:.45rem;left:0;right:0;height:2px;background:#e2e8f0;z-index:0}.cf-step:first-child::before{left:50%}.cf-step:last-child::before{right:50%}.cf-step span{position:relative;z-index:1;width:.9rem;height:.9rem;border-radius:999px;background:#cbd5e1;border:3px solid #fff;box-shadow:0 0 0 1px #cbd5e1}.cf-step.done span,.cf-step.active span{background:var(--cf-primary);box-shadow:0 0 0 1px var(--cf-primary)}.cf-step.done,.cf-step.active{color:#0f172a}.cf-step small{font-size:.68rem;line-height:1.2}.cf-timeline{display:grid;gap:.75rem}.cf-timeline-item{display:grid;grid-template-columns:130px 1fr;gap:.75rem;padding-bottom:.75rem;border-bottom:1px solid #eef2f7}.cf-timeline-item:last-child{border-bottom:0;padding-bottom:0}.cf-op-alert-grid{display:grid;grid-template-columns:repeat(4,minmax(130px,1fr));gap:.75rem}.cf-op-alert{border:1px solid var(--cf-line);border-radius:13px;padding:.85rem;background:#fff}.cf-op-alert span{display:block;color:#64748b;font-size:.72rem;font-weight:900;text-transform:uppercase}.cf-op-alert strong{display:block;font-size:1.35rem;line-height:1;margin-top:.35rem}.cf-outbox-row-failed td,.cf-outbox-row-blocked td,.cf-work-row-priority-alta td,.cf-work-row-priority-urgente td{background:#fff7f7!important}.cf-outbox-row-pending td{background:#fffaf0!important}.cf-ia-tabs{display:flex;gap:.45rem;overflow-x:auto;padding:.25rem 0 .7rem;margin-bottom:.4rem}.cf-ia-tabs a{white-space:nowrap;border:1px solid #dbeafe;background:#eff6ff;color:#1d4ed8;border-radius:999px;padding:.42rem .72rem;font-weight:850;font-size:.78rem}.cf-ia-tabs a:hover{background:#dbeafe}.cf-op-grid{display:grid;grid-template-columns:repeat(6,minmax(0,1fr));gap:.75rem}.cf-op-card{border:1px solid var(--cf-line);border-radius:13px;padding:.9rem;background:#fff;display:grid;gap:.4rem;min-height:130px}.cf-op-icon{width:2.2rem;height:2.2rem;display:grid;place-items:center;border-radius:999px;background:#eff6ff;color:var(--cf-primary);font-weight:900}.cf-op-title{font-weight:850;font-size:.88rem}.cf-op-actions{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:.5rem}.cf-op-action-form{display:flex;gap:.4rem;align-items:center}.cf-op-action-form .form-control{min-width:0}.htmx-indicator{display:none}.htmx-request .htmx-indicator,.htmx-request.htmx-indicator{display:inline-flex}.cf-live-panel{scroll-margin-top:5rem}.cf-outbox-mini{background:#f8fafc;border:1px solid var(--cf-line);border-radius:13px}.cf-validation-list{margin:0;padding-left:1.15rem} + +.cf-ops-summary{display:grid;grid-template-columns:repeat(4,minmax(170px,1fr));gap:14px;margin-bottom:16px} +.cf-ops-counter{background:#fff;border:1px solid var(--cf-line);border-radius:16px;box-shadow:var(--cf-shadow);padding:15px 16px;display:flex;justify-content:space-between;gap:12px;align-items:center} +.cf-ops-counter span{display:block;color:var(--cf-muted);font-size:.75rem;font-weight:900;text-transform:uppercase;letter-spacing:.055em}.cf-ops-counter strong{font-size:1.85rem;letter-spacing:-.055em;line-height:1;color:#0f172a}.cf-ops-counter i{width:40px;height:40px;display:grid;place-items:center;border-radius:14px;background:#eff6ff;color:var(--cf-primary)}.cf-ops-counter.danger i{background:var(--cf-red-2);color:var(--cf-red)}.cf-ops-counter.warn i{background:var(--cf-orange-2);color:var(--cf-orange)}.cf-ops-counter.purple i{background:var(--cf-purple-2);color:var(--cf-purple)} +.cf-ops-filterbar{display:flex;flex-wrap:wrap;gap:8px;margin-bottom:16px}.cf-ops-filterbar a{display:inline-flex;align-items:center;border:1px solid var(--cf-line);background:#fff;border-radius:999px;padding:.5rem .85rem;font-weight:850;color:#475569}.cf-ops-filterbar a.active,.cf-ops-filterbar a:hover{background:var(--cf-primary-2);border-color:#bfdbfe;color:#1d4ed8} +.cf-work-section{display:grid;gap:12px;margin-bottom:20px}.cf-work-section-title{display:flex;align-items:center;gap:.55rem;margin:22px 0 10px;color:#475569;font-weight:950;text-transform:uppercase;letter-spacing:.08em;font-size:.78rem}.cf-work-section-title::after{content:"";height:1px;background:var(--cf-line);flex:1} +.cf-work-card{background:#fff;border:1px solid var(--cf-line);border-radius:18px;box-shadow:var(--cf-shadow);padding:18px;display:grid;gap:14px;border-left:5px solid #cbd5e1}.cf-work-card.high{border-left-color:var(--cf-red)}.cf-work-card.normal{border-left-color:var(--cf-primary)}.cf-work-card.blocked{border-left-color:var(--cf-orange)}.cf-work-card-head{display:flex;justify-content:space-between;gap:14px;align-items:flex-start}.cf-work-client{font-size:1.05rem;font-weight:950;color:#0f172a;line-height:1.15}.cf-work-process{color:var(--cf-muted);font-weight:750;margin-top:.25rem}.cf-work-next{background:#f8fafc;border:1px solid var(--cf-line);border-radius:14px;padding:12px}.cf-work-next span{display:block;color:var(--cf-muted);font-size:.74rem;font-weight:950;text-transform:uppercase;letter-spacing:.055em}.cf-work-next strong{display:block;font-size:1.05rem;margin-top:.15rem}.cf-work-meta{display:flex;flex-wrap:wrap;gap:8px}.cf-work-actions{display:flex;flex-wrap:wrap;gap:8px}.cf-work-actions .btn{min-width:0}.cf-work-empty{background:#fff;border:1px dashed #cbd5e1;border-radius:18px;padding:30px;text-align:center;color:#64748b} +@media(max-width:900px){.cf-ops-summary{grid-template-columns:repeat(2,minmax(0,1fr))}.cf-work-card-head{display:grid}.cf-work-actions .btn{width:100%}} + +@media(max-width:1200px){:root{--cf-sidebar-width:88px}.cf-brand-text,.cf-nav-title,.cf-nav-link span:last-child,.cf-sidebar-footer{display:none}.cf-brand{justify-content:center;padding-left:0;padding-right:0}.cf-nav-link{justify-content:center;padding:.75rem .45rem}.cf-nav-admin{margin-left:0;padding-left:0;border-left:0}.cf-kpi-grid,.cards{grid-template-columns:repeat(2,minmax(160px,1fr))}.cf-two-col{grid-template-columns:1fr}.cf-side{position:static}.cf-op-grid{grid-template-columns:repeat(3,minmax(0,1fr))}.cf-kanban,.kanban,.kanban-board{grid-template-columns:repeat(5,270px)}} + + +/* v4.8.6 — clean opportunity board layout. + The board uses minimum column widths instead of squeezing four columns into + the viewport. This avoids badge/text overlap and keeps each card readable. */ +.cf-opportunities-board-meta { + display: flex; + justify-content: space-between; + align-items: center; + gap: .75rem; + margin-bottom: .75rem; + color: #64748b; + font-size: .82rem; + font-weight: 700; +} + +.cf-opportunities-board-scroll { + width: 100%; + overflow-x: auto; + padding-bottom: .35rem; + scrollbar-gutter: stable; +} + +.cf-opportunities-board-grid { + display: grid; + grid-template-columns: repeat(4, minmax(300px, 1fr)); + gap: 1rem; + align-items: start; + min-width: 1220px; +} + +.cf-opportunity-stage { + min-width: 0; + background: #f8fafc; + border: 1px solid #dbe3ee; + border-radius: 16px; + overflow: hidden; + box-shadow: 0 10px 24px rgba(15, 23, 42, .045); +} + +.cf-opportunity-stage-header { + min-height: 44px; + display: flex; + justify-content: space-between; + align-items: center; + gap: .75rem; + padding: .8rem 1rem; + background: #ffffff; + border-bottom: 1px solid #e2e8f0; +} + +.cf-opportunity-stage-header strong { + min-width: 0; + color: #0f172a; + font-size: .92rem; + font-weight: 900; + line-height: 1.2; +} + +.cf-opportunity-stage-header span { + flex: 0 0 auto; + display: inline-grid; + min-width: 1.6rem; + height: 1.6rem; + place-items: center; + border-radius: 999px; + background: #f1f5f9; + color: #334155; + font-size: .78rem; + font-weight: 900; +} + +.cf-opportunity-stage-body { + display: grid; + gap: .85rem; + padding: .85rem; +} + +.cf-opportunity-card-clean { + display: grid; + gap: .85rem; + padding: 1rem; + border: 1px solid #e2e8f0; + border-radius: 14px; + background: #ffffff; + box-shadow: 0 6px 18px rgba(15, 23, 42, .055); +} + +.cf-opportunity-card-clean.has-blocker { + border-color: #fed7aa; + background: #fff7ed; +} + +.cf-opportunity-card-main { + display: grid; + gap: .45rem; + min-width: 0; +} + +.cf-opportunity-card-main:hover { + text-decoration: none; +} + +.cf-opportunity-card-title { + color: #0f172a; + font-size: .96rem; + font-weight: 900; + line-height: 1.25; + overflow-wrap: anywhere; +} + +.cf-opportunity-card-subtitle { + color: #64748b; + font-size: .78rem; + font-weight: 650; + line-height: 1.25; + overflow-wrap: anywhere; +} + +.cf-opportunity-card-subject { + margin-top: .2rem; + color: #111827; + font-size: .9rem; + font-weight: 800; + line-height: 1.3; + overflow-wrap: anywhere; +} + +.cf-opportunity-next-action { + display: grid; + gap: .2rem; + margin-top: .25rem; + padding: .75rem; + border: 1px solid #e2e8f0; + border-radius: 12px; + background: #f8fafc; +} + +.cf-opportunity-next-action span { + color: #64748b; + font-size: .72rem; + font-weight: 900; + letter-spacing: .04em; + text-transform: uppercase; +} + +.cf-opportunity-next-action strong { + color: #0f172a; + font-size: .92rem; + font-weight: 900; + line-height: 1.25; + overflow-wrap: anywhere; +} + +.cf-opportunity-card-cta { + margin: 0 !important; +} + +@media(max-width:760px){.cf-app{display:block}.cf-sidebar{position:static;width:100%;height:auto;max-height:none;min-height:0;display:flex;flex-direction:row;align-items:center;gap:10px;overflow-x:auto;overflow-y:hidden;padding:12px 10px}.cf-brand{padding:0}.cf-brand-mark{width:32px;height:32px}.cf-admin-nav{display:flex;align-items:center;gap:5px;margin-top:0}.cf-admin-nav[open] .cf-nav-admin{display:flex}.cf-nav{display:flex;flex-direction:row;min-width:max-content}.cf-nav-link{min-height:38px;padding:.5rem .65rem}.cf-content{width:100%}.cf-topbar{position:sticky;top:0;padding:0 14px;height:62px}.cf-global-search{display:none}.cf-pill.cf-date{display:none}.cf-page{width:calc(100% - 24px);padding:20px 0 34px}.cf-page-title{flex-direction:column}.cf-page-title::after{display:none}.cf-kpi-grid,.cards,.cf-op-alert-grid,.cf-op-grid,.cf-op-actions{grid-template-columns:1fr}.row{--bs-gutter-x:.75rem}.cf-table{border-spacing:0 8px!important}.cf-table thead{display:none}.cf-table tbody tr{display:block;margin-bottom:10px}.cf-table tbody td{display:block!important;border-left:1px solid var(--cf-line)!important;border-right:1px solid var(--cf-line)!important;border-top:0!important;border-bottom:0!important;border-radius:0!important}.cf-table tbody td:first-child{border-top:1px solid var(--cf-line)!important;border-radius:12px 12px 0 0!important}.cf-table tbody td:last-child{border-bottom:1px solid var(--cf-line)!important;border-radius:0 0 12px 12px!important}.btn,button{min-height:38px}.task-card a[class^="btn"],.btn-chatwoot,.btn-more,.btn-external-opportunity,.detail-button{width:100%;margin-right:0}.cf-timeline-item{grid-template-columns:1fr}.cf-stage-progress{grid-template-columns:repeat(3,1fr)}} +@media(max-width:430px){body{font-size:.9rem}.cf-kpi strong{font-size:1.55rem}.cf-page-title h1{font-size:1.55rem}.cf-sidebar{padding:10px 8px}.cf-nav-icon{width:20px;height:20px}.cf-topbar-actions{gap:6px}.cf-icon-btn{width:32px;height:32px}.cf-pill{padding:7px 9px}} + + +/* v4.7 admin navigation */ +/* v4.7.1 admin navigation overflow fix */ +.cf-admin-nav{margin-top:2px;flex:0 0 auto;min-height:0} +.cf-admin-nav summary{cursor:pointer;list-style:none} +.cf-admin-nav summary::-webkit-details-marker{display:none} +.cf-nav-admin{margin-left:10px;padding-left:10px;border-left:1px solid var(--cf-line,#e5e7eb);padding-bottom:8px} +.cf-nav-admin .cf-nav-link{min-height:38px;font-size:.92rem} +.cf-sidebar::-webkit-scrollbar{width:10px} +.cf-sidebar::-webkit-scrollbar-track{background:transparent} +.cf-sidebar::-webkit-scrollbar-thumb{background:#cbd5e1;border-radius:999px;border:3px solid transparent;background-clip:content-box} +.cf-sidebar::-webkit-scrollbar-thumb:hover{background:#94a3b8;border:3px solid transparent;background-clip:content-box} + + +/* v4.7.3 HTMX partial refresh polish */ +.cf-live-panel{scroll-margin-top:5rem} +.cf-live-panel.htmx-swapping{opacity:.75} +.cf-live-panel .htmx-indicator{align-items:center;gap:.35rem} +.cf-ops-filterbar a[hx-get],.cf-task-tabs a[hx-get]{cursor:pointer} + + +/* v4.8.2 guided operations UI */ +.cf-fiscal-mini{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:.5rem;margin:.75rem 0}.cf-fiscal-mini>div{background:#f8fafc;border:1px solid #e2e8f0;border-radius:.75rem;padding:.55rem}.cf-fiscal-mini span,.cf-fiscal-label{display:block;color:#64748b;font-size:.72rem;font-weight:900;text-transform:uppercase;letter-spacing:.02em}.cf-fiscal-mini strong{display:block;color:#0f172a;font-size:.86rem;overflow-wrap:anywhere}.cf-fiscal-contact-grid{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:.75rem}.cf-fiscal-box{border:1px solid #e2e8f0;border-radius:1rem;padding:1rem;background:#f8fafc}.cf-fiscal-box.fiscal{border-color:#bfdbfe;background:#eff6ff}.cf-fiscal-box.contact{border-color:#e9d5ff;background:#faf5ff}.cf-readiness-card{border-left:4px solid #0d6efd}.cf-readiness-pill{display:inline-flex;align-items:center;gap:.35rem;border-radius:999px;padding:.45rem .7rem;font-weight:800;font-size:.84rem}.cf-readiness-pill.missing{background:#fef2f2;color:#991b1b;border:1px solid #fecaca}.cf-readiness-pill.ok{background:#dcfce7;color:#166534;border:1px solid #86efac}.cf-next-step-panel{border:1px solid #bfdbfe;background:linear-gradient(135deg,#eff6ff,#fff);border-radius:1rem;padding:1rem}.cf-next-step-panel span{display:block;color:#64748b;font-size:.75rem;font-weight:900;text-transform:uppercase}.cf-next-step-panel strong{display:block;font-size:1.12rem;color:#0f172a}.cf-health-status{border-radius:1rem;padding:1rem;border:1px solid #e2e8f0;background:white}.cf-health-status.ok{border-color:#86efac;background:#f0fdf4}.cf-health-status.warn{border-color:#fde68a;background:#fffbeb}.cf-health-status.critical{border-color:#fecaca;background:#fef2f2}.cf-outbox-human-error{background:#fff7ed;border:1px solid #fed7aa;border-radius:.85rem;padding:.75rem}.cf-outbox-human-error strong{display:block;color:#9a3412}.cf-outbox-human-error .small{color:#7c2d12}@media(max-width:720px){.cf-fiscal-mini,.cf-fiscal-contact-grid{grid-template-columns:1fr}} + +/* v4.8.3 Operations card simplification */ +.cf-work-next.compact{margin-top:.75rem;padding:.8rem .9rem;border-radius:.9rem;background:#f8fafc;border:1px solid var(--cf-line)} +.cf-work-blockers{margin-top:.75rem;padding:.75rem .9rem;border-radius:.9rem;background:#fff7ed;border:1px solid #fed7aa;color:#9a3412} +.cf-work-blockers strong{display:block;font-size:.78rem;text-transform:uppercase;letter-spacing:.04em;margin-bottom:.25rem} +.cf-work-blockers ul{margin:.25rem 0 0 1rem;padding:0} +.cf-fiscal-mini.compact{grid-template-columns:1fr} + + +/* v4.8.9 Operations card polish */ +.cf-work-card{gap:12px} +.cf-work-card-head{border-bottom:1px solid #eef2f7;padding-bottom:.75rem} +.cf-work-client{overflow-wrap:anywhere} +.cf-work-next.compact{background:linear-gradient(135deg,#f8fafc,#ffffff)} +.cf-work-actions{align-items:center;justify-content:space-between;border-top:1px solid #eef2f7;padding-top:.75rem;margin-top:.1rem} +.cf-work-primary-action{font-weight:900;min-width:150px} +.cf-work-secondary-actions{display:flex;flex-wrap:wrap;gap:8px;justify-content:flex-end} +.cf-work-secondary-actions .btn{font-weight:750} +@media(max-width:720px){.cf-work-card-head{display:grid}.cf-work-actions{display:grid;gap:8px;justify-content:stretch}.cf-work-primary-action,.cf-work-secondary-actions .btn{width:100%}.cf-work-secondary-actions{display:grid;gap:8px;justify-content:stretch}} + + +/* v4.9.5 reconciliation operation suggestions */ +.cf-reconcile-suggestion { + border: 1px solid rgba(37, 99, 235, 0.22); + background: rgba(37, 99, 235, 0.06); + border-radius: 12px; + padding: 0.75rem; + text-align: left; +} +.cf-reconcile-manual-link summary { + cursor: pointer; + font-weight: 700; +} +.cf-reconcile-manual-link[open] { + border-top: 1px dashed rgba(100, 116, 139, 0.35); + padding-top: 0.35rem; +} + +/* v4.9.9 process timeline reconstruction */ +.cf-reconcile-process-card{border:1px solid var(--cf-line);border-radius:16px;background:#fff;box-shadow:var(--cf-shadow);padding:16px;display:grid;gap:14px} +.cf-reconcile-process-state{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:10px} +.cf-reconcile-process-state>div{background:#f8fafc;border:1px solid var(--cf-line);border-radius:13px;padding:10px} +.cf-reconcile-process-state span{display:block;color:var(--cf-muted);font-size:.72rem;font-weight:900;text-transform:uppercase;letter-spacing:.05em} +.cf-reconcile-process-state strong{display:block;margin-top:.15rem;color:#0f172a} +.cf-reconcile-timeline{display:grid;gap:8px;border-left:3px solid #dbeafe;padding-left:12px} +.cf-reconcile-timeline-step{position:relative;background:#f8fafc;border:1px solid var(--cf-line);border-radius:12px;padding:10px} +.cf-reconcile-timeline-step::before{content:"";position:absolute;left:-19px;top:14px;width:9px;height:9px;border-radius:999px;background:var(--cf-primary);box-shadow:0 0 0 3px #fff} +@media(max-width:760px){.cf-reconcile-process-state{grid-template-columns:1fr}} + + +/* v4.9.24 reconciliation explanations */ +.cf-reconcile-explain{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:10px;background:#f8fafc;border:1px solid var(--cf-line);border-radius:14px;padding:10px} +.cf-reconcile-explain strong{display:block;font-size:.72rem;text-transform:uppercase;letter-spacing:.05em;color:var(--cf-muted);margin-bottom:4px} +.cf-reconcile-explain ul{margin:0;padding-left:1.05rem;color:#334155;font-size:.84rem} +.cf-reconcile-explain li+li{margin-top:2px} +@media(max-width:760px){.cf-reconcile-explain{grid-template-columns:1fr}} + + +/* v4.9.28 reconciliation decision board */ +.cf-reconcile-decision-grid{display:grid;grid-template-columns:repeat(5,minmax(0,1fr));gap:12px}.cf-decision-card{display:grid;gap:.25rem;text-decoration:none;color:#0f172a;background:#fff;border:1px solid var(--cf-line);border-radius:16px;padding:14px;box-shadow:var(--cf-shadow)}.cf-decision-card.active{border-color:#bfdbfe;background:linear-gradient(135deg,#eff6ff,#fff)}.cf-decision-card span{color:var(--cf-muted);font-size:.72rem;font-weight:900;text-transform:uppercase;letter-spacing:.05em}.cf-decision-card strong{font-size:1.6rem;line-height:1}.cf-decision-card small{color:var(--cf-muted);font-weight:650}.cf-reconcile-decision-note{display:grid;gap:.2rem;background:#eff6ff;border:1px solid #bfdbfe;border-radius:14px;padding:10px}.cf-reconcile-decision-note strong{color:#1d4ed8}.cf-reconcile-decision-note span{color:#475569;font-size:.86rem}@media(max-width:1000px){.cf-reconcile-decision-grid{grid-template-columns:repeat(2,minmax(0,1fr))}}@media(max-width:560px){.cf-reconcile-decision-grid{grid-template-columns:1fr}} +""" diff --git a/app/admin_ui/view_models/__init__.py b/app/admin_ui/view_models/__init__.py new file mode 100644 index 0000000..74a1cf3 --- /dev/null +++ b/app/admin_ui/view_models/__init__.py @@ -0,0 +1 @@ +"""View-model builders used by the modular admin UI.""" diff --git a/app/admin_ui/view_models/operations.py b/app/admin_ui/view_models/operations.py new file mode 100644 index 0000000..a679d36 --- /dev/null +++ b/app/admin_ui/view_models/operations.py @@ -0,0 +1,235 @@ +"""View model for the operator workbench. + +This module is intentionally UI-focused: it does not change workflow rules, it +only prepares the data returned by operations_service for full-page and HTMX +partial rendering. +""" +from __future__ import annotations + +from datetime import datetime, timezone +from typing import Any + +from app.operations_service import get_operations_summary +from app.admin_ui.labels import primary_action_label +from app.admin_ui.guidance import item_requires_fiscal_customer, work_item_blockers + +# Legacy label kept for regression context: Ambíguas. +OPERATION_FILTERS = [ + ("all", "A fazer", "/operations?scope=all", "/operations/partials/work-items?scope=all"), + ("bloqueadas", "Bloqueadas", "/operations?scope=bloqueadas", "/operations/partials/work-items?scope=bloqueadas"), + ("ambiguas", "Associações por confirmar", "/operations?scope=ambiguas", "/operations/partials/work-items?scope=ambiguas"), + ("atrasadas", "Atrasadas", "/operations?scope=atrasadas", "/operations/partials/work-items?scope=atrasadas"), + ("vendas", "Vendas", "/operations?scope=vendas", "/operations/partials/work-items?scope=vendas"), + ("financeiro", "Financeiro", "/operations?scope=financeiro", "/operations/partials/work-items?scope=financeiro"), + ("logistica", "Logística", "/operations?scope=logistica", "/operations/partials/work-items?scope=logistica"), + ("revisao", "Revisão", "/operations?scope=revisao", "/operations/partials/work-items?scope=revisao"), + ("done", "Concluídas hoje", "/tasks?status=done", "/tasks/partials/list?status=done"), +] + + + + +def _is_numeric_reference(value: Any) -> bool: + value = str(value or "").strip() + return bool(value) and value.isdigit() + + +def is_classification_failed(item: dict[str, Any]) -> bool: + detail = str(item.get("detail") or "").casefold() + return "classificação da mensagem falhou" in detail or "classificacao da mensagem falhou" in detail + + +def has_no_commercial_opportunity(item: dict[str, Any]) -> bool: + reason = str(item.get("no_opportunity_reason") or "").strip() + action_code = str(item.get("action_code") or "").upper() + return bool(reason) or action_code in {"REVIEW_MANUALLY", "REMOVE_FROM_LIST", "IGNORE_SPAM", "NO_ACTION"} + + +def _looks_like_generic_label(value: str) -> bool: + value = " ".join(str(value or "").strip().casefold().split()) + return value in {"", "geral", "cliente", "contacto", "sem nome"} + + +# Legacy label intentionally no longer used as primary title: "Contacto Chatwoot #". +# It is only suitable as technical metadata/fallback, not as operator identity. +def operation_card_title(item: dict[str, Any]) -> str: + if str(item.get("source") or "") == "outbox": + return str(item.get("customer_name") or item.get("title") or "Integração") + fiscal = str(item.get("fiscal_customer_name") or "").strip() + contact = str(item.get("contact_display_name") or item.get("customer_name") or "").strip() + if fiscal and not _is_numeric_reference(fiscal) and not _looks_like_generic_label(fiscal): + return fiscal + if contact and not _is_numeric_reference(contact) and not _looks_like_generic_label(contact): + return contact + return "Contacto sem identificação" + + +def operation_card_subtitle(item: dict[str, Any]) -> str: + if is_classification_failed(item): + return "Classificação por rever" + reason = str(item.get("no_opportunity_reason") or "").strip() + if reason in {"system_or_bounce_message", "bounce_ignored"}: + return "Mensagem automática / bounce" + if reason: + return "Sem oportunidade comercial" + opportunity = str(item.get("opportunity_title") or "").strip() + if opportunity: + return opportunity + if str(item.get("source") or "") == "outbox": + return str(item.get("title") or "Integração") + if str(item.get("action_code") or "").upper() in {"REMOVE_FROM_LIST", "IGNORE_SPAM", "NO_ACTION"}: + return "Sem oportunidade comercial" + conv = str(item.get("conversation_id") or "").strip() + if operation_card_title(item) == "Contacto sem identificação" and conv: + return f"Conversa Chatwoot #{conv}" + return "Sem oportunidade associada" + + +def operation_has_useful_details(item: dict[str, Any]) -> bool: + if str(item.get("source") or "") == "outbox": + return True + if str(item.get("opportunity_id") or "").strip(): + return True + if str(item.get("fiscal_customer_name") or "").strip(): + return True + if item_requires_fiscal_customer(item): + return True + if work_item_blockers(item): + return True + return False + + +def operation_card_detail(item: dict[str, Any]) -> str: + if is_classification_failed(item): + return "O sistema não conseguiu classificar com segurança. Abrir no Chatwoot e escolher a ação correta." + reason = str(item.get("no_opportunity_reason") or "").strip() + if reason in {"system_or_bounce_message", "bounce_ignored"}: + return "Mensagem automática detetada. Não entra no fluxo operacional." + if reason == "send_info_without_clear_commercial_intent": + return "Responder sem criar oportunidade até existir intenção comercial clara." + return str(item.get("detail") or item.get("request_text") or "").strip() + + +def operation_status_chip(item: dict[str, Any]) -> str: + blockers = work_item_blockers(item) + if blockers: + return "bloqueada" + if is_high_priority(item): + return "prioridade alta" + if has_no_commercial_opportunity(item): + return "sem oportunidade comercial" + return "normal" + + +def is_ambiguous(item: dict[str, Any]) -> bool: + return str(item.get("opportunity_linking_status") or "").lower() == "ambiguous" + + +def is_overdue(item: dict[str, Any]) -> bool: + created = item.get("created_at") + if not created: + return False + if isinstance(created, str): + try: + created_dt = datetime.fromisoformat(created.replace("Z", "+00:00")) + except ValueError: + return False + else: + created_dt = created + if created_dt.tzinfo is None: + created_dt = created_dt.replace(tzinfo=timezone.utc) + queue = str(item.get("queue") or "").lower() + hours = {"suporte": 2, "vendas": 4, "financeiro": 8, "operacoes": 24, "logistica": 24, "rever": 24}.get(queue, 24) + return (datetime.now(timezone.utc) - created_dt).total_seconds() > hours * 3600 + +def is_high_priority(item: dict[str, Any]) -> bool: + status = str(item.get("status") or "").lower() + source = str(item.get("source") or "").lower() + queue = str(item.get("queue") or "").lower() + action_code = str(item.get("action_code") or "").upper() + priority = str(item.get("priority") or "").lower() + + if is_ambiguous(item): + return True + if source == "outbox" and status in {"failed", "blocked", "stale"}: + return True + if is_overdue(item): + return True + if is_classification_failed(item): + return False + if queue == "marketing" or action_code in {"REMOVE_FROM_LIST", "IGNORE_SPAM", "NO_ACTION", "REVIEW_MANUALLY"}: + return False + if action_code in {"SEND_PROFORMA", "SEND_INVOICE", "CONFIRM_PAYMENT", "PREPARE_ORDER", "CREATE_SHIPMENT"}: + return priority in {"alta", "high", "urgente", "critical"} or status in {"failed", "blocked"} + return priority in {"alta", "high", "urgente"} or status in {"failed", "blocked"} + + +def is_blocked(item: dict[str, Any]) -> bool: + status = str(item.get("status") or "").lower() + source = str(item.get("source") or "").lower() + return status in {"failed", "blocked", "stale"} or source == "outbox" or bool(work_item_blockers(item)) + + +def operation_item_matches_scope(item: dict[str, Any], scope: str) -> bool: + queue = str(item.get("queue") or "").lower() + action_code = str(item.get("action_code") or "").upper() + scope = str(scope or "all").strip().lower() + if scope in {"", "all", "todas"}: + return True + if scope == "bloqueadas": + return is_blocked(item) + if scope in {"ambiguas", "ambíguas"}: + return is_ambiguous(item) + if scope == "atrasadas": + return is_overdue(item) + if scope == "logistica": + return queue in {"operacoes", "logistica", "logística"} + if scope == "revisao": + return queue in {"rever", "revisao", "revisão"} or action_code == "REVIEW_MANUALLY" + return queue == scope + + +def operation_primary_label(item: dict[str, Any]) -> str: + if is_classification_failed(item): + return "Rever mensagem" + if str(item.get("source") or "") == "outbox": + status = str(item.get("status") or "").lower() + return "Reprocessar" if status == "failed" else "Ver bloqueio" + return primary_action_label(item.get("action_code") or item.get("action_label"), fallback="Abrir") + + +def operation_origin_label(item: dict[str, Any]) -> str: + source = str(item.get("source_system") or item.get("source") or "").strip() + conv = str(item.get("conversation_id") or "").strip() + if conv: + return f"Chatwoot #{conv}" + if source: + return source + return "ClientFlow" + + +def build_operations_view_model(scope: str = "all", *, limit: int = 30) -> dict[str, Any]: + data = get_operations_summary(limit=limit) + counts = data.get("counts") or {} + scope = str(scope or "all").strip().lower() + work_items = list(data.get("work_items") or []) + visible_items = [item for item in work_items if operation_item_matches_scope(item, scope)] + high_items = [item for item in visible_items if is_high_priority(item)] + normal_items = [item for item in visible_items if not is_high_priority(item)] + review_total = int(counts.get("review_tasks", 0) or 0) + int(counts.get("communications_needs_review", 0) or 0) + blocked_total = int(counts.get("blocked_outbox", 0) or 0) + sum(1 for item in work_items if is_blocked(item)) + ambiguous_total = sum(1 for item in work_items if is_ambiguous(item)) + overdue_total = int(counts.get("overdue_tasks", 0) or 0) + sum(1 for item in work_items if item.get("source") == "outbox" and str(item.get("status") or "").lower() in {"failed", "stale"}) + return { + "scope": scope, + "counts": counts, + "review_total": review_total, + "blocked_total": blocked_total, + "ambiguous_total": ambiguous_total, + "overdue_total": overdue_total, + "today_label": datetime.now(timezone.utc).strftime("%d/%m/%Y"), + "filters": OPERATION_FILTERS, + "visible_items": visible_items, + "high_items": high_items, + "normal_items": normal_items, + } diff --git a/app/analyzer.py b/app/analyzer.py new file mode 100644 index 0000000..91630df --- /dev/null +++ b/app/analyzer.py @@ -0,0 +1,90 @@ +from app.action_decider import decide_action +from app.action_mapper import map_action_decision +from app.config import settings +from app.persistence import save_action_run +from app.schemas import ActionDecision, AnalyzeRequest, AnalyzeResponse, UsageInfo +from app.task_service import create_task_from_action_result + + +async def analyze(request: AnalyzeRequest, raw_event_id: str | None = None) -> AnalyzeResponse: + needs_review = False + + try: + decision, action_result, usage, decision_source = await decide_action(request) + except Exception as exc: + needs_review = True + decision = ActionDecision( + action_code="REVIEW_MANUALLY", + note=f"Falha na decisão Action Core: {exc}", + confidence=0.0, + ) + action_result = map_action_decision(decision) + usage = UsageInfo( + id=None, + model=settings.openrouter_model, + provider="error_fallback", + prompt_tokens=0, + completion_tokens=0, + total_tokens=0, + cost=0.0, + ) + decision_source = "fallback" + + if str(action_result.action_code or "").upper() == "IGNORE_BOUNCE": + # v4.8.5: NDR/bounce emails are already readable in Chatwoot/Thunderbird. + # They must not create ClientFlow work, opportunities or fiscal customers. + return AnalyzeResponse( + app="ClientFlow", + model=settings.openrouter_model, + action_decision=decision, + action_result=action_result, + usage=usage, + needs_review=False, + action_run_id=None, + message_id=None, + task_id=None, + ) + + if not action_result.safe_to_post: + needs_review = True + + action_run_id, message_id = save_action_run( + request=request, + action_decision=decision, + action_result=action_result, + usage=usage, + needs_review=needs_review, + model=settings.openrouter_model, + decision_source=decision_source, + raw_body=request.last_customer_message, + clean_body=request.last_customer_message, + raw_event_id=raw_event_id, + ) + + task_id = create_task_from_action_result( + action_result=action_result, + action_run_id=action_run_id, + message_id=message_id, + raw_event_id=raw_event_id, + conversation_id=request.conversation_id, + contact_id=request.contact_id, + source_system=request.source or "manual", + source_event_id=message_id, + metadata={ + "created_from_analyzer": True, + "needs_review": needs_review, + "decision_source": decision_source, + }, + ) + + return AnalyzeResponse( + app="ClientFlow", + model=settings.openrouter_model, + action_decision=decision, + action_result=action_result, + usage=usage, + needs_review=needs_review, + action_run_id=action_run_id, + message_id=message_id, + task_id=task_id, + ) diff --git a/app/api/__init__.py b/app/api/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/api/internal.py b/app/api/internal.py new file mode 100644 index 0000000..302f91f --- /dev/null +++ b/app/api/internal.py @@ -0,0 +1,100 @@ +"""Small internal operational API. + +These endpoints are intentionally read-only and are used by future HTMX/API +partials and external monitoring. They do not replace the current admin pages. +""" +from __future__ import annotations + +from hmac import compare_digest + +from fastapi import APIRouter, Depends, HTTPException, Request +from sqlalchemy import text + +from app.config import settings +from app.db import engine + +def require_internal_access(request: Request) -> None: + expected = (settings.clientflow_admin_token or "").strip() + if not expected: + return + received = ( + request.headers.get("X-ClientFlow-Admin-Token") + or request.cookies.get("clientflow_admin_token") + or request.query_params.get("admin_token") + or "" + ).strip() + if not received or not compare_digest(received, expected): + raise HTTPException(status_code=401, detail="internal api auth required") + + +router = APIRouter(prefix="/api/internal", tags=["internal"], dependencies=[Depends(require_internal_access)]) + + +@router.get("/health") +def internal_health() -> dict: + return { + "status": "ok", + "app": settings.app_name, + "env": settings.env, + "jasmin_enabled": bool(settings.jasmin_enabled), + "packlink_enabled": bool(settings.packlink_enabled), + } + + +@router.get("/outbox/summary") +def outbox_summary() -> dict: + with engine.begin() as conn: + rows = conn.execute(text(""" + SELECT target_system, status, count(*)::int AS total + FROM integration_outbox + GROUP BY target_system, status + ORDER BY target_system, status + """)).mappings().all() + + summary: dict[str, dict[str, int]] = {} + for row in rows: + target = row["target_system"] or "unknown" + status = row["status"] or "unknown" + summary.setdefault(target, {})[status] = row["total"] + return {"items": summary} + + +@router.get("/documents/summary") +def documents_summary() -> dict: + with engine.begin() as conn: + rows = conn.execute(text(""" + SELECT document_kind, status, count(*)::int AS total + FROM commercial_documents + GROUP BY document_kind, status + ORDER BY document_kind, status + """)).mappings().all() + + summary: dict[str, dict[str, int]] = {} + for row in rows: + kind = row["document_kind"] or "unknown" + status = row["status"] or "unknown" + summary.setdefault(kind, {})[status] = row["total"] + return {"items": summary} + +@router.get("/operations/summary") +def operations_summary() -> dict: + from app.operations_service import get_operations_summary + return get_operations_summary(limit=10) + + +@router.get("/system/health") +def system_health_summary() -> dict: + from app.operations_service import get_system_health_summary + return get_system_health_summary() + + +@router.get("/communications/summary") +def communications_summary() -> dict: + from app.communication_service import get_communications_summary + return get_communications_summary() + + +@router.get("/communications/recent") +def communications_recent(limit: int = 20) -> dict: + from app.communication_service import list_communications + return {"items": list_communications(limit=limit)} diff --git a/app/chatwoot_client.py b/app/chatwoot_client.py new file mode 100644 index 0000000..93e16bc --- /dev/null +++ b/app/chatwoot_client.py @@ -0,0 +1,94 @@ +from typing import Dict + +import httpx + +from app.config import settings +from app.posting_policy import can_post_private_note +from app.schemas import ActionResult +import os + + +def build_private_note(action_result: ActionResult) -> str: + return f"""🤖 ClientFlow + +Ação: +{action_result.action} + +Resumo: +{action_result.note} +""" + + +async def add_private_note( + conversation_id: str, + action_result: ActionResult, +) -> Dict: + if os.getenv("CLIENTFLOW_DISABLE_CHATWOOT_PRIVATE_NOTES", "true").lower() in {"1", "true", "yes", "sim"}: + return { + "enabled": False, + "status": "disabled_by_env", + "reason": "CLIENTFLOW_DISABLE_CHATWOOT_PRIVATE_NOTES=true", + } + + if not settings.chatwoot_write_enabled: + return { + "enabled": False, + "status": "skipped", + "reason": "CHATWOOT_WRITE_ENABLED=false", + "action_result": action_result.model_dump(), + } + + if not can_post_private_note(action_result): + return { + "enabled": True, + "status": "skipped", + "reason": "posting_policy_blocked", + "action_result": action_result.model_dump(), + } + + if not settings.chatwoot_base_url or not settings.chatwoot_account_id or not settings.chatwoot_api_token: + return { + "enabled": True, + "status": "skipped", + "reason": "missing Chatwoot config", + "action_result": action_result.model_dump(), + } + + url = ( + settings.chatwoot_base_url.rstrip("/") + + f"/api/v1/accounts/{settings.chatwoot_account_id}" + + f"/conversations/{conversation_id}/messages" + ) + + payload = { + "content": build_private_note(action_result), + "message_type": "outgoing", + "private": True, + "content_type": "text", + "content_attributes": {}, + } + + headers = { + "Content-Type": "application/json", + "api_access_token": settings.chatwoot_api_token, + } + + async with httpx.AsyncClient(timeout=30) as client: + response = await client.post(url, headers=headers, json=payload) + + if response.status_code >= 400: + return { + "enabled": True, + "status": "failed", + "status_code": response.status_code, + "body": response.text, + "action_result": action_result.model_dump(), + } + + return { + "enabled": True, + "status": "sent", + "status_code": response.status_code, + "response": response.json(), + "action_result": action_result.model_dump(), + } diff --git a/app/commercial/__init__.py b/app/commercial/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/commercial_service.py b/app/commercial_service.py new file mode 100644 index 0000000..cd438d4 --- /dev/null +++ b/app/commercial_service.py @@ -0,0 +1,885 @@ +"""Domínio comercial ClientFlow. + +Guarda clientes locais, documentos Jasmin (orçamentos/faturas) e envios +externos de forma normalizada. Esta camada evita transformar operation_links +num armazenamento principal e permite vários orçamentos/faturas por cliente e +por oportunidade. +""" + +from __future__ import annotations + +import json +import re +import uuid +from decimal import Decimal +from typing import Any, Dict, Iterable, List, Optional + +from sqlalchemy import text + +from app.db import engine + +_SCHEMA_READY = False + + +def _json(value: Any) -> str: + return json.dumps(value or {}, ensure_ascii=False, default=str) + + +def _clean(value: Any) -> str: + return str(value or "").strip() + + +def normalize_tax_id(value: Any) -> str: + """Normaliza NIF/VAT PT para uso local e Jasmin. + + Nos testes reais Jasmin, getCustomerByCompanyTaxId aceitou 504931946 e + não encontrou PT504931946. Por isso guardamos sem prefixo PT. + """ + tax_id = _clean(value).upper().replace(" ", "").replace("-", "") + if tax_id.startswith("PT"): + tax_id = tax_id[2:] + return tax_id + + +def normalize_fiscal_name(value: Any) -> str: + """Normaliza nome fiscal para evitar duplicados quando falta NIF. + + No domínio ClientFlow o nome fiscal de empresa é tratado como chave de + identidade forte. Esta normalização é deliberadamente simples e estável: + remove pontuação/ruído de espaçamento, preservando a designação legal. + """ + name = _clean(value).casefold() + name = re.sub(r"[^0-9a-záàâãéêíóôõúç]+", " ", name, flags=re.IGNORECASE) + return re.sub(r"\s+", " ", name).strip() + + +def ensure_commercial_schema() -> None: + global _SCHEMA_READY + if _SCHEMA_READY: + return + + with engine.begin() as conn: + conn.execute(text(""" + CREATE TABLE IF NOT EXISTS customers ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + name TEXT NOT NULL, + tax_id TEXT, + email TEXT, + phone TEXT, + street_name TEXT, + postal_zone TEXT, + city_name TEXT, + country TEXT NOT NULL DEFAULT 'PT', + jasmin_customer_party_key TEXT, + jasmin_customer_id TEXT, + metadata JSONB NOT NULL DEFAULT '{}'::jsonb, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + """)) + conn.execute(text(""" + CREATE UNIQUE INDEX IF NOT EXISTS ux_customers_tax_id + ON customers(tax_id) + WHERE tax_id IS NOT NULL AND tax_id <> '' + """)) + conn.execute(text("CREATE INDEX IF NOT EXISTS idx_customers_name ON customers(name)")) + conn.execute(text("CREATE INDEX IF NOT EXISTS idx_customers_jasmin_key ON customers(jasmin_customer_party_key)")) + conn.execute(text("CREATE INDEX IF NOT EXISTS idx_customers_external_identity ON customers USING gin (metadata)")) + + conn.execute(text(""" + CREATE TABLE IF NOT EXISTS commercial_documents ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + customer_id UUID REFERENCES customers(id) ON DELETE SET NULL, + opportunity_id UUID REFERENCES opportunities(id) ON DELETE SET NULL, + system TEXT NOT NULL DEFAULT 'jasmin', + document_kind TEXT NOT NULL, + external_id TEXT, + external_url TEXT, + company TEXT, + document_type TEXT, + serie TEXT, + series_number INTEGER, + document_number TEXT, + customer_party_key TEXT, + status TEXT NOT NULL DEFAULT 'draft', + amount NUMERIC(12,2), + tax_amount NUMERIC(12,2), + total_amount NUMERIC(12,2), + currency TEXT DEFAULT 'EUR', + parent_document_id UUID REFERENCES commercial_documents(id) ON DELETE SET NULL, + version_number INTEGER, + role TEXT NOT NULL DEFAULT 'current', + is_primary BOOLEAN NOT NULL DEFAULT TRUE, + is_active BOOLEAN NOT NULL DEFAULT TRUE, + document_date DATE, + due_date DATE, + payload JSONB NOT NULL DEFAULT '{}'::jsonb, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + """)) + # Upgrades aditivos para bases que já tenham a tabela. + for stmt in [ + "ALTER TABLE commercial_documents ADD COLUMN IF NOT EXISTS external_url TEXT", + "ALTER TABLE commercial_documents ADD COLUMN IF NOT EXISTS parent_document_id UUID REFERENCES commercial_documents(id) ON DELETE SET NULL", + "ALTER TABLE commercial_documents ADD COLUMN IF NOT EXISTS version_number INTEGER", + "ALTER TABLE commercial_documents ADD COLUMN IF NOT EXISTS role TEXT NOT NULL DEFAULT 'current'", + "ALTER TABLE commercial_documents ADD COLUMN IF NOT EXISTS is_primary BOOLEAN NOT NULL DEFAULT TRUE", + "ALTER TABLE commercial_documents ADD COLUMN IF NOT EXISTS is_active BOOLEAN NOT NULL DEFAULT TRUE", + "ALTER TABLE commercial_documents ADD COLUMN IF NOT EXISTS tax_amount NUMERIC(12,2)", + "ALTER TABLE commercial_documents ADD COLUMN IF NOT EXISTS total_amount NUMERIC(12,2)", + ]: + conn.execute(text(stmt)) + conn.execute(text("CREATE INDEX IF NOT EXISTS idx_commercial_documents_customer ON commercial_documents(customer_id)")) + conn.execute(text("CREATE INDEX IF NOT EXISTS idx_commercial_documents_opportunity ON commercial_documents(opportunity_id)")) + conn.execute(text("CREATE INDEX IF NOT EXISTS idx_commercial_documents_external ON commercial_documents(system, external_id)")) + conn.execute(text("CREATE INDEX IF NOT EXISTS idx_commercial_documents_kind ON commercial_documents(document_kind, status)")) + conn.execute(text("CREATE INDEX IF NOT EXISTS idx_commercial_documents_role ON commercial_documents(opportunity_id, document_kind, role, is_primary)")) + + # Hotfix v4927.1: em bases já usadas podem existir vários documentos + # Jasmin/Odoo ligados à mesma oportunidade e ao mesmo tipo. Ao adicionar + # role/is_primary com DEFAULT current/TRUE, todos passariam a ser + # “atuais”, fazendo o índice único falhar no arranque e deixando o + # serviço indisponível atrás do nginx. Antes de criar o índice, mantemos + # só o documento mais recente como primário e movemos os restantes para + # histórico. + conn.execute(text(""" + WITH ranked AS ( + SELECT + id, + ROW_NUMBER() OVER ( + PARTITION BY opportunity_id, system, document_kind, COALESCE(role, 'current') + ORDER BY COALESCE(updated_at, created_at) DESC, created_at DESC, id DESC + ) AS rn + FROM commercial_documents + WHERE opportunity_id IS NOT NULL + AND COALESCE(is_primary, TRUE) = TRUE + AND COALESCE(role, 'current') IN ('current', 'accepted') + ) + UPDATE commercial_documents AS d + SET role = CASE + WHEN COALESCE(d.role, 'current') IN ('current', 'accepted') THEN 'historical' + ELSE COALESCE(d.role, 'historical') + END, + is_primary = FALSE, + is_active = FALSE, + updated_at = now() + FROM ranked AS r + WHERE d.id = r.id + AND r.rn > 1 + """)) + # v4927.2: não criar índice UNIQUE durante o arranque. + # Em produção podem existir duplicados históricos que ainda não foram + # reconciliados; se o índice único falhar, toda a app fica indisponível + # e o nginx devolve 502. A regra de “um principal por fase” fica + # aplicada pela normalização acima e pelos serviços que promovem um + # documento a atual. O índice de apoio é não único e seguro em bases + # reais com dados legados. + conn.execute(text("DROP INDEX IF EXISTS ux_commercial_documents_primary_role")) + conn.execute(text("CREATE INDEX IF NOT EXISTS idx_commercial_documents_primary_role ON commercial_documents(opportunity_id, system, document_kind, role) WHERE opportunity_id IS NOT NULL AND COALESCE(is_primary, TRUE) = TRUE AND COALESCE(role, 'current') IN ('current', 'accepted')")) + + # v4928.1: migração leve de registos antigos/reconstruídos. + # Não apaga documentos nem altera fases. Apenas evita que orçamentos e + # pró-formas legados continuem a aparecer como documento principal quando + # já existe fatura Jasmin atual/aceite para a mesma oportunidade. + conn.execute(text(""" + WITH primary_invoice AS ( + SELECT DISTINCT ON (opportunity_id, system) + id, opportunity_id, system, COALESCE(document_date, created_at::date) AS invoice_date, created_at + FROM commercial_documents + WHERE opportunity_id IS NOT NULL + AND system = 'jasmin' + AND document_kind = 'invoice' + AND COALESCE(is_primary, TRUE) = TRUE + AND COALESCE(role, 'current') IN ('current', 'accepted') + ORDER BY opportunity_id, system, COALESCE(document_date, created_at::date) DESC, created_at DESC + ) + UPDATE commercial_documents AS d + SET role = 'historical', + is_primary = FALSE, + is_active = FALSE, + payload = COALESCE(d.payload, '{}'::jsonb) || jsonb_build_object('legacy_reason', 'superseded_by_invoice_v4928_1'), + updated_at = now() + FROM primary_invoice AS inv + WHERE d.opportunity_id = inv.opportunity_id + AND d.system = inv.system + AND d.document_kind IN ('quotation', 'proforma') + AND COALESCE(d.role, 'current') IN ('current', 'accepted') + AND COALESCE(d.is_primary, TRUE) = TRUE + AND COALESCE(d.document_date, d.created_at::date) <= inv.invoice_date + """)) + + # Marca oportunidades reconstruídas com fatura e sem task pendente para a UI + # poder explicar que são processos antigos/auditoria, não um funil limpo. + conn.execute(text(""" + UPDATE opportunities AS o + SET metadata = COALESCE(o.metadata, '{}'::jsonb) || jsonb_build_object( + 'clientflow_record_mode', COALESCE(o.metadata->>'clientflow_record_mode', 'reconstructed_invoice_review'), + 'clientflow_legacy_migrated_by', 'v4928_1' + ), + updated_at = now() + WHERE EXISTS ( + SELECT 1 FROM commercial_documents d + WHERE d.opportunity_id = o.id + AND d.system = 'jasmin' + AND d.document_kind = 'invoice' + AND COALESCE(d.role, 'current') IN ('current', 'accepted') + AND COALESCE(d.is_primary, TRUE) = TRUE + ) + AND NOT EXISTS ( + SELECT 1 FROM tasks t + WHERE t.opportunity_id = o.id + AND t.status = 'pending' + ) + AND COALESCE(o.metadata->>'clientflow_record_mode', '') = '' + """)) + + conn.execute(text(""" + CREATE TABLE IF NOT EXISTS commercial_document_lines ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + document_id UUID NOT NULL REFERENCES commercial_documents(id) ON DELETE CASCADE, + opportunity_item_id UUID REFERENCES opportunity_items(id) ON DELETE SET NULL, + line_index INTEGER NOT NULL DEFAULT 0, + local_product_id UUID, + jasmin_sales_item TEXT, + description TEXT NOT NULL, + quantity NUMERIC(12,3) NOT NULL DEFAULT 1, + unit TEXT DEFAULT 'UN', + unit_price NUMERIC(12,2) NOT NULL DEFAULT 0, + tax_schema TEXT DEFAULT 'NORMAL', + total_amount NUMERIC(12,2), + payload JSONB NOT NULL DEFAULT '{}'::jsonb, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + """)) + conn.execute(text("CREATE INDEX IF NOT EXISTS idx_commercial_document_lines_document ON commercial_document_lines(document_id)")) + + conn.execute(text(""" + CREATE TABLE IF NOT EXISTS shipments ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + customer_id UUID REFERENCES customers(id) ON DELETE SET NULL, + opportunity_id UUID REFERENCES opportunities(id) ON DELETE SET NULL, + system TEXT NOT NULL DEFAULT 'packlink', + external_reference TEXT, + carrier TEXT, + service_id TEXT, + service_name TEXT, + status TEXT NOT NULL DEFAULT 'draft', + tracking_code TEXT, + tracking_url TEXT, + label_url TEXT, + price NUMERIC(12,2), + currency TEXT DEFAULT 'EUR', + payload JSONB NOT NULL DEFAULT '{}'::jsonb, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + """)) + conn.execute(text("CREATE INDEX IF NOT EXISTS idx_shipments_opportunity ON shipments(opportunity_id)")) + conn.execute(text("CREATE INDEX IF NOT EXISTS idx_shipments_external ON shipments(system, external_reference)")) + + + # Relação explícita oportunidade -> cliente local. Mantém o campo antigo + # opportunities.customer_id intacto para compatibilidade com Chatwoot/legado. + conn.execute(text("ALTER TABLE opportunities ADD COLUMN IF NOT EXISTS local_customer_id UUID REFERENCES customers(id) ON DELETE SET NULL")) + conn.execute(text("CREATE INDEX IF NOT EXISTS idx_opportunities_local_customer ON opportunities(local_customer_id)")) + + _SCHEMA_READY = True + + +def get_customer_by_tax_id(tax_id: str) -> Optional[Dict[str, Any]]: + ensure_commercial_schema() + normalized = normalize_tax_id(tax_id) + if not normalized: + return None + with engine.begin() as conn: + row = conn.execute(text(""" + SELECT id::text, name, tax_id, email, phone, street_name, postal_zone, + city_name, country, jasmin_customer_party_key, jasmin_customer_id, + metadata, created_at, updated_at + FROM customers + WHERE tax_id = :tax_id + LIMIT 1 + """), {"tax_id": normalized}).mappings().first() + return dict(row) if row else None + + +def get_customer(customer_id: str) -> Optional[Dict[str, Any]]: + ensure_commercial_schema() + with engine.begin() as conn: + row = conn.execute(text(""" + SELECT id::text, name, tax_id, email, phone, street_name, postal_zone, + city_name, country, jasmin_customer_party_key, jasmin_customer_id, + metadata, created_at, updated_at + FROM customers + WHERE id = CAST(:id AS UUID) + LIMIT 1 + """), {"id": customer_id}).mappings().first() + return dict(row) if row else None + + +def upsert_customer(data: Dict[str, Any]) -> Dict[str, Any]: + ensure_commercial_schema() + tax_id = normalize_tax_id(data.get("tax_id") or data.get("companyTaxID")) + name = _clean(data.get("name") or data.get("customerName") or data.get("customer_name")) + if not name: + raise ValueError("Nome do cliente é obrigatório") + params = { + "name": name, + "tax_id": tax_id or None, + "email": _clean(data.get("email") or data.get("electronicMail")) or None, + "phone": _clean(data.get("phone") or data.get("telephone")) or None, + "street_name": _clean(data.get("street_name") or data.get("streetName")) or None, + "postal_zone": _clean(data.get("postal_zone") or data.get("postalZone")) or None, + "city_name": _clean(data.get("city_name") or data.get("cityName")) or None, + "country": _clean(data.get("country") or "PT") or "PT", + "jasmin_customer_party_key": _clean(data.get("jasmin_customer_party_key") or data.get("customerPartyKey")) or None, + "jasmin_customer_id": _clean(data.get("jasmin_customer_id") or data.get("jasmin_id") or data.get("id")) or None, + "metadata": _json(data.get("metadata") or {}), + } + with engine.begin() as conn: + if params["tax_id"]: + row = conn.execute(text(""" + INSERT INTO customers ( + name, tax_id, email, phone, street_name, postal_zone, city_name, country, + jasmin_customer_party_key, jasmin_customer_id, metadata, updated_at + ) VALUES ( + :name, :tax_id, :email, :phone, :street_name, :postal_zone, :city_name, :country, + :jasmin_customer_party_key, :jasmin_customer_id, CAST(:metadata AS JSONB), now() + ) + ON CONFLICT (tax_id) WHERE tax_id IS NOT NULL AND tax_id <> '' + DO UPDATE SET + name = EXCLUDED.name, + email = COALESCE(EXCLUDED.email, customers.email), + phone = COALESCE(EXCLUDED.phone, customers.phone), + street_name = COALESCE(EXCLUDED.street_name, customers.street_name), + postal_zone = COALESCE(EXCLUDED.postal_zone, customers.postal_zone), + city_name = COALESCE(EXCLUDED.city_name, customers.city_name), + country = COALESCE(EXCLUDED.country, customers.country), + jasmin_customer_party_key = COALESCE(EXCLUDED.jasmin_customer_party_key, customers.jasmin_customer_party_key), + jasmin_customer_id = COALESCE(EXCLUDED.jasmin_customer_id, customers.jasmin_customer_id), + metadata = customers.metadata || EXCLUDED.metadata, + updated_at = now() + RETURNING id::text, name, tax_id, email, phone, street_name, postal_zone, city_name, + country, jasmin_customer_party_key, jasmin_customer_id, metadata, created_at, updated_at + """), params).mappings().first() + else: + normalized_name = normalize_fiscal_name(params["name"]) + row = conn.execute(text(""" + SELECT id::text, name, tax_id, email, phone, street_name, postal_zone, + city_name, country, jasmin_customer_party_key, jasmin_customer_id, + metadata, created_at, updated_at + FROM customers + WHERE COALESCE(tax_id, '') = '' + AND trim(regexp_replace(regexp_replace(lower(name), '[^0-9a-záàâãéêíóôõúç]+', ' ', 'g'), '\\s+', ' ', 'g')) = :normalized_name + LIMIT 1 + """), {"normalized_name": normalized_name}).mappings().first() + if row: + row = conn.execute(text(""" + UPDATE customers + SET email = COALESCE(:email, email), + phone = COALESCE(:phone, phone), + street_name = COALESCE(:street_name, street_name), + postal_zone = COALESCE(:postal_zone, postal_zone), + city_name = COALESCE(:city_name, city_name), + country = COALESCE(:country, country), + jasmin_customer_party_key = COALESCE(:jasmin_customer_party_key, jasmin_customer_party_key), + jasmin_customer_id = COALESCE(:jasmin_customer_id, jasmin_customer_id), + metadata = COALESCE(metadata, '{}'::jsonb) || CAST(:metadata AS JSONB), + updated_at = now() + WHERE id = CAST(:id AS UUID) + RETURNING id::text, name, tax_id, email, phone, street_name, postal_zone, + city_name, country, jasmin_customer_party_key, jasmin_customer_id, metadata, created_at, updated_at + """), {**params, "id": row["id"]}).mappings().first() + else: + row = conn.execute(text(""" + INSERT INTO customers ( + name, email, phone, street_name, postal_zone, city_name, country, + jasmin_customer_party_key, jasmin_customer_id, metadata, updated_at + ) VALUES ( + :name, :email, :phone, :street_name, :postal_zone, :city_name, :country, + :jasmin_customer_party_key, :jasmin_customer_id, CAST(:metadata AS JSONB), now() + ) + RETURNING id::text, name, tax_id, email, phone, street_name, postal_zone, city_name, + country, jasmin_customer_party_key, jasmin_customer_id, metadata, created_at, updated_at + """), params).mappings().first() + return dict(row or {}) + + +def _next_document_version(conn, opportunity_id: Optional[str], document_kind: str) -> int: + if not opportunity_id: + return 1 + value = conn.execute(text(""" + SELECT COALESCE(MAX(version_number), 0) + 1 + FROM commercial_documents + WHERE opportunity_id = CAST(:opportunity_id AS UUID) + AND document_kind = :document_kind + """), {"opportunity_id": opportunity_id, "document_kind": document_kind}).scalar() + return int(value or 1) + + +def create_commercial_document( + *, + document_kind: str, + customer_id: Optional[str] = None, + opportunity_id: Optional[str] = None, + system: str = "jasmin", + external_id: Optional[str] = None, + external_url: Optional[str] = None, + company: Optional[str] = None, + document_type: Optional[str] = None, + serie: Optional[str] = None, + series_number: Optional[int] = None, + document_number: Optional[str] = None, + customer_party_key: Optional[str] = None, + status: str = "created", + amount: Any = None, + tax_amount: Any = None, + total_amount: Any = None, + currency: str = "EUR", + parent_document_id: Optional[str] = None, + payload: Optional[Dict[str, Any]] = None, + document_date: Optional[str] = None, + due_date: Optional[str] = None, +) -> Dict[str, Any]: + ensure_commercial_schema() + document_kind = _clean(document_kind) + if not document_kind: + raise ValueError("document_kind é obrigatório") + with engine.begin() as conn: + version_number = _next_document_version(conn, opportunity_id, document_kind) + if document_kind == "quotation" and opportunity_id: + conn.execute(text(""" + UPDATE commercial_documents + SET status = CASE WHEN status IN ('created','sent','draft') THEN 'superseded' ELSE status END, + is_active = FALSE, + updated_at = now() + WHERE opportunity_id = CAST(:opportunity_id AS UUID) + AND document_kind = 'quotation' + AND is_active = TRUE + """), {"opportunity_id": opportunity_id}) + row = conn.execute(text(""" + INSERT INTO commercial_documents ( + customer_id, opportunity_id, system, document_kind, external_id, external_url, + company, document_type, serie, series_number, document_number, customer_party_key, + status, amount, tax_amount, total_amount, currency, parent_document_id, + version_number, is_active, document_date, due_date, payload, updated_at + ) VALUES ( + CAST(:customer_id AS UUID), CAST(:opportunity_id AS UUID), :system, :document_kind, + :external_id, :external_url, :company, :document_type, :serie, :series_number, + :document_number, :customer_party_key, :status, :amount, :tax_amount, :total_amount, + :currency, CAST(:parent_document_id AS UUID), :version_number, TRUE, + CAST(:document_date AS DATE), CAST(:due_date AS DATE), CAST(:payload AS JSONB), now() + ) + RETURNING id::text, customer_id::text, opportunity_id::text, system, document_kind, + external_id, external_url, company, document_type, serie, series_number, + document_number, customer_party_key, status, amount, tax_amount, total_amount, + currency, parent_document_id::text, version_number, is_active, document_date, + due_date, payload, created_at, updated_at + """), { + "customer_id": customer_id, + "opportunity_id": opportunity_id, + "system": system, + "document_kind": document_kind, + "external_id": external_id, + "external_url": external_url, + "company": company, + "document_type": document_type, + "serie": serie, + "series_number": series_number, + "document_number": document_number, + "customer_party_key": customer_party_key, + "status": status, + "amount": amount, + "tax_amount": tax_amount, + "total_amount": total_amount if total_amount is not None else amount, + "currency": currency, + "parent_document_id": parent_document_id, + "version_number": version_number, + "document_date": document_date, + "due_date": due_date, + "payload": _json(payload), + }).mappings().first() + return dict(row or {}) + + +def add_document_lines(document_id: str, lines: Iterable[Dict[str, Any]]) -> None: + ensure_commercial_schema() + with engine.begin() as conn: + for idx, line in enumerate(lines): + conn.execute(text(""" + INSERT INTO commercial_document_lines ( + document_id, opportunity_item_id, line_index, local_product_id, jasmin_sales_item, + description, quantity, unit, unit_price, tax_schema, total_amount, payload + ) VALUES ( + CAST(:document_id AS UUID), CAST(:opportunity_item_id AS UUID), :line_index, + CAST(:local_product_id AS UUID), :jasmin_sales_item, :description, :quantity, + :unit, :unit_price, :tax_schema, :total_amount, CAST(:payload AS JSONB) + ) + """), { + "document_id": document_id, + "opportunity_item_id": line.get("opportunity_item_id"), + "line_index": int(line.get("line_index") if line.get("line_index") is not None else idx), + "local_product_id": line.get("local_product_id"), + "jasmin_sales_item": line.get("jasmin_sales_item"), + "description": line.get("description") or "Linha", + "quantity": line.get("quantity") or 1, + "unit": line.get("unit") or "UN", + "unit_price": line.get("unit_price") or 0, + "tax_schema": line.get("tax_schema") or "NORMAL", + "total_amount": line.get("total_amount"), + "payload": _json(line.get("payload") or {}), + }) + + +def list_commercial_documents(opportunity_id: Optional[str] = None, customer_id: Optional[str] = None, limit: int = 100) -> List[Dict[str, Any]]: + ensure_commercial_schema() + filters = [] + params: Dict[str, Any] = {"limit": int(limit)} + if opportunity_id: + filters.append("cd.opportunity_id = CAST(:opportunity_id AS UUID)") + params["opportunity_id"] = opportunity_id + if customer_id: + filters.append("cd.customer_id = CAST(:customer_id AS UUID)") + params["customer_id"] = customer_id + where = "WHERE " + " AND ".join(filters) if filters else "" + with engine.begin() as conn: + rows = conn.execute(text(f""" + SELECT cd.id::text, cd.customer_id::text, cd.opportunity_id::text, cd.system, + cd.document_kind, cd.external_id, cd.external_url, cd.company, cd.document_type, + cd.serie, cd.series_number, cd.document_number, cd.customer_party_key, + cd.status, cd.amount, cd.tax_amount, cd.total_amount, cd.currency, + cd.parent_document_id::text, cd.version_number, cd.role, cd.is_primary, cd.is_active, + cd.document_date, cd.due_date, cd.payload, cd.created_at, cd.updated_at, + c.name AS customer_name, c.tax_id AS customer_tax_id + FROM commercial_documents cd + LEFT JOIN customers c ON c.id = cd.customer_id + {where} + ORDER BY + CASE COALESCE(cd.role, 'current') WHEN 'current' THEN 0 WHEN 'accepted' THEN 1 WHEN 'related' THEN 2 WHEN 'historical' THEN 3 ELSE 4 END, + COALESCE(cd.is_primary, FALSE) DESC, + cd.created_at DESC + LIMIT :limit + """), params).mappings().all() + return [dict(r) for r in rows] + + + +def get_commercial_document(document_id: str) -> Optional[Dict[str, Any]]: + ensure_commercial_schema() + with engine.begin() as conn: + row = conn.execute(text(""" + SELECT cd.id::text, cd.customer_id::text, cd.opportunity_id::text, cd.system, + cd.document_kind, cd.external_id, cd.external_url, cd.company, cd.document_type, + cd.serie, cd.series_number, cd.document_number, cd.customer_party_key, + cd.status, cd.amount, cd.tax_amount, cd.total_amount, cd.currency, + cd.parent_document_id::text, cd.version_number, cd.role, cd.is_primary, cd.is_active, + cd.document_date, cd.due_date, cd.payload, cd.created_at, cd.updated_at, + c.name AS customer_name, c.tax_id AS customer_tax_id + FROM commercial_documents cd + LEFT JOIN customers c ON c.id = cd.customer_id + WHERE cd.id = CAST(:id AS UUID) + LIMIT 1 + """), {"id": document_id}).mappings().first() + return dict(row) if row else None + +def get_latest_active_quotation(opportunity_id: str) -> Optional[Dict[str, Any]]: + ensure_commercial_schema() + with engine.begin() as conn: + row = conn.execute(text(""" + SELECT id::text, customer_id::text, opportunity_id::text, system, document_kind, + external_id, external_url, company, document_type, serie, series_number, + document_number, customer_party_key, status, amount, total_amount, currency, + parent_document_id::text, version_number, is_active, payload, created_at, updated_at + FROM commercial_documents + WHERE opportunity_id = CAST(:opportunity_id AS UUID) + AND document_kind = 'quotation' + AND status NOT IN ('cancelled','failed') + ORDER BY is_active DESC, created_at DESC + LIMIT 1 + """), {"opportunity_id": opportunity_id}).mappings().first() + return dict(row) if row else None + + +def find_invoice_for_parent(parent_document_id: str) -> Optional[Dict[str, Any]]: + ensure_commercial_schema() + with engine.begin() as conn: + row = conn.execute(text(""" + SELECT id::text, external_id, document_number, status, created_at + FROM commercial_documents + WHERE parent_document_id = CAST(:parent_document_id AS UUID) + AND document_kind = 'invoice' + ORDER BY created_at DESC + LIMIT 1 + """), {"parent_document_id": parent_document_id}).mappings().first() + return dict(row) if row else None + + +def mark_document_status(document_id: str, status: str, payload: Optional[Dict[str, Any]] = None) -> None: + ensure_commercial_schema() + with engine.begin() as conn: + conn.execute(text(""" + UPDATE commercial_documents + SET status = :status, + payload = payload || CAST(:payload AS JSONB), + updated_at = now() + WHERE id = CAST(:id AS UUID) + """), {"id": document_id, "status": status, "payload": _json(payload or {})}) + + +def update_commercial_document_details(document_id: str, details: Dict[str, Any]) -> Optional[Dict[str, Any]]: + """Atualiza campos normalizados depois de consultar o documento no sistema externo. + + Usado sobretudo para Jasmin, porque o POST devolve apenas o UUID e o + número/série precisam de um GET subsequente. Campos ausentes não apagam + dados já existentes. + """ + ensure_commercial_schema() + payload = details.get("payload") or {} + with engine.begin() as conn: + row = conn.execute(text(""" + UPDATE commercial_documents + SET company = COALESCE(:company, company), + document_type = COALESCE(:document_type, document_type), + serie = COALESCE(:serie, serie), + series_number = COALESCE(:series_number, series_number), + document_number = COALESCE(:document_number, document_number), + amount = COALESCE(:amount, amount), + tax_amount = COALESCE(:tax_amount, tax_amount), + total_amount = COALESCE(:total_amount, total_amount), + currency = COALESCE(:currency, currency), + document_date = COALESCE(CAST(:document_date AS DATE), document_date), + due_date = COALESCE(CAST(:due_date AS DATE), due_date), + payload = payload || CAST(:payload AS JSONB), + updated_at = now() + WHERE id = CAST(:id AS UUID) + RETURNING id::text, customer_id::text, opportunity_id::text, system, document_kind, + external_id, external_url, company, document_type, serie, series_number, + document_number, customer_party_key, status, amount, tax_amount, total_amount, + currency, parent_document_id::text, version_number, is_active, document_date, + due_date, payload, created_at, updated_at + """), { + "id": document_id, + "company": details.get("company"), + "document_type": details.get("document_type"), + "serie": details.get("serie"), + "series_number": details.get("series_number"), + "document_number": details.get("document_number"), + "amount": details.get("amount"), + "tax_amount": details.get("tax_amount"), + "total_amount": details.get("total_amount"), + "currency": details.get("currency"), + "document_date": details.get("document_date"), + "due_date": details.get("due_date"), + "payload": _json(payload), + }).mappings().first() + return dict(row) if row else None + + +def upsert_shipment_record(data: Dict[str, Any]) -> Dict[str, Any]: + ensure_commercial_schema() + params = { + "customer_id": data.get("customer_id"), + "opportunity_id": data.get("opportunity_id"), + "system": data.get("system") or "packlink", + "external_reference": data.get("external_reference") or data.get("reference"), + "carrier": data.get("carrier"), + "service_id": data.get("service_id"), + "service_name": data.get("service_name"), + "status": data.get("status") or "created", + "tracking_code": data.get("tracking_code"), + "tracking_url": data.get("tracking_url"), + "label_url": data.get("label_url"), + "price": data.get("price"), + "currency": data.get("currency") or "EUR", + "payload": _json(data.get("payload") or {}), + } + with engine.begin() as conn: + row = conn.execute(text(""" + INSERT INTO shipments ( + customer_id, opportunity_id, system, external_reference, carrier, service_id, + service_name, status, tracking_code, tracking_url, label_url, price, currency, + payload, updated_at + ) VALUES ( + CAST(:customer_id AS UUID), CAST(:opportunity_id AS UUID), :system, + :external_reference, :carrier, :service_id, :service_name, :status, + :tracking_code, :tracking_url, :label_url, :price, :currency, + CAST(:payload AS JSONB), now() + ) + RETURNING id::text, customer_id::text, opportunity_id::text, system, external_reference, + carrier, service_id, service_name, status, tracking_code, tracking_url, + label_url, price, currency, payload, created_at, updated_at + """), params).mappings().first() + return dict(row or {}) + +def list_customers(q: Optional[str] = None, limit: int = 200) -> List[Dict[str, Any]]: + """Lista clientes locais normalizados. + + A página Customers passa a ser a fonte principal dos dados fiscais. + """ + ensure_commercial_schema() + params: Dict[str, Any] = {"limit": int(limit)} + where = "" + if q: + params["q"] = f"%{str(q).strip()}%" + where = """ + WHERE name ILIKE :q + OR COALESCE(tax_id, '') ILIKE :q + OR COALESCE(email, '') ILIKE :q + OR COALESCE(phone, '') ILIKE :q + OR COALESCE(jasmin_customer_party_key, '') ILIKE :q + """ + with engine.begin() as conn: + rows = conn.execute(text(f""" + SELECT c.id::text, c.name, c.tax_id, c.email, c.phone, c.street_name, + c.postal_zone, c.city_name, c.country, c.jasmin_customer_party_key, + c.jasmin_customer_id, c.metadata, c.created_at, c.updated_at, + COUNT(DISTINCT o.id) AS opportunity_count, + COUNT(DISTINCT cd.id) FILTER (WHERE cd.document_kind = 'quotation') AS quotation_count, + COUNT(DISTINCT cd.id) FILTER (WHERE cd.document_kind = 'invoice') AS invoice_count + FROM customers c + LEFT JOIN opportunities o ON COALESCE(o.local_customer_id::text, '') = c.id::text + LEFT JOIN commercial_documents cd ON cd.customer_id = c.id + {where} + GROUP BY c.id + ORDER BY c.updated_at DESC, c.created_at DESC + LIMIT :limit + """), params).mappings().all() + return [dict(r) for r in rows] + + +def update_customer(customer_id: str, data: Dict[str, Any]) -> Dict[str, Any]: + """Atualiza uma ficha de cliente local. + + Não chama Jasmin diretamente; a sincronização é feita por jasmin_service/outbox. + """ + ensure_commercial_schema() + params = { + "id": customer_id, + "name": _clean(data.get("name")), + "tax_id": normalize_tax_id(data.get("tax_id")), + "email": _clean(data.get("email")) or None, + "phone": _clean(data.get("phone")) or None, + "street_name": _clean(data.get("street_name")) or None, + "postal_zone": _clean(data.get("postal_zone")) or None, + "city_name": _clean(data.get("city_name")) or None, + "country": _clean(data.get("country") or "PT") or "PT", + "jasmin_customer_party_key": _clean(data.get("jasmin_customer_party_key")) or None, + "jasmin_customer_id": _clean(data.get("jasmin_customer_id")) or None, + "metadata": _json(data.get("metadata") or {}), + } + if not params["name"]: + raise ValueError("Nome do cliente é obrigatório") + with engine.begin() as conn: + if params["tax_id"]: + duplicate = conn.execute(text(""" + SELECT id::text, name, tax_id, email, phone, street_name, postal_zone, city_name, country, + jasmin_customer_party_key, jasmin_customer_id, metadata, created_at, updated_at + FROM customers + WHERE tax_id = :tax_id + AND id <> CAST(:id AS UUID) + LIMIT 1 + """), {"tax_id": params["tax_id"], "id": params["id"]}).mappings().first() + if duplicate: + raise DuplicateCustomerTaxIdError(params["tax_id"], dict(duplicate)) + row = conn.execute(text(""" + UPDATE customers + SET name = :name, + tax_id = NULLIF(:tax_id, ''), + email = :email, + phone = :phone, + street_name = :street_name, + postal_zone = :postal_zone, + city_name = :city_name, + country = :country, + jasmin_customer_party_key = :jasmin_customer_party_key, + jasmin_customer_id = :jasmin_customer_id, + metadata = COALESCE(metadata, '{}'::jsonb) || CAST(:metadata AS JSONB), + updated_at = now() + WHERE id = CAST(:id AS UUID) + RETURNING id::text, name, tax_id, email, phone, street_name, postal_zone, + city_name, country, jasmin_customer_party_key, jasmin_customer_id, + metadata, created_at, updated_at + """), params).mappings().first() + if not row: + raise ValueError("Cliente não encontrado") + return dict(row) + + +def link_customer_to_opportunity(customer_id: str, opportunity_id: str) -> None: + """Liga um cliente local a uma oportunidade sem duplicar dados fiscais.""" + ensure_commercial_schema() + with engine.begin() as conn: + conn.execute(text(""" + UPDATE opportunities + SET local_customer_id = CAST(:customer_id AS UUID), + updated_at = now() + WHERE id = CAST(:opportunity_id AS UUID) + """), {"customer_id": customer_id, "opportunity_id": opportunity_id}) + + +def unlink_customer_from_opportunity(opportunity_id: str) -> None: + ensure_commercial_schema() + with engine.begin() as conn: + conn.execute(text(""" + UPDATE opportunities + SET local_customer_id = NULL, + updated_at = now() + WHERE id = CAST(:opportunity_id AS UUID) + """), {"opportunity_id": opportunity_id}) + + +def get_customer_for_opportunity(opportunity_id: str) -> Optional[Dict[str, Any]]: + ensure_commercial_schema() + with engine.begin() as conn: + row = conn.execute(text(""" + SELECT c.id::text, c.name, c.tax_id, c.email, c.phone, c.street_name, + c.postal_zone, c.city_name, c.country, c.jasmin_customer_party_key, + c.jasmin_customer_id, c.metadata, c.created_at, c.updated_at + FROM opportunities o + JOIN customers c ON c.id = o.local_customer_id + WHERE o.id = CAST(:opportunity_id AS UUID) + LIMIT 1 + """), {"opportunity_id": opportunity_id}).mappings().first() + return dict(row) if row else None + + +def list_opportunities_for_customer(customer_id: str, limit: int = 50) -> List[Dict[str, Any]]: + ensure_commercial_schema() + with engine.begin() as conn: + rows = conn.execute(text(""" + SELECT id::text, title, stage, status, customer_name, customer_email, + product_interest, value_amount, currency, updated_at, created_at + FROM opportunities + WHERE local_customer_id = CAST(:customer_id AS UUID) + ORDER BY updated_at DESC + LIMIT :limit + """), {"customer_id": customer_id, "limit": int(limit)}).mappings().all() + return [dict(r) for r in rows] + + +def list_shipments(opportunity_id: Optional[str] = None, customer_id: Optional[str] = None, limit: int = 100) -> List[Dict[str, Any]]: + ensure_commercial_schema() + filters = [] + params: Dict[str, Any] = {"limit": int(limit)} + if opportunity_id: + filters.append("s.opportunity_id = CAST(:opportunity_id AS UUID)") + params["opportunity_id"] = opportunity_id + if customer_id: + filters.append("s.customer_id = CAST(:customer_id AS UUID)") + params["customer_id"] = customer_id + where = "WHERE " + " AND ".join(filters) if filters else "" + with engine.begin() as conn: + rows = conn.execute(text(f""" + SELECT s.id::text, s.customer_id::text, s.opportunity_id::text, s.system, + s.external_reference, s.carrier, s.service_id, s.service_name, + s.status, s.tracking_code, s.tracking_url, s.label_url, s.price, + s.currency, s.payload, s.created_at, s.updated_at, + c.name AS customer_name + FROM shipments s + LEFT JOIN customers c ON c.id = s.customer_id + {where} + ORDER BY s.created_at DESC + LIMIT :limit + """), params).mappings().all() + return [dict(r) for r in rows] diff --git a/app/communication_service.py b/app/communication_service.py new file mode 100644 index 0000000..a6491d3 --- /dev/null +++ b/app/communication_service.py @@ -0,0 +1,322 @@ +"""Communication/inbox-classification helpers for ClientFlow v4.5. + +A communication is the durable record of an inbound/outbound email, Chatwoot +message or other customer message. Tasks and outbox items are the actions that +come from it; this module deliberately keeps the original communication separate +from human work and automation work. +""" +from __future__ import annotations + +import json +from typing import Any, Dict, List, Optional + +from sqlalchemy import text + +from app.db import engine + + +COMMUNICATION_STATUSES = { + "new", + "classified", + "needs_review", + "linked", + "task_created", + "done", + "ignored", +} + + +ACTION_CLASSIFICATION_MAP: dict[str, tuple[str, str, str]] = { + "pedido_orcamento": ("Comercial", "Criar/validar oportunidade", "cf-chip-blue"), + "pedido_informacao": ("Comercial", "Responder pedido de informação", "cf-chip-blue"), + "aceitacao_orcamento": ("Financeiro", "Converter em fatura/pró-forma", "cf-chip-green"), + "comprovativo_pagamento": ("Financeiro", "Confirmar pagamento", "cf-chip-green"), + "pedido_fatura": ("Financeiro", "Emitir/enviar fatura", "cf-chip-green"), + "dados_fiscais": ("Financeiro", "Atualizar dados fiscais", "cf-chip-green"), + "pedido_tracking": ("Operações", "Verificar envio/tracking", "cf-chip-purple"), + "reclamacao": ("Suporte", "Responder reclamação", "cf-chip-orange"), + "pedido_remocao_lista": ("Marketing", "Remover contacto da lista", "cf-chip-gray"), +} + + +def ensure_communication_schema() -> None: + """Create/upgrade v4.5 communications and timeline tables. + + The statements are additive to be safe on field deployments. + """ + with engine.begin() as conn: + conn.execute(text("CREATE EXTENSION IF NOT EXISTS pgcrypto")) + + conn.execute(text(""" + CREATE TABLE IF NOT EXISTS communications ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + source_system TEXT NOT NULL DEFAULT 'email', + source_message_id TEXT, + thread_id TEXT, + conversation_id TEXT, + contact_id TEXT, + direction TEXT NOT NULL DEFAULT 'inbound', + sender_name TEXT, + sender_email TEXT, + recipient TEXT, + subject TEXT, + body TEXT, + classification TEXT, + confidence NUMERIC(4,3), + status TEXT NOT NULL DEFAULT 'new', + customer_id UUID, + opportunity_id UUID, + task_id UUID, + metadata JSONB NOT NULL DEFAULT '{}'::jsonb, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + """)) + + for statement in [ + "ALTER TABLE communications ADD COLUMN IF NOT EXISTS source_system TEXT NOT NULL DEFAULT 'email'", + "ALTER TABLE communications ADD COLUMN IF NOT EXISTS source_message_id TEXT", + "ALTER TABLE communications ADD COLUMN IF NOT EXISTS thread_id TEXT", + "ALTER TABLE communications ADD COLUMN IF NOT EXISTS conversation_id TEXT", + "ALTER TABLE communications ADD COLUMN IF NOT EXISTS contact_id TEXT", + "ALTER TABLE communications ADD COLUMN IF NOT EXISTS direction TEXT NOT NULL DEFAULT 'inbound'", + "ALTER TABLE communications ADD COLUMN IF NOT EXISTS sender_name TEXT", + "ALTER TABLE communications ADD COLUMN IF NOT EXISTS sender_email TEXT", + "ALTER TABLE communications ADD COLUMN IF NOT EXISTS recipient TEXT", + "ALTER TABLE communications ADD COLUMN IF NOT EXISTS subject TEXT", + "ALTER TABLE communications ADD COLUMN IF NOT EXISTS body TEXT", + "ALTER TABLE communications ADD COLUMN IF NOT EXISTS classification TEXT", + "ALTER TABLE communications ADD COLUMN IF NOT EXISTS confidence NUMERIC(4,3)", + "ALTER TABLE communications ADD COLUMN IF NOT EXISTS status TEXT NOT NULL DEFAULT 'new'", + "ALTER TABLE communications ADD COLUMN IF NOT EXISTS customer_id UUID", + "ALTER TABLE communications ADD COLUMN IF NOT EXISTS opportunity_id UUID", + "ALTER TABLE communications ADD COLUMN IF NOT EXISTS task_id UUID", + "ALTER TABLE communications ADD COLUMN IF NOT EXISTS metadata JSONB NOT NULL DEFAULT '{}'::jsonb", + "ALTER TABLE communications ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ NOT NULL DEFAULT now()", + "ALTER TABLE communications ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ NOT NULL DEFAULT now()", + ]: + conn.execute(text(statement)) + + conn.execute(text(""" + CREATE UNIQUE INDEX IF NOT EXISTS ux_communications_source_message + ON communications(source_system, source_message_id) + WHERE source_message_id IS NOT NULL + """)) + conn.execute(text("CREATE INDEX IF NOT EXISTS idx_communications_created ON communications(created_at DESC)")) + conn.execute(text("CREATE INDEX IF NOT EXISTS idx_communications_status ON communications(status)")) + conn.execute(text("CREATE INDEX IF NOT EXISTS idx_communications_classification ON communications(classification)")) + conn.execute(text("CREATE INDEX IF NOT EXISTS idx_communications_sender_email ON communications(sender_email)")) + conn.execute(text("CREATE INDEX IF NOT EXISTS idx_communications_customer ON communications(customer_id)")) + conn.execute(text("CREATE INDEX IF NOT EXISTS idx_communications_opportunity ON communications(opportunity_id)")) + + # v4.5 task context columns. Existing code still uses action/route; these + # columns let the UI connect a task back to the communication/document/etc. + for statement in [ + "ALTER TABLE tasks ADD COLUMN IF NOT EXISTS communication_id UUID", + "ALTER TABLE tasks ADD COLUMN IF NOT EXISTS document_id UUID", + "ALTER TABLE tasks ADD COLUMN IF NOT EXISTS shipment_id UUID", + "ALTER TABLE tasks ADD COLUMN IF NOT EXISTS outbox_id UUID", + "ALTER TABLE tasks ADD COLUMN IF NOT EXISTS priority TEXT NOT NULL DEFAULT 'normal'", + "ALTER TABLE tasks ADD COLUMN IF NOT EXISTS assigned_to TEXT", + ]: + conn.execute(text(statement)) + + conn.execute(text("CREATE INDEX IF NOT EXISTS idx_tasks_communication ON tasks(communication_id)")) + conn.execute(text("CREATE INDEX IF NOT EXISTS idx_tasks_document ON tasks(document_id)")) + conn.execute(text("CREATE INDEX IF NOT EXISTS idx_tasks_shipment ON tasks(shipment_id)")) + conn.execute(text("CREATE INDEX IF NOT EXISTS idx_tasks_outbox ON tasks(outbox_id)")) + conn.execute(text("CREATE INDEX IF NOT EXISTS idx_tasks_priority ON tasks(priority)")) + conn.execute(text("CREATE INDEX IF NOT EXISTS idx_tasks_due_status ON tasks(status, due_at)")) + + conn.execute(text(""" + CREATE TABLE IF NOT EXISTS timeline_events ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + opportunity_id UUID, + customer_id UUID, + event_type TEXT NOT NULL, + title TEXT NOT NULL, + description TEXT, + source TEXT NOT NULL DEFAULT 'clientflow', + related_type TEXT, + related_id TEXT, + payload JSONB NOT NULL DEFAULT '{}'::jsonb, + created_by TEXT NOT NULL DEFAULT 'system', + created_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + """)) + conn.execute(text("CREATE INDEX IF NOT EXISTS idx_timeline_opportunity ON timeline_events(opportunity_id, created_at DESC)")) + conn.execute(text("CREATE INDEX IF NOT EXISTS idx_timeline_customer ON timeline_events(customer_id, created_at DESC)")) + conn.execute(text("CREATE INDEX IF NOT EXISTS idx_timeline_related ON timeline_events(related_type, related_id)")) + + +def _row_dict(row: Any) -> Dict[str, Any]: + return dict(row) if row is not None else {} + + +def normalize_status(status: Optional[str]) -> Optional[str]: + if not status: + return None + status = status.strip().lower() + return status if status in COMMUNICATION_STATUSES else None + + +def list_communications( + *, + status: Optional[str] = None, + classification: Optional[str] = None, + q: Optional[str] = None, + opportunity_id: Optional[str] = None, + limit: int = 50, +) -> List[Dict[str, Any]]: + """List classified inbox items with optional filters.""" + filters = [] + params: Dict[str, Any] = {"limit": int(limit)} + if normalize_status(status): + filters.append("c.status = :status") + params["status"] = normalize_status(status) + if classification: + filters.append("c.classification = :classification") + params["classification"] = classification + if opportunity_id: + filters.append("c.opportunity_id = CAST(:opportunity_id AS UUID)") + params["opportunity_id"] = opportunity_id + if q: + filters.append("(c.sender_email ILIKE :q OR c.sender_name ILIKE :q OR c.subject ILIKE :q OR c.body ILIKE :q OR c.classification ILIKE :q)") + params["q"] = f"%{q}%" + where = "WHERE " + " AND ".join(filters) if filters else "" + + with engine.begin() as conn: + rows = conn.execute(text(f""" + SELECT c.id::text, c.source_system, c.source_message_id, c.thread_id, + c.conversation_id, c.contact_id, c.direction, c.sender_name, + c.sender_email, c.recipient, c.subject, c.body, c.classification, + c.confidence, c.status, c.customer_id::text, c.opportunity_id::text, + c.task_id::text, c.metadata, c.created_at, c.updated_at, + cu.name AS customer_name, o.title AS opportunity_title + FROM communications c + LEFT JOIN customers cu ON cu.id = c.customer_id + LEFT JOIN opportunities o ON o.id = c.opportunity_id + {where} + ORDER BY c.created_at DESC + LIMIT :limit + """), params).mappings().all() + return [dict(row) for row in rows] + + +def list_communications_for_opportunity(opportunity_id: str, limit: int = 20) -> List[Dict[str, Any]]: + return list_communications(opportunity_id=opportunity_id, limit=limit) + + +def get_communication(communication_id: str) -> Optional[Dict[str, Any]]: + with engine.begin() as conn: + row = conn.execute(text(""" + SELECT c.id::text, c.source_system, c.source_message_id, c.thread_id, + c.conversation_id, c.contact_id, c.direction, c.sender_name, + c.sender_email, c.recipient, c.subject, c.body, c.classification, + c.confidence, c.status, c.customer_id::text, c.opportunity_id::text, + c.task_id::text, c.metadata, c.created_at, c.updated_at, + cu.name AS customer_name, o.title AS opportunity_title + FROM communications c + LEFT JOIN customers cu ON cu.id = c.customer_id + LEFT JOIN opportunities o ON o.id = c.opportunity_id + WHERE c.id = CAST(:id AS UUID) + """), {"id": communication_id}).mappings().first() + return dict(row) if row else None + + +def get_communications_summary() -> Dict[str, int]: + with engine.begin() as conn: + row = conn.execute(text(""" + SELECT + COUNT(*)::int AS total, + COUNT(*) FILTER (WHERE status IN ('new','classified','needs_review'))::int AS open, + COUNT(*) FILTER (WHERE status = 'needs_review')::int AS needs_review, + COUNT(*) FILTER (WHERE customer_id IS NULL AND direction = 'inbound')::int AS without_customer, + COUNT(*) FILTER (WHERE opportunity_id IS NULL AND direction = 'inbound')::int AS without_opportunity, + COUNT(*) FILTER (WHERE task_id IS NOT NULL)::int AS with_task, + COUNT(*) FILTER (WHERE created_at >= now() - interval '24 hours')::int AS last_24h + FROM communications + """)).mappings().first() + return {k: int(v or 0) for k, v in dict(row or {}).items()} + + +def set_communication_status(communication_id: str, status: str) -> None: + status = normalize_status(status) + if not status: + raise ValueError("Estado de comunicação inválido.") + with engine.begin() as conn: + conn.execute(text(""" + UPDATE communications + SET status = :status, updated_at = now() + WHERE id = CAST(:id AS UUID) + """), {"id": communication_id, "status": status}) + + +def link_communication_to_customer(communication_id: str, customer_id: Optional[str]) -> None: + with engine.begin() as conn: + conn.execute(text(""" + UPDATE communications + SET customer_id = CASE WHEN :customer_id = '' THEN NULL ELSE CAST(:customer_id AS UUID) END, + status = CASE WHEN status IN ('new','classified','needs_review') THEN 'linked' ELSE status END, + updated_at = now() + WHERE id = CAST(:id AS UUID) + """), {"id": communication_id, "customer_id": customer_id or ""}) + + +def link_communication_to_opportunity(communication_id: str, opportunity_id: Optional[str]) -> None: + with engine.begin() as conn: + conn.execute(text(""" + UPDATE communications + SET opportunity_id = CASE WHEN :opportunity_id = '' THEN NULL ELSE CAST(:opportunity_id AS UUID) END, + status = CASE WHEN status IN ('new','classified','needs_review') THEN 'linked' ELSE status END, + updated_at = now() + WHERE id = CAST(:id AS UUID) + """), {"id": communication_id, "opportunity_id": opportunity_id or ""}) + + +def create_timeline_event( + *, + opportunity_id: Optional[str] = None, + customer_id: Optional[str] = None, + event_type: str, + title: str, + description: str = "", + source: str = "clientflow", + related_type: Optional[str] = None, + related_id: Optional[str] = None, + payload: Optional[Dict[str, Any]] = None, + created_by: str = "system", +) -> Optional[str]: + if not opportunity_id and not customer_id: + return None + with engine.begin() as conn: + row = conn.execute(text(""" + INSERT INTO timeline_events ( + opportunity_id, customer_id, event_type, title, description, + source, related_type, related_id, payload, created_by + ) VALUES ( + CASE WHEN :opportunity_id = '' THEN NULL ELSE CAST(:opportunity_id AS UUID) END, + CASE WHEN :customer_id = '' THEN NULL ELSE CAST(:customer_id AS UUID) END, + :event_type, :title, :description, :source, :related_type, + :related_id, CAST(:payload AS JSONB), :created_by + ) + RETURNING id::text + """), { + "opportunity_id": opportunity_id or "", + "customer_id": customer_id or "", + "event_type": event_type, + "title": title, + "description": description, + "source": source, + "related_type": related_type, + "related_id": related_id, + "payload": json.dumps(payload or {}, ensure_ascii=False), + "created_by": created_by, + }).first() + return str(row[0]) if row else None + + +def classification_action(classification: Optional[str]) -> Dict[str, str]: + key = (classification or "").strip().lower() + queue, action, chip = ACTION_CLASSIFICATION_MAP.get(key, ("Revisão", "Rever classificação", "cf-chip-orange")) + return {"queue": queue, "action": action, "chip": chip} diff --git a/app/config.py b/app/config.py new file mode 100644 index 0000000..1c1efec --- /dev/null +++ b/app/config.py @@ -0,0 +1,107 @@ +from pydantic_settings import BaseSettings, SettingsConfigDict + + +class Settings(BaseSettings): + app_name: str = "ClientFlow" + env: str = "dev" + + openrouter_api_key: str + openrouter_model: str = "qwen/qwen3-30b-a3b" + openrouter_url: str = "https://openrouter.ai/api/v1/chat/completions" + + # ClientFlow requer PostgreSQL. SQLite não suporta JSONB, UUID, + # TIMESTAMPTZ nem os índices usados pelo schema core. + database_url: str + clientflow_persist: bool = True + + clientflow_webhook_secret: str = "" + clientflow_admin_token: str = "" + + # UI warning for cases where the original opportunity contact and the + # linked fiscal customer look different. Disabled by default because many + # legitimate contacts use abbreviated names or personal contacts for a + # company/fiscal customer. Enable only if this warning proves useful. + clientflow_customer_mismatch_warning_enabled: bool = False + + chatwoot_write_enabled: bool = False + chatwoot_base_url: str = "" + chatwoot_public_url: str = "" + chatwoot_account_id: str = "" + chatwoot_api_token: str = "" + + # External operational systems + odoo_enabled: bool = False + odoo_base_url: str = "" + odoo_public_url: str = "" + odoo_db: str = "" + odoo_username: str = "" + odoo_api_key: str = "" + odoo_api_mode: str = "xmlrpc" + + jasmin_enabled: bool = False + jasmin_base_url: str = "" + jasmin_public_url: str = "" + jasmin_account: str = "" + jasmin_subscription: str = "" + jasmin_client_id: str = "" + jasmin_client_secret: str = "" + jasmin_token_url: str = "https://identity.primaverabss.com/connect/token" + jasmin_scope: str = "application" + jasmin_company_key: str = "" + jasmin_quotation_type: str = "ORC" + jasmin_quotation_serie: str = "" + jasmin_default_price_list: str = "03" + jasmin_default_payment_method: str = "TRA" + jasmin_default_payment_term: str = "00" + jasmin_default_delivery_term: str = "TRANSP" + jasmin_default_currency: str = "EUR" + jasmin_default_country: str = "PT" + jasmin_default_customer_group: str = "02" + jasmin_default_party_tax_schema: str = "CONTINENTE" + jasmin_default_unit: str = "UN" + jasmin_default_item_tax_schema: str = "NORMAL" + jasmin_default_sales_item: str = "" + + packlink_enabled: bool = False + packlink_base_url: str = "" + packlink_public_url: str = "" + packlink_api_key: str = "" + packlink_default_service_id: str = "20571" + packlink_default_service: str = "Paq 24" + packlink_default_carrier: str = "Correos Express" + packlink_source: str = "PRO" + packlink_platform: str = "PRO" + packlink_platform_country: str = "UN" + + # External company/contact lookup API used by the fiscal enrichment worker. + # Example base URL: http://127.0.0.1:8000 from informa_pipeline_api. + external_company_lookup_enabled: bool = False + external_company_lookup_base_url: str = "" + external_company_lookup_api_key: str = "" + external_company_lookup_timeout: int = 10 + external_company_lookup_auto_threshold: float = 95.0 + + # LLM-assisted identity extraction from email body/signature. + email_identity_extraction_enabled: bool = True + email_identity_extraction_use_llm: bool = True + email_identity_llm_model: str = "" + email_identity_llm_fallback_model: str = "" + email_identity_llm_timeout_seconds: int = 20 + email_identity_llm_max_body_chars: int = 3500 + + max_tokens: int = 2200 + temperature: float = 0.0 + + model_config = SettingsConfigDict( + env_file=".env", + env_prefix="", + extra="ignore", + ) + + +settings = Settings() + +if settings.database_url.strip().lower().startswith("sqlite"): + raise RuntimeError( + "ClientFlow requer PostgreSQL. Defina DATABASE_URL com postgresql+psycopg://..." + ) diff --git a/app/customers/__init__.py b/app/customers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/db.py b/app/db.py new file mode 100644 index 0000000..adf6a57 --- /dev/null +++ b/app/db.py @@ -0,0 +1,324 @@ +from sqlalchemy import create_engine, text +from sqlalchemy.orm import sessionmaker + +from app.config import settings + + +engine = create_engine(settings.database_url, pool_pre_ping=True) + +SessionLocal = sessionmaker( + autocommit=False, + autoflush=False, + bind=engine, +) + + +_CORE_SCHEMA_READY = False + + +def ensure_core_schema() -> None: + """Cria o schema base do ClientFlow numa base PostgreSQL vazia. + + As versões anteriores assumiam que `tasks`, `messages`, `raw_events` e + `action_runs` já existiam. Numa instalação limpa isso fazia o arranque + falhar quando a camada de oportunidades tentava executar: + + ALTER TABLE tasks ADD COLUMN ... + + Esta função é aditiva e segura para bases existentes: cria tabelas e + índices apenas se não existirem e adiciona colunas opcionais em falta. + """ + global _CORE_SCHEMA_READY + if _CORE_SCHEMA_READY: + return + + with engine.begin() as conn: + # Necessário para DEFAULT gen_random_uuid(). Em PostgreSQL moderno, + # pgcrypto é a forma mais simples de gerar UUIDs no próprio servidor. + conn.execute(text("CREATE EXTENSION IF NOT EXISTS pgcrypto")) + + conn.execute(text(""" + CREATE TABLE IF NOT EXISTS schema_migrations ( + version TEXT PRIMARY KEY, + name TEXT NOT NULL, + applied_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + """)) + + conn.execute(text(""" + CREATE TABLE IF NOT EXISTS raw_events ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + source_system TEXT NOT NULL DEFAULT 'clientflow', + event_type TEXT, + source_event_id TEXT, + conversation_id TEXT, + contact_id TEXT, + payload JSONB NOT NULL DEFAULT '{}'::jsonb, + processed BOOLEAN NOT NULL DEFAULT FALSE, + ignored BOOLEAN NOT NULL DEFAULT FALSE, + processing_error TEXT, + message_id UUID, + action_run_id UUID, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + processed_at TIMESTAMPTZ + ) + """)) + + conn.execute(text(""" + CREATE UNIQUE INDEX IF NOT EXISTS ux_raw_events_source_event + ON raw_events(source_system, source_event_id) + WHERE source_event_id IS NOT NULL + """)) + + conn.execute(text(""" + CREATE TABLE IF NOT EXISTS messages ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + raw_event_id UUID, + source_system TEXT NOT NULL DEFAULT 'clientflow', + source_event_id TEXT, + conversation_id TEXT, + contact_id TEXT, + direction TEXT NOT NULL DEFAULT 'inbound', + raw_body TEXT, + clean_body TEXT, + previous_context TEXT, + metadata JSONB NOT NULL DEFAULT '{}'::jsonb, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + """)) + + conn.execute(text(""" + CREATE TABLE IF NOT EXISTS action_runs ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + message_id UUID, + raw_event_id UUID, + conversation_id TEXT, + contact_id TEXT, + source_system TEXT NOT NULL DEFAULT 'clientflow', + model TEXT, + provider TEXT, + openrouter_generation_id TEXT, + decision_source TEXT, + action_decision JSONB NOT NULL DEFAULT '{}'::jsonb, + action_result JSONB NOT NULL DEFAULT '{}'::jsonb, + prompt_tokens INTEGER, + completion_tokens INTEGER, + total_tokens INTEGER, + cost NUMERIC(12,6), + usage JSONB NOT NULL DEFAULT '{}'::jsonb, + needs_review BOOLEAN NOT NULL DEFAULT FALSE, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + """)) + + conn.execute(text(""" + CREATE TABLE IF NOT EXISTS tasks ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + action_run_id UUID, + message_id UUID, + raw_event_id UUID, + opportunity_id UUID, + conversation_id TEXT, + contact_id TEXT, + customer_id TEXT, + action_code TEXT NOT NULL DEFAULT 'REVIEW_MANUALLY', + route TEXT NOT NULL DEFAULT 'rever', + action TEXT NOT NULL DEFAULT 'Rever manualmente', + note TEXT, + action_required BOOLEAN NOT NULL DEFAULT FALSE, + safe_to_post BOOLEAN NOT NULL DEFAULT FALSE, + status TEXT NOT NULL DEFAULT 'pending', + source_system TEXT NOT NULL DEFAULT 'clientflow', + source_event_id TEXT, + idempotency_key TEXT, + due_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + done_at TIMESTAMPTZ, + done_by TEXT, + metadata JSONB NOT NULL DEFAULT '{}'::jsonb + ) + """)) + + # Colunas adicionadas em upgrades anteriores. Mantidas aqui para + # compatibilidade quando a tabela já existe numa base antiga. + for statement in [ + "ALTER TABLE tasks ADD COLUMN IF NOT EXISTS opportunity_id UUID", + "ALTER TABLE tasks ADD COLUMN IF NOT EXISTS due_at TIMESTAMPTZ", + "ALTER TABLE tasks ADD COLUMN IF NOT EXISTS done_at TIMESTAMPTZ", + "ALTER TABLE tasks ADD COLUMN IF NOT EXISTS done_by TEXT", + "ALTER TABLE tasks ADD COLUMN IF NOT EXISTS metadata JSONB NOT NULL DEFAULT '{}'::jsonb", + "ALTER TABLE tasks ADD COLUMN IF NOT EXISTS idempotency_key TEXT", + ]: + conn.execute(text(statement)) + + conn.execute(text(""" + CREATE UNIQUE INDEX IF NOT EXISTS ux_tasks_idempotency_key + ON tasks(idempotency_key) + WHERE idempotency_key IS NOT NULL + """)) + conn.execute(text("CREATE UNIQUE INDEX IF NOT EXISTS idx_tasks_idempotency_key ON tasks(idempotency_key)")) + + conn.execute(text(""" + CREATE TABLE IF NOT EXISTS task_preparations ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + task_id UUID NOT NULL, + conversation_id TEXT, + contact_id TEXT, + prep_type TEXT NOT NULL DEFAULT 'generic', + status TEXT NOT NULL DEFAULT 'draft', + extracted_data JSONB NOT NULL DEFAULT '{}'::jsonb, + missing_fields JSONB NOT NULL DEFAULT '[]'::jsonb, + suggested_reply TEXT, + confidence NUMERIC(4,3), + model TEXT, + provider TEXT, + total_tokens INTEGER NOT NULL DEFAULT 0, + cost NUMERIC(12,6) NOT NULL DEFAULT 0, + raw_response JSONB NOT NULL DEFAULT '{}'::jsonb, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + """)) + + conn.execute(text("CREATE INDEX IF NOT EXISTS idx_task_preparations_task ON task_preparations(task_id, created_at DESC)")) + conn.execute(text("CREATE INDEX IF NOT EXISTS idx_task_preparations_conversation ON task_preparations(conversation_id)")) + + conn.execute(text(""" + CREATE TABLE IF NOT EXISTS task_events ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + task_id UUID NOT NULL, + event_type TEXT NOT NULL, + payload JSONB NOT NULL DEFAULT '{}'::jsonb, + created_by TEXT NOT NULL DEFAULT 'system', + created_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + """)) + + conn.execute(text(""" + CREATE TABLE IF NOT EXISTS business_events ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + event_type TEXT NOT NULL, + task_id UUID, + action_run_id UUID, + message_id UUID, + raw_event_id UUID, + customer_id TEXT, + conversation_id TEXT, + contact_id TEXT, + payload JSONB NOT NULL DEFAULT '{}'::jsonb, + idempotency_key TEXT, + created_by TEXT NOT NULL DEFAULT 'system', + created_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + """)) + + conn.execute(text(""" + CREATE UNIQUE INDEX IF NOT EXISTS ux_business_events_idempotency_key + ON business_events(idempotency_key) + WHERE idempotency_key IS NOT NULL + """)) + conn.execute(text("CREATE UNIQUE INDEX IF NOT EXISTS idx_business_events_idempotency_key ON business_events(idempotency_key)")) + + conn.execute(text(""" + CREATE TABLE IF NOT EXISTS integration_outbox ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + business_event_id UUID, + target_system TEXT NOT NULL, + action_type TEXT NOT NULL, + payload JSONB NOT NULL DEFAULT '{}'::jsonb, + status TEXT NOT NULL DEFAULT 'pending', + retry_count INTEGER NOT NULL DEFAULT 0, + idempotency_key TEXT, + last_error TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + sent_at TIMESTAMPTZ + ) + """)) + + for statement in [ + "ALTER TABLE integration_outbox ADD COLUMN IF NOT EXISTS locked_at TIMESTAMPTZ", + "ALTER TABLE integration_outbox ADD COLUMN IF NOT EXISTS lock_owner TEXT", + "ALTER TABLE integration_outbox ADD COLUMN IF NOT EXISTS ignored_at TIMESTAMPTZ", + ]: + conn.execute(text(statement)) + + conn.execute(text(""" + CREATE UNIQUE INDEX IF NOT EXISTS ux_integration_outbox_idempotency_key + ON integration_outbox(idempotency_key) + WHERE idempotency_key IS NOT NULL + """)) + conn.execute(text("CREATE UNIQUE INDEX IF NOT EXISTS idx_integration_outbox_idempotency_key ON integration_outbox(idempotency_key)")) + conn.execute(text(""" + CREATE INDEX IF NOT EXISTS idx_integration_outbox_pending_dispatch + ON integration_outbox(target_system, created_at) + WHERE status = 'pending' + """)) + + conn.execute(text(""" + CREATE TABLE IF NOT EXISTS external_mappings ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + local_system TEXT NOT NULL DEFAULT 'clientflow', + local_entity_type TEXT NOT NULL, + local_entity_id TEXT NOT NULL, + external_system TEXT NOT NULL, + external_entity_type TEXT NOT NULL, + external_entity_id TEXT, + external_url TEXT, + match_key TEXT, + match_value TEXT, + confidence NUMERIC(4,3), + metadata JSONB NOT NULL DEFAULT '{}'::jsonb, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + """)) + + # Índices de consulta mais usados no admin e webhooks. + conn.execute(text("CREATE INDEX IF NOT EXISTS idx_tasks_status ON tasks(status)")) + conn.execute(text("CREATE INDEX IF NOT EXISTS idx_tasks_route ON tasks(route)")) + conn.execute(text("CREATE INDEX IF NOT EXISTS idx_tasks_conversation ON tasks(conversation_id)")) + conn.execute(text("CREATE INDEX IF NOT EXISTS idx_tasks_contact ON tasks(contact_id)")) + conn.execute(text("CREATE INDEX IF NOT EXISTS idx_tasks_created_at ON tasks(created_at DESC)")) + conn.execute(text("CREATE INDEX IF NOT EXISTS idx_raw_events_created_at ON raw_events(created_at DESC)")) + conn.execute(text("CREATE INDEX IF NOT EXISTS idx_raw_events_conversation ON raw_events(conversation_id)")) + conn.execute(text("CREATE INDEX IF NOT EXISTS idx_messages_conversation ON messages(conversation_id)")) + conn.execute(text("CREATE INDEX IF NOT EXISTS idx_action_runs_created_at ON action_runs(created_at DESC)")) + conn.execute(text("CREATE INDEX IF NOT EXISTS idx_task_events_task ON task_events(task_id, created_at DESC)")) + conn.execute(text("CREATE INDEX IF NOT EXISTS idx_business_events_created_at ON business_events(created_at DESC)")) + conn.execute(text("CREATE INDEX IF NOT EXISTS idx_outbox_status ON integration_outbox(status)")) + conn.execute(text("CREATE INDEX IF NOT EXISTS idx_outbox_target ON integration_outbox(target_system)")) + conn.execute(text("CREATE INDEX IF NOT EXISTS idx_external_mappings_local ON external_mappings(local_system, local_entity_type, local_entity_id)")) + + _CORE_SCHEMA_READY = True + + +def init_db() -> None: + """Inicialização do schema ClientFlow. + + Ordem importante para bases novas: + 1. core: raw_events/messages/action_runs/tasks/outbox; + 2. oportunidades: depende de tasks; + 3. produtos: depende de opportunities. + """ + try: + ensure_core_schema() + + from app.opportunity_service import ensure_opportunity_schema + from app.product_service import ensure_product_schema + from app.operation_service import ensure_operation_schema + from app.commercial_service import ensure_commercial_schema + from app.communication_service import ensure_communication_schema + from app.reconciliation_service import ensure_reconciliation_schema + + ensure_opportunity_schema() + ensure_product_schema() + ensure_operation_schema() + ensure_commercial_schema() + ensure_communication_schema() + ensure_reconciliation_schema() + except Exception as exc: + print(f"ClientFlow schema init failed: {exc}", flush=True) + raise + return None diff --git a/app/email_identity_cleanup_service.py b/app/email_identity_cleanup_service.py new file mode 100644 index 0000000..794bbc7 --- /dev/null +++ b/app/email_identity_cleanup_service.py @@ -0,0 +1,117 @@ +"""Cleanup helpers for stale email identity suggestions/extractions.""" +from __future__ import annotations + +import json +from typing import Any, Dict, List + +from sqlalchemy import text + +from app.db import engine +from app.email_identity_extraction_service import is_plausible_company_mention + +INVALID_EMAIL_IDENTITY_TOKENS = {"pt", "com", "net", "org", "www", "http", "https", "mail", "email"} + + +def _clean(value: Any) -> str: + return str(value or "").strip() + + +def _valid_company_mentions(values: Any) -> List[str]: + out: List[str] = [] + if not isinstance(values, list): + return out + for value in values: + v = _clean(value).strip(" ,.;:-") + if v and is_plausible_company_mention(v): + out.append(v) + return out + + +def cleanup_invalid_email_identity_state( + *, + opportunity_id: str | None = None, + include_accepted: bool = True, + fix_extractions: bool = True, + apply: bool = True, +) -> Dict[str, Any]: + where = ["lookup_type LIKE 'email_identity%'"] + params: dict[str, Any] = {} + if opportunity_id: + where.append("opportunity_id = CAST(:opportunity_id AS UUID)") + params["opportunity_id"] = opportunity_id + where.append("status IN ('pending', 'accepted', 'rejected')" if include_accepted else "status = 'pending'") + invalid_sql = ", ".join("'" + v.replace("'", "") + "'" for v in sorted(INVALID_EMAIL_IDENTITY_TOKENS)) + where.append(f"lower(trim(COALESCE(lookup_value, ''))) IN ({invalid_sql})") + sql_where = " AND ".join(where) + + result: Dict[str, Any] = {"invalid_suggestions": 0, "rejected": 0, "extractions_to_fix": 0, "fixed_extractions": 0} + with engine.begin() as conn: + rows = conn.execute(text(f""" + SELECT id::text, opportunity_id::text, suggested_name, suggested_nif, + lookup_type, lookup_value, confidence, status, reason, created_at + FROM fiscal_customer_suggestions + WHERE {sql_where} + ORDER BY created_at DESC + """), params).mappings().all() + result["invalid_suggestions"] = len(rows) + result["suggestions"] = [dict(row) for row in rows] + if apply and rows: + ids = [r["id"] for r in rows] + conn.execute(text(""" + UPDATE fiscal_customer_suggestions + SET status = 'rejected', + reason = COALESCE(reason, '') || ' | rejected_invalid_email_identity_token', + resolved_by = 'cleanup_invalid_email_identity_state', + resolved_at = now(), + updated_at = now() + WHERE id = ANY(CAST(:ids AS UUID[])) + """), {"ids": ids}) + result["rejected"] = len(ids) + + if fix_extractions: + extraction_where = [] + extraction_params: dict[str, Any] = {} + if opportunity_id: + extraction_where.append("opportunity_id = CAST(:opportunity_id AS UUID)") + extraction_params["opportunity_id"] = opportunity_id + extraction_sql = "WHERE " + " AND ".join(extraction_where) if extraction_where else "" + ex_rows = conn.execute(text(f""" + SELECT id::text, opportunity_id::text, company_mentions, confidence, raw_payload + FROM email_identity_extractions + {extraction_sql} + ORDER BY updated_at DESC + """), extraction_params).mappings().all() + changed = [] + for row in ex_rows: + original = row.get("company_mentions") or [] + valid = _valid_company_mentions(original) + if list(original or []) == valid: + continue + item = {"id": row["id"], "opportunity_id": row["opportunity_id"], "before": original, "after": valid} + changed.append(item) + if apply: + try: + confidence = float(row.get("confidence") or 0) + except Exception: + confidence = 0.0 + if not valid: + confidence = min(confidence, 0.45) + raw_payload = dict(row.get("raw_payload") or {}) if isinstance(row.get("raw_payload"), dict) else {} + raw_payload["filtered_invalid_company_mentions"] = list(original or []) + conn.execute(text(""" + UPDATE email_identity_extractions + SET company_mentions = CAST(:company_mentions AS JSONB), + confidence = :confidence, + raw_payload = COALESCE(raw_payload, '{}'::jsonb) || CAST(:raw_payload AS JSONB), + updated_at = now() + WHERE id = CAST(:id AS UUID) + """), { + "id": row["id"], + "company_mentions": json.dumps(valid, ensure_ascii=False), + "confidence": confidence, + "raw_payload": json.dumps(raw_payload, ensure_ascii=False, default=str), + }) + result["extractions_to_fix"] = len(changed) + result["fixed_extractions"] = len(changed) if apply else 0 + result["extractions"] = changed + return result diff --git a/app/email_identity_extraction_service.py b/app/email_identity_extraction_service.py new file mode 100644 index 0000000..73617f2 --- /dev/null +++ b/app/email_identity_extraction_service.py @@ -0,0 +1,798 @@ +"""Email identity extraction for fiscal enrichment. + +v4.9.26 adds an evidence layer before fiscal lookup. The goal is not to +confirm a fiscal customer by itself; it extracts identity signals from message +bodies/signatures so the enrichment worker can prefer explicit company evidence +(Dietimport S.A. in a signature/disclaimer, for example) over a weaker domain +match returned by an external endpoint. +""" +from __future__ import annotations + +import json +import os +import re +import urllib.request +import urllib.error +import unicodedata +from datetime import datetime, timezone +from typing import Any, Dict, List, Optional + +from sqlalchemy import text + +from app.config import settings +from app.db import engine +from app.commercial_service import normalize_fiscal_name, normalize_tax_id + +_SCHEMA_READY = False + +COMPANY_SUFFIX_RE = re.compile( + r"\b([A-ZÁÀÂÃÉÈÊÍÌÎÓÒÔÕÚÙÛÇ0-9][A-Za-zÁÀÂÃÉÈÊÍÌÎÓÒÔÕÚÙÛÇáàâãéèêíìîóòôõúùûç0-9&.,ºª\-' ]{2,}?\s+(?:S\.?A\.?|LDA\.?|UNIPESSOAL\s*,?\s*LDA\.?|LIMITADA|SOCIEDADE\s+UNIPESSOAL|S\.A\.|SA))\b", + flags=re.I, +) +PHONE_RE = re.compile(r"(?:\+351\s*)?(?:\d[\s.-]?){9,}") +POSTCODE_RE = re.compile(r"\b\d{4}-\d{3}\b") +URL_RE = re.compile(r"(?:https?://)?(?:www\.)?([a-z0-9][a-z0-9.-]+\.[a-z]{2,})(?:/[^\s)]*)?", flags=re.I) +EMAIL_RE = re.compile(r"[A-Z0-9._%+\-]+@[A-Z0-9.\-]+\.[A-Z]{2,}", flags=re.I) +PUBLIC_EMAIL_DOMAINS = { + "gmail.com", "googlemail.com", "hotmail.com", "hotmail.pt", "outlook.com", + "outlook.pt", "live.com", "msn.com", "yahoo.com", "yahoo.pt", + "icloud.com", "me.com", "mac.com", "sapo.pt", "mail.telepac.pt", + "aol.com", "proton.me", "protonmail.com", "pm.me", +} + +# Tokens that are often hallucinated/extracted from domains and TLDs. +# They must never be treated as a company mention. +INVALID_COMPANY_MENTION_TOKENS = { + "pt", "com", "net", "org", "www", "http", "https", "email", "mail", + "lda", "sa", "s.a", "unipessoal", +} + + + +def _clean(value: Any) -> str: + return re.sub(r"\s+", " ", str(value or "")).strip() + + +def _remove_unexpected_unicode(value: Any) -> str: + """Remove characters often produced by noisy LLM responses. + + Keep Portuguese/European punctuation and letters, but remove CJK/Hangul/Kana + artifacts such as the observed ``Luis Roch游戏副本a`` case. + """ + out: List[str] = [] + for ch in str(value or ""): + if not ch: + continue + name = unicodedata.name(ch, "") + if any(token in name for token in ("CJK", "HIRAGANA", "KATAKANA", "HANGUL", "BOPOMOFO")): + continue + # Drop control/private characters; keep normal letters, numbers, spaces and punctuation. + if unicodedata.category(ch).startswith(("C",)): + continue + out.append(ch) + return _clean("".join(out)) + + +def _json(value: Any) -> str: + return json.dumps(value or {}, ensure_ascii=False, default=str) + + +def _now_iso() -> str: + return datetime.now(timezone.utc).isoformat() + + +def _domain_from_email(email: Any) -> str: + email_value = _clean(email).lower() + if "@" not in email_value: + return "" + return normalize_domain(email_value.rsplit("@", 1)[1]) + + +def normalize_domain(value: Any) -> str: + raw = _clean(value).lower() + if not raw: + return "" + raw = raw.replace("https://", "").replace("http://", "") + raw = raw.split("/", 1)[0].split("?", 1)[0].split("#", 1)[0] + raw = raw.strip(". ") + if raw.startswith("www."): + raw = raw[4:] + return raw + + +def _looks_like_domain_token(value: Any) -> bool: + v = _clean(value).lower().strip(" ,.;:-") + if not v: + return False + if v in INVALID_COMPANY_MENTION_TOKENS: + return True + if re.fullmatch(r"[a-z]{2,3}", v): + return True + # Full domains/hostnames are website evidence, not company names. + if "." in v and normalize_domain(v) == v and re.search(r"\.[a-z]{2,}$", v): + return True + return False + + +def is_plausible_company_mention(value: Any) -> bool: + v = _clean(value).strip(" ,.;:-") + if not v or _looks_like_domain_token(v): + return False + normalized = normalize_fiscal_name(v) or v.casefold() + if not normalized or len(normalized) < 4: + return False + if normalized in INVALID_COMPANY_MENTION_TOKENS: + return False + if re.fullmatch(r"[a-z]{2,3}", normalized): + return False + # Avoid bare legal suffixes and tiny fragments extracted from emails/domains. + meaningful_tokens = [t for t in normalized.split() if len(t) >= 4 and t not in {"unipessoal", "limitada"}] + if not meaningful_tokens and not re.search(r"\b(?:lda|s\.?a|unipessoal|limitada)\b", v, flags=re.I): + return False + return True + + +def _dedupe(values: List[str]) -> List[str]: + out: List[str] = [] + seen = set() + for value in values: + v = _clean(value).strip(" ,.;:-") + if not v: + continue + key = normalize_fiscal_name(v) or v.casefold() + if key not in seen: + seen.add(key) + out.append(v) + return out + + +COMPANY_NOISE_WORDS = { + "anexos", "aviso", "confidencial", "copiar", "destinatario", "destinatário", + "distribuir", "emissor", "exclusivo", "mensagem", "opinioes", "opiniões", + "recebeu", "uso", "utilizar", "informacao", "informação", "ficheiro", + "responsavel", "responsável", "solicitada", "eliminacao", "eliminação", + "confirmacao", "confirmação", "tratamento", "dados", "contacto", "breve", +} +COMPANY_BAD_START_RE = re.compile( + r"^(?:dos|das|do|da|de|obter|informamos|informamos,|enquanto|tacto|contacto|entramos|entraremos|esta|este|esses|essas)\b", + flags=re.I, +) +VALEDICTION_RE = re.compile(r"^(?:com os melhores cumprimentos|melhores cumprimentos|cumprimentos|obrigad[ao]s?|boa tarde|bom dia|exmos?\.?)\b", flags=re.I) +BAD_PERSON_RE = re.compile( + r"^(?:enviado do meu|sent from my|assinado por|aviso|disclaimer|cid:|image|logo|telefone|telem|tel\.?|email|e-mail)\b", + flags=re.I, +) +ROLE_OR_SECTION_RE = re.compile(r"\b(?:engenheiro|respons[aá]vel|departamento|manuten[cç][aã]o|telem|tel|telefone|aviso|rua|avenida|morada|nif|email)\b", flags=re.I) +LEGAL_SUFFIX = r"(?:[Ss]\.?\s*[Aa]\.?|[Ss][Aa]|[Ll][Dd][Aa]\.?|[Uu]nipessoal\s*,?\s*[Ll][Dd][Aa]\.?|[Ll]imitada|[Ss]ociedade\s+[Uu]nipessoal)" +LEGAL_COMPANY_RE = re.compile( + rf"\b([A-ZÁÀÂÃÉÈÊÍÌÎÓÒÔÕÚÙÛÇ0-9][A-Za-zÁÀÂÃÉÈÊÍÌÎÓÒÔÕÚÙÛÇáàâãéèêíìîóòôõúùûç0-9&.,ºª\-' ]{{0,75}}?\s*,?\s*{LEGAL_SUFFIX})\b" +) +ADDRESS_RE = re.compile( + r"\b((?:Rua|R\.|Avenida|Av\.|Estrada|Travessa|Largo|Pra[cç]a|Praceta|Alameda)\s+.{3,180}?\b\d{4}-\d{3}\s+[A-Za-zÁÀÂÃÉÈÊÍÌÎÓÒÔÕÚÙÛÇáàâãéèêíìîóòôõúùûç .'-]+)", + flags=re.I | re.S, +) + + +def _sanitize_company_mention(value: Any) -> str: + """Return a concise plausible company mention. + + This is deliberately stricter than the LLM. Provider responses and regex + can include entire GDPR/legal paragraphs; fiscal enrichment should only see + short organization labels, for example ``SOPAC, S.A`` and not the whole + sentence containing it. + """ + v = _remove_unexpected_unicode(value).strip(" ,;:-") + if not v or _looks_like_domain_token(v): + return "" + + # Normalize common punctuation spacing before trying to extract a legal name. + v = re.sub(r"\s+([,.;:])", r"\1", v) + v = re.sub(r"([,.;:])(?=\S)", r"\1 ", v) + v = _clean(v).strip(" ,;:-") + + def _tidy_company_name(name: str) -> str: + name = _clean(name).strip(" ,;:-.") + protected = re.sub(r"\b([Ss])\.?\s*([Aa])\.?\b", r"\1__DOT__\2__DOT__", name) + sentence_parts = [part.strip(" ,;:-.") for part in re.split(r"\.\s+", protected) if part.strip(" ,;:-.")] + if len(sentence_parts) > 1: + protected = sentence_parts[-1] + name = protected.replace("__DOT__", ".") + name = re.sub(r"\b[Ss]\.?\s*[Aa]\.?\b", "S.A", name) + name = re.sub(r"\b[Ll][Dd][Aa]\.?\b", "Lda", name) + return name + + candidates = [] + for match in LEGAL_COMPANY_RE.finditer(v): + candidate = _tidy_company_name(match.group(1)) + if candidate: + candidates.append(candidate) + + # Specific tail found in legal disclaimers: "... da Dietimport S.A.". + tail = re.search(rf"\b(?:da|de|do|pela|pelo|responsável é a|responsavel e a|é a|e a)\s+(.+?{LEGAL_SUFFIX})\.?$", v, flags=re.I) + if tail: + candidate = _tidy_company_name(tail.group(1)) + extracted = [m.group(1).strip(" ,;:-.") for m in LEGAL_COMPANY_RE.finditer(candidate)] + candidates.extend(extracted or [candidate]) + + if candidates: + # Prefer the shortest clean candidate, because long ones are usually + # paragraphs ending in a company suffix. + clean_candidates = [] + for candidate in candidates: + c = _tidy_company_name(candidate) + words = c.split() + normalized_words = set((normalize_fiscal_name(c) or c.casefold()).replace(".", " ").split()) + if not c or len(c) > 90 or len(words) > 8: + continue + if normalized_words & COMPANY_NOISE_WORDS: + continue + if COMPANY_BAD_START_RE.search(c): + continue + clean_candidates.append(c) + if clean_candidates: + return sorted(clean_candidates, key=len)[0] + + words = v.split() + normalized_words = set((normalize_fiscal_name(v) or v.casefold()).replace(".", " ").split()) + noisy = bool(normalized_words & COMPANY_NOISE_WORDS) + if len(v) > 80 or len(words) > 8 or noisy or COMPANY_BAD_START_RE.search(v): + return "" + + # Avoid returning full person names as companies. Single/two-token brand + # mentions such as "Inkey", "Coimpack", "Badoni" are still allowed. + if BAD_PERSON_RE.search(v) or VALEDICTION_RE.search(v): + return "" + tidied = _tidy_company_name(v) + if not is_plausible_company_mention(tidied): + return "" + return tidied + + +def _sanitize_person_name(value: Any) -> str: + v = _remove_unexpected_unicode(value).strip(" ,.;:-") + if not v: + return "" + v = re.sub(r"\b(?:Sr\.?|Sra\.?|Dr\.?|Dra\.?)\s+", "", v, flags=re.I).strip() + if not v or "@" in v or any(ch.isdigit() for ch in v): + return "" + if BAD_PERSON_RE.search(v) or VALEDICTION_RE.search(v): + return "" + if len(v) > 80 or len(v.split()) > 7: + return "" + if re.search(r"\b(?:LDA|S\.?A\.?|UNIPESSOAL|LIMITADA|AVISO|CONFIDENCIAL)\b", v, flags=re.I): + return "" + return v + + +def _is_person_name_candidate(value: Any) -> bool: + v = _sanitize_person_name(value) + if not v or "@" in v or any(ch.isdigit() for ch in v): + return False + if VALEDICTION_RE.search(v) or ROLE_OR_SECTION_RE.search(v): + return False + words = v.split() + if not (2 <= len(words) <= 5): + return False + # Require at least two human-looking capitalized tokens. + capitalized = [w for w in words if re.match(r"^[A-ZÁÀÂÃÉÈÊÍÌÎÓÒÔÕÚÙÛÇ][A-Za-zÁÀÂÃÉÈÊÍÌÎÓÒÔÕÚÙÛÇáàâãéèêíìîóòôõúùûç'.-]+$", w)] + return len(capitalized) >= 2 + + +def _extract_address(text_value: str, postcodes: List[str]) -> str: + compact = _clean(text_value) + match = ADDRESS_RE.search(compact) + if match: + address = _clean(match.group(1)).strip(" ,.;") + address = re.split(r"\b(?:Telem|Tel|Telefone|Phone|Email|E-mail|AVISO)\b", address, maxsplit=1, flags=re.I)[0] + return _clean(address).strip(" ,.;") + if not postcodes: + return "" + lines = [_clean(x) for x in text_value.splitlines() if _clean(x)] + for idx, line in enumerate(lines): + if any(pc in line for pc in postcodes): + before = lines[idx - 1] if idx > 0 and re.search(r"\b(?:Rua|R\.|Avenida|Av\.|Estrada|Travessa|Largo|Pra[cç]a|Praceta|Alameda)\b", lines[idx - 1], flags=re.I) else "" + candidate = _clean((before + " " + line).strip()) + street = re.search(r"\b(?:Rua|R\.|Avenida|Av\.|Estrada|Travessa|Largo|Pra[cç]a|Praceta|Alameda)\b.*", candidate, flags=re.I) + return _clean(street.group(0) if street else candidate).strip(" ,.;") + return "" + + +def ensure_email_identity_schema() -> None: + global _SCHEMA_READY + if _SCHEMA_READY: + return + with engine.begin() as conn: + conn.execute(text(""" + CREATE TABLE IF NOT EXISTS email_identity_extractions ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + opportunity_id UUID REFERENCES opportunities(id) ON DELETE CASCADE, + task_id UUID REFERENCES tasks(id) ON DELETE SET NULL, + message_id UUID REFERENCES messages(id) ON DELETE SET NULL, + conversation_id TEXT, + contact_id TEXT, + email TEXT, + domain TEXT, + person_name TEXT, + company_mentions JSONB NOT NULL DEFAULT '[]'::jsonb, + address TEXT, + phones JSONB NOT NULL DEFAULT '[]'::jsonb, + websites JSONB NOT NULL DEFAULT '[]'::jsonb, + extraction_method TEXT NOT NULL DEFAULT 'regex', + confidence NUMERIC(5,2) NOT NULL DEFAULT 0, + evidence JSONB NOT NULL DEFAULT '[]'::jsonb, + raw_payload JSONB NOT NULL DEFAULT '{}'::jsonb, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + """)) + for stmt in [ + "ALTER TABLE email_identity_extractions ADD COLUMN IF NOT EXISTS confidence NUMERIC(5,2) NOT NULL DEFAULT 0", + "ALTER TABLE email_identity_extractions ADD COLUMN IF NOT EXISTS extraction_method TEXT NOT NULL DEFAULT 'regex'", + "CREATE INDEX IF NOT EXISTS idx_email_identity_extractions_opp ON email_identity_extractions(opportunity_id, updated_at DESC)", + "CREATE INDEX IF NOT EXISTS idx_email_identity_extractions_domain ON email_identity_extractions(domain)", + ]: + conn.execute(text(stmt)) + _SCHEMA_READY = True + + +def extract_first_json_object(raw: str) -> str: + s = str(raw or "").strip() + if s.startswith("```"): + lines = s.splitlines() + if lines and lines[0].strip().startswith("```"): + lines = lines[1:] + if lines and lines[-1].strip().startswith("```"): + lines = lines[:-1] + s = "\n".join(lines).strip() + start = s.find("{") + if start == -1: + raise ValueError("no JSON object found") + in_string = False + escaped = False + depth = 0 + for i in range(start, len(s)): + ch = s[i] + if escaped: + escaped = False + continue + if ch == "\\": + escaped = True + continue + if ch == '"': + in_string = not in_string + continue + if in_string: + continue + if ch == "{": + depth += 1 + elif ch == "}": + depth -= 1 + if depth == 0: + return s[start:i + 1] + raise ValueError("incomplete JSON object") + + +def _regex_identity(text_value: str, *, email: str = "", subject: str = "") -> Dict[str, Any]: + text_value = str(text_value or "") + emails = EMAIL_RE.findall(text_value) + if email: + emails.insert(0, email) + domains = [normalize_domain(m.group(1)) for m in URL_RE.finditer(text_value)] + for e in emails: + d = _domain_from_email(e) + if d: + domains.insert(0, d) + + company_mentions: List[str] = [] + for match in COMPANY_SUFFIX_RE.finditer(text_value): + cleaned_company = _sanitize_company_mention(match.group(1)) + if cleaned_company: + company_mentions.append(cleaned_company) + + # Portuguese disclaimers often contain "da " near the organization. + disclaimer_match = re.search( + r"opini[oõ]es emitidas.+?\b(?:da|de|do)\s+([A-ZÁÀÂÃÉÈÊÍÌÎÓÒÔÕÚÙÛÇ][A-Za-zÁÀÂÃÉÈÊÍÌÎÓÒÔÕÚÙÛÇáàâãéèêíìîóòôõúùûç0-9&.,ºª\-' ]{2,}?\s+(?:S\.?A\.?|LDA\.?|LIMITADA))", + text_value, + flags=re.I | re.S, + ) + if disclaimer_match: + cleaned_company = _sanitize_company_mention(disclaimer_match.group(1)) + if cleaned_company: + company_mentions.append(cleaned_company) + + # Marketing/reply subjects often contain "para a ". Treat this as + # a weak mention useful for review, not as fiscal confirmation by itself. + subject_company = re.search( + r"\bpara\s+(?:a|o|à|ao)\s+([A-ZÁÀÂÃÉÈÊÍÌÎÓÒÔÕÚÙÛÇ0-9][A-Za-zÁÀÂÃÉÈÊÍÌÎÓÒÔÕÚÙÛÇáàâãéèêíìîóòôõúùûç0-9&.,ºª\-' ]{2,60})$", + _clean(subject), + flags=re.I, + ) + if subject_company: + subject_name = _sanitize_company_mention(subject_company.group(1)) + if subject_name: + company_mentions.append(subject_name) + + phones = [m.group(0).strip() for m in PHONE_RE.finditer(text_value)] + postcodes = POSTCODE_RE.findall(text_value) + address = _extract_address(text_value, postcodes) + + person_name = "" + lines = [_clean(x) for x in text_value.splitlines() if _clean(x)] + for idx, line in enumerate(lines[:30]): + if re.search(r"cumprimentos|obrigad", line, flags=re.I): + for candidate in lines[idx + 1: idx + 7]: + if _is_person_name_candidate(candidate): + person_name = _sanitize_person_name(candidate) + break + if person_name: + break + + evidence: List[str] = [] + if company_mentions: + evidence.append("empresa mencionada no corpo/assinatura: " + company_mentions[0]) + if address: + evidence.append("morada extraída da assinatura") + if phones: + evidence.append("telefone extraído da assinatura") + + confidence = 0.0 + if company_mentions: + confidence += 0.55 + if domains: + confidence += 0.15 + if address: + confidence += 0.15 + if phones: + confidence += 0.10 + if person_name: + confidence += 0.05 + + return { + "person_name": person_name, + "company_mentions": _dedupe(company_mentions), + "address": address, + "phones": _dedupe(phones), + "websites": _dedupe(domains), + "email": _clean(email or (emails[0] if emails else "")).lower(), + "domain": normalize_domain(domains[0]) if domains else _domain_from_email(email), + "subject": subject, + "confidence": min(round(confidence, 2), 0.95), + "evidence": evidence, + "method": "regex", + } + + +def _llm_enabled() -> bool: + return bool(getattr(settings, "openrouter_api_key", "")) + + +def _llm_timeout_seconds() -> int: + try: + value = os.getenv("EMAIL_IDENTITY_LLM_TIMEOUT_SECONDS") or getattr(settings, "email_identity_llm_timeout_seconds", 20) + return max(5, min(60, int(value))) + except Exception: + return 20 + + +def _llm_max_body_chars() -> int: + try: + value = os.getenv("EMAIL_IDENTITY_LLM_MAX_BODY_CHARS") or getattr(settings, "email_identity_llm_max_body_chars", 3500) + return max(1000, min(12000, int(value))) + except Exception: + return 3500 + + +def _llm_model(model_override: str = "") -> str: + return ( + _clean(model_override) + or os.getenv("EMAIL_IDENTITY_LLM_MODEL") + or getattr(settings, "email_identity_llm_model", "") + or getattr(settings, "openrouter_model", "") + ) + + +def _llm_fallback_model() -> str: + return ( + os.getenv("EMAIL_IDENTITY_LLM_FALLBACK_MODEL") + or getattr(settings, "email_identity_llm_fallback_model", "") + or "" + ) + + +def _is_public_domain(domain: Any) -> bool: + return normalize_domain(domain) in PUBLIC_EMAIL_DOMAINS + + +def _identity_needs_fallback(identity: Dict[str, Any]) -> bool: + companies = identity.get("company_mentions") or [] + domain = normalize_domain(identity.get("domain")) + confidence = float(identity.get("confidence") or 0) + if identity.get("_timeout") or identity.get("_error"): + return True + if not companies and domain and not _is_public_domain(domain): + return True + if not companies and confidence < 0.65: + return True + return False + + +def _call_llm_identity(text_value: str, *, email: str = "", subject: str = "", model_override: str = "") -> Optional[Dict[str, Any]]: + if not _llm_enabled() or not text_value.strip(): + return None + text_for_llm = str(text_value or "")[:_llm_max_body_chars()] + prompt = f"""Extrai identidade de um email B2B para reconciliação fiscal. Não confirmes cliente fiscal; apenas extrai evidências explícitas. + +Devolve apenas JSON válido com este formato: +{{ + "person_name": "", + "person_role": "", + "department": "", + "company_mentions": [], + "address": "", + "phones": [], + "websites": [], + "nif_candidates": [], + "intent_hint": "", + "confidence": 0.0, + "evidence": [] +}} + +Regras: +- company_mentions deve conter apenas nomes curtos de empresas/organizações, não frases completas. +- Não copies parágrafos legais/GDPR para company_mentions. Extrai só o nome, por exemplo "SOPAC, S.A". +- Não uses o domínio do email para inventar empresa. +- Se a empresa aparecer só no assunto tipo "para a Empresa X", podes incluir Empresa X, mas não inventes NIF. +- Se aparecer aviso legal tipo “opiniões ... da Empresa X”, inclui apenas Empresa X como company_mentions. +- person_name nunca deve ser "Enviado do meu Galaxy", valediction, cargo ou disclaimer. +- evidence deve citar fragmentos curtos do email. + +Assunto: {subject or ''} +Email origem: {email or ''} + +Mensagem: +{text_for_llm} +""" + payload = { + "model": _llm_model(model_override), + "messages": [ + {"role": "system", "content": "És um extrator JSON rigoroso de identidade empresarial em emails B2B."}, + {"role": "user", "content": prompt}, + ], + "temperature": 0, + "max_tokens": 500, + } + request = urllib.request.Request( + getattr(settings, "openrouter_url", "https://openrouter.ai/api/v1/chat/completions"), + data=json.dumps(payload, ensure_ascii=False).encode("utf-8"), + method="POST", + headers={ + "Authorization": f"Bearer {getattr(settings, 'openrouter_api_key', '')}", + "Content-Type": "application/json", + }, + ) + try: + with urllib.request.urlopen(request, timeout=_llm_timeout_seconds()) as response: # noqa: S310 configured API + raw = json.loads(response.read().decode("utf-8")) + content = raw["choices"][0]["message"].get("content") or "" + data = json.loads(extract_first_json_object(content)) + if isinstance(data, dict): + data["method"] = "llm" + data["llm_model"] = _llm_model(model_override) + data["raw_llm_content"] = content[:1000] + return data + except Exception: + return None + return None + + +def _finalize_identity(data: Dict[str, Any]) -> Dict[str, Any]: + finalized = dict(data) + finalized["person_name"] = _sanitize_person_name(finalized.get("person_name")) + + companies: List[str] = [] + for item in finalized.get("company_mentions") or []: + cleaned = _sanitize_company_mention(item) + if cleaned: + companies.append(cleaned) + finalized["company_mentions"] = _dedupe(companies) + + phones = [_clean(x) for x in finalized.get("phones") or [] if _clean(x)] + finalized["phones"] = _dedupe(phones) + websites = [normalize_domain(x) for x in finalized.get("websites") or [] if normalize_domain(x)] + finalized["websites"] = _dedupe(websites) + + email_domain = _domain_from_email(finalized.get("email")) + if email_domain: + finalized["domain"] = email_domain + else: + finalized["domain"] = normalize_domain(finalized.get("domain")) + + try: + confidence = float(finalized.get("confidence") or 0) + except Exception: + confidence = 0.0 + if not finalized["company_mentions"]: + # Identity may be good for a person, but fiscal identity remains weak. + confidence = min(confidence, 0.45) + finalized["confidence"] = round(max(0.0, min(confidence, 0.98)), 2) + return finalized + + +def merge_identity(regex_data: Dict[str, Any], llm_data: Optional[Dict[str, Any]]) -> Dict[str, Any]: + if not llm_data: + return _finalize_identity(regex_data) + merged = dict(regex_data) + for key in ("person_name", "person_role", "department", "address", "intent_hint"): + if _clean(llm_data.get(key)): + merged[key] = _clean(llm_data.get(key)) + for key in ("company_mentions", "phones", "websites", "nif_candidates", "evidence"): + values = [] + # For company names, prefer the LLM explicit extraction and then add + # sanitized regex fallback values. This avoids showing a long disclaimer + # paragraph before the actual company name. + sources = (llm_data, regex_data) if key == "company_mentions" else (regex_data, llm_data) + for source in sources: + if isinstance(source.get(key), list): + for item in source.get(key): + value = _sanitize_company_mention(item) if key == "company_mentions" else str(item) + if value: + values.append(value) + merged[key] = _dedupe(values) + merged["confidence"] = max(float(regex_data.get("confidence") or 0), float(llm_data.get("confidence") or 0)) + merged["method"] = "regex+llm" + merged["llm_model"] = llm_data.get("llm_model") or _llm_model() + merged["raw_llm_content"] = llm_data.get("raw_llm_content") + return _finalize_identity(merged) + + +def extract_email_identity(text_value: str, *, email: str = "", subject: str = "", use_llm: bool = True) -> Dict[str, Any]: + regex_data = _regex_identity(text_value, email=email, subject=subject) + llm_data = _call_llm_identity(text_value, email=email, subject=subject) if use_llm else None + identity = merge_identity(regex_data, llm_data) + + fallback_model = _llm_fallback_model() + first_model = _clean(identity.get("llm_model") or _llm_model()) + if use_llm and fallback_model and fallback_model != first_model and _identity_needs_fallback(identity): + fallback_data = _call_llm_identity(text_value, email=email, subject=subject, model_override=fallback_model) + fallback_identity = merge_identity(regex_data, fallback_data) if fallback_data else identity + if (fallback_identity.get("company_mentions") and not identity.get("company_mentions")) or float(fallback_identity.get("confidence") or 0) > float(identity.get("confidence") or 0): + fallback_identity["fallback_used"] = True + fallback_identity["fallback_from_model"] = first_model + identity = fallback_identity + return identity + + +def _latest_message_for_opportunity(opportunity_id: str) -> Optional[Dict[str, Any]]: + with engine.begin() as conn: + row = conn.execute(text(""" + SELECT + t.id::text AS task_id, + t.message_id::text AS message_id, + t.conversation_id, + t.contact_id, + COALESCE(NULLIF(m.clean_body, ''), NULLIF(m.raw_body, '')) AS body, + COALESCE( + NULLIF(m.metadata->>'subject', ''), + NULLIF(re.payload->'conversation'->'additional_attributes'->>'mail_subject', ''), + NULLIF(re.payload->'content_attributes'->'email'->>'subject', ''), + NULLIF(re.payload->'conversation'->'messages'->0->'content_attributes'->'email'->>'subject', '') + ) AS subject, + COALESCE( + NULLIF(re.payload->'sender'->>'email', ''), + NULLIF(re.payload->'conversation'->'meta'->'sender'->>'email', ''), + NULLIF(re.payload->'conversation'->'contact_inbox'->>'source_id', '') + ) AS event_email + FROM tasks t + LEFT JOIN messages m ON m.id = t.message_id + LEFT JOIN raw_events re ON re.id = t.raw_event_id + WHERE t.opportunity_id = CAST(:opportunity_id AS UUID) + ORDER BY t.created_at DESC + LIMIT 1 + """), {"opportunity_id": opportunity_id}).mappings().first() + return dict(row) if row and row.get("body") else None + + +def upsert_identity_extraction_for_opportunity(opportunity_id: str, *, use_llm: bool = True) -> Optional[Dict[str, Any]]: + ensure_email_identity_schema() + msg = _latest_message_for_opportunity(opportunity_id) + if not msg: + return None + identity = extract_email_identity( + msg.get("body") or "", + email=msg.get("event_email") or "", + subject=msg.get("subject") or "", + use_llm=use_llm, + ) + params = { + "opportunity_id": opportunity_id, + "task_id": msg.get("task_id"), + "message_id": msg.get("message_id"), + "conversation_id": msg.get("conversation_id"), + "contact_id": msg.get("contact_id"), + "email": identity.get("email") or msg.get("event_email"), + "domain": normalize_domain(identity.get("domain")), + "person_name": identity.get("person_name"), + "company_mentions": _json(identity.get("company_mentions") or []), + "address": identity.get("address"), + "phones": _json(identity.get("phones") or []), + "websites": _json(identity.get("websites") or []), + "method": identity.get("method") or "regex", + "confidence": float(identity.get("confidence") or 0), + "evidence": _json(identity.get("evidence") or []), + "raw_payload": _json(identity), + } + with engine.begin() as conn: + row = conn.execute(text(""" + INSERT INTO email_identity_extractions ( + opportunity_id, task_id, message_id, conversation_id, contact_id, email, domain, + person_name, company_mentions, address, phones, websites, extraction_method, + confidence, evidence, raw_payload, updated_at + ) VALUES ( + CAST(:opportunity_id AS UUID), CAST(:task_id AS UUID), CAST(:message_id AS UUID), + :conversation_id, :contact_id, :email, :domain, :person_name, + CAST(:company_mentions AS JSONB), :address, CAST(:phones AS JSONB), + CAST(:websites AS JSONB), :method, :confidence, CAST(:evidence AS JSONB), + CAST(:raw_payload AS JSONB), now() + ) + RETURNING id::text, opportunity_id::text, task_id::text, message_id::text, + conversation_id, contact_id, email, domain, person_name, + company_mentions, address, phones, websites, extraction_method, + confidence, evidence, raw_payload, created_at, updated_at + """), params).mappings().first() + return dict(row) if row else None + + +def _normalize_stored_identity(identity: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]: + """Apply current quality filters to identities already stored in DB. + + Older rows may contain noisy company_mentions such as ``pt`` extracted from + domains/TLDs. The UI and fiscal review must not keep showing or matching + those stale tokens just because they were saved before the stricter filters. + """ + if not identity: + return identity + normalized = dict(identity) + companies: List[str] = [] + invalid: List[str] = [] + for item in normalized.get("company_mentions") or []: + cleaned = _sanitize_company_mention(item) + if cleaned and is_plausible_company_mention(cleaned): + companies.append(cleaned) + elif _clean(item): + invalid.append(_clean(item)) + normalized["company_mentions"] = _dedupe(companies) + if invalid: + raw_payload = dict(normalized.get("raw_payload") or {}) if isinstance(normalized.get("raw_payload"), dict) else {} + raw_payload["filtered_invalid_company_mentions"] = _dedupe(invalid) + normalized["raw_payload"] = raw_payload + try: + confidence = float(normalized.get("confidence") or 0) + except Exception: + confidence = 0.0 + if not normalized["company_mentions"]: + confidence = min(confidence, 0.45) + normalized["confidence"] = round(max(0.0, min(confidence, 0.98)), 2) + return normalized + + +def latest_identity_for_opportunity(opportunity_id: str) -> Optional[Dict[str, Any]]: + ensure_email_identity_schema() + with engine.begin() as conn: + row = conn.execute(text(""" + SELECT id::text, opportunity_id::text, task_id::text, message_id::text, + conversation_id, contact_id, email, domain, person_name, company_mentions, + address, phones, websites, extraction_method, confidence, evidence, + raw_payload, created_at, updated_at + FROM email_identity_extractions + WHERE opportunity_id = CAST(:opportunity_id AS UUID) + ORDER BY updated_at DESC + LIMIT 1 + """), {"opportunity_id": opportunity_id}).mappings().first() + return _normalize_stored_identity(dict(row)) if row else None + + +def extract_identity_for_opportunity(opportunity_id: str, *, refresh: bool = False, use_llm: bool = True) -> Optional[Dict[str, Any]]: + existing = None if refresh else latest_identity_for_opportunity(opportunity_id) + if existing: + return existing + return upsert_identity_extraction_for_opportunity(opportunity_id, use_llm=use_llm) diff --git a/app/external_reconciliation_sync.py b/app/external_reconciliation_sync.py new file mode 100644 index 0000000..40eebf6 --- /dev/null +++ b/app/external_reconciliation_sync.py @@ -0,0 +1,994 @@ +"""External API sync for operational reconciliation. + +This module is deliberately conservative: it fetches loose evidence from +Jasmin/Odoo/Packlink and creates reconciliation candidates. It does not create +opportunities, confirm payments, issue invoices or close processes without an +operator decision. +""" +from __future__ import annotations + +import asyncio +import json +import re +from datetime import date, datetime, timedelta, timezone +from decimal import Decimal, InvalidOperation +from typing import Any, Dict, Iterable, List, Optional + +from sqlalchemy import text + +from app.config import settings +from app.db import engine +from app.commercial_service import upsert_customer +from app.reconciliation_service import upsert_reconciliation_item + + +JASMIN_QUOTATION_TYPE = "jasmin_quotation" +JASMIN_INVOICE_TYPE = "jasmin_invoice" +JASMIN_PROFORMA_TYPE = "jasmin_proforma" +ODOO_SALE_ORDER_TYPE = "odoo_sale_order" +PACKLINK_SHIPMENT_TYPE = "packlink_shipment" + + +def _clean(value: Any) -> str: + return str(value or "").strip() + + +def _normalize_tax_id(value: Any) -> str: + raw = _clean(value).upper().replace(" ", "").replace("-", "").replace(".", "") + if raw.startswith("PT"): + raw = raw[2:] + return raw + + +def _first(record: Dict[str, Any], *keys: str) -> Any: + for key in keys: + if key in record and record.get(key) not in (None, ""): + return record.get(key) + return None + + +def _as_list(data: Any) -> List[Dict[str, Any]]: + """Normalize common REST/OData payload shapes into a list of records.""" + if data is None: + return [] + if isinstance(data, list): + return [x for x in data if isinstance(x, dict)] + if isinstance(data, dict): + for key in ("value", "items", "data", "results", "shipments"): + value = data.get(key) + if isinstance(value, list): + return [x for x in value if isinstance(x, dict)] + return [] + + +def _decimal_or_none(value: Any) -> Optional[str]: + raw = _clean(value).replace("€", "").replace(" ", "").replace(",", ".") + if not raw: + return None + try: + return str(Decimal(raw).quantize(Decimal("0.01"))) + except (InvalidOperation, ValueError): + return None + + +def _date_or_none(value: Any) -> Optional[str]: + if not value: + return None + if isinstance(value, datetime): + return value.date().isoformat() + if isinstance(value, date): + return value.isoformat() + text_value = _clean(value) + if not text_value: + return None + # Jasmin/Odoo commonly return ISO timestamps. PostgreSQL DATE accepts the + # YYYY-MM-DD prefix and rejecting unclear values keeps the sync safe. + if len(text_value) >= 10 and text_value[4:5] == "-" and text_value[7:8] == "-": + return text_value[:10] + return None + + +def _date_sort_key(value: Optional[str]) -> str: + # Keep undated records last when sorting newest first. + return value or "0000-00-00" + + +def _recent_window_start(days: int, *, today: Optional[date] = None) -> date: + """Return the inclusive start date for a human "last N days" window. + + Operators read "últimos 3 dias" as today plus the previous two days. + For example, on 2026-06-05, days=3 starts at 2026-06-03, not + 2026-06-02. + """ + days = max(int(days or 1), 1) + return (today or datetime.now(timezone.utc).date()) - timedelta(days=days - 1) + + +def _record_date(record: Dict[str, Any]) -> Optional[str]: + return _date_or_none(_first(record, "documentDate", "date", "creationDate", "postingDate")) + + +def _within_days(record: Dict[str, Any], *, days: int, today: Optional[date] = None) -> bool: + days = max(int(days or 0), 1) + date_value = _record_date(record) + if not date_value: + return False + start = _recent_window_start(days, today=today) + return date_value >= start.isoformat() + + +def _currency(record: Dict[str, Any]) -> str: + value = _first(record, "currency", "currencyKey", "currency_id", "currencyCode") + if isinstance(value, (list, tuple)) and value: + value = value[1] if len(value) > 1 else value[0] + return _clean(value) or "EUR" + + +def _external_id(record: Dict[str, Any], *, fallback_prefix: str) -> str: + value = _first( + record, + "id", + "key", + "naturalKey", + "documentKey", + "documentId", + "externalId", + "name", + "reference", + "shipmentReference", + "tracking_number", + ) + if value: + return _clean(value) + digest = json.dumps(record, ensure_ascii=False, default=str, sort_keys=True)[:120] + return f"{fallback_prefix}:{abs(hash(digest))}" + + +def _document_number(record: Dict[str, Any]) -> str: + composed = _first(record, "documentNumber", "number", "naturalKey", "name", "reference") + if composed: + return _clean(composed) + doc_type = _clean(_first(record, "documentType", "documentTypeKey")) + serie = _clean(_first(record, "serie", "serieKey", "series")) + number = _clean(_first(record, "seriesNumber", "sequenceNumber")) + return " ".join(part for part in [doc_type, serie, number] if part) + + +def _customer_name(record: Dict[str, Any]) -> str: + value = _first( + record, + "customerName", + "partyName", + "buyerCustomerPartyName", + "sellerSupplierPartyName", + "name", + "partner_name", + "recipient_name", + ) + if isinstance(value, (list, tuple)) and value: + value = value[1] if len(value) > 1 else value[0] + return _clean(value) + + +def _customer_email(record: Dict[str, Any]) -> str: + return _clean(_first(record, "customerEmail", "email", "electronicMail", "recipient_email", "partner_email")) + + +def _customer_tax_id(record: Dict[str, Any]) -> str: + """Extract NIF/VAT from common Jasmin/Odoo/Packlink payload shapes.""" + direct = _first( + record, + "customerTaxId", + "customerTaxID", + "customer_tax_id", + "companyTaxID", + "companyTaxId", + "partyTaxId", + "partyTaxID", + "buyerCustomerPartyTaxId", + "buyerCustomerPartyTaxID", + "taxId", + "taxID", + "vat", + "partner_vat", + "nif", + ) + if direct: + return _normalize_tax_id(direct) + for container_key in ("customer", "customerParty", "buyerCustomerParty", "party", "partner", "recipient"): + nested = record.get(container_key) + if isinstance(nested, dict): + nested_value = _first(nested, "tax_id", "taxId", "taxID", "companyTaxID", "vat", "nif") + if nested_value: + return _normalize_tax_id(nested_value) + return "" + + +COMPANY_NAME_MARKERS = ( + "lda", "limitada", "unipessoal", "s.a", " sa", "sociedade anónima", + "sociedade anonima", "sgps", "ace", "crl", "sucursal", "empresa", + "construções", "construcoes", "soluções", "solucoes", "indústria", + "industria", "comércio", "comercio", "portugal", +) + + +def _looks_like_company_name(value: Any) -> bool: + text_value = f" {_clean(value).casefold()} " + if not text_value.strip(): + return False + return any(marker in text_value for marker in COMPANY_NAME_MARKERS) + + +def _external_customer_key(record: Dict[str, Any], *, source_system: str) -> str: + if source_system == "jasmin": + return _clean(_first(record, "partyKey", "customerPartyKey", "naturalKey", "key", "id")) + if source_system == "odoo": + return _clean(_first(record, "partner_external_id", "id")) + return _clean(_first(record, "id", "key", "externalId")) + + +def _looks_like_odoo_sale_order_reference(value: Any) -> bool: + """Return True for Odoo sale order numbers such as S00265. + + Sale order references identify the commercial process, not the fiscal + customer. They must never be stored in customers.name. + """ + return bool(re.fullmatch(r"S\d{4,}", _clean(value).upper())) + + +def _odoo_fiscal_customer_name(record: Dict[str, Any]) -> str: + """Prefer Odoo partner legal name over sale.order.name. + + Recent Odoo sale orders have ``name`` equal to S00xxx and the real fiscal + entity in ``partner_name`` or in the many2one ``partner_id`` display name. + """ + name = _clean(record.get("partner_name")) + if not name: + partner_id = record.get("partner_id") + if isinstance(partner_id, (list, tuple)) and partner_id: + name = _clean(partner_id[1] if len(partner_id) > 1 else partner_id[0]) + if name and not _looks_like_odoo_sale_order_reference(name): + return name + + # res.partner customer seeding records use ``name`` as the partner name. + raw_name = _customer_name(record) + if raw_name and not _looks_like_odoo_sale_order_reference(raw_name): + return raw_name + return "" + + +def _country_code(value: Any) -> str: + if isinstance(value, (list, tuple)) and value: + return _clean(value[1] if len(value) > 1 else value[0]) or "PT" + return _clean(value) or "PT" + + +def _customer_seed_data_from_record(record: Dict[str, Any], *, source_system: str) -> Optional[Dict[str, Any]]: + if source_system == "odoo": + name = _odoo_fiscal_customer_name(record) + else: + name = _customer_name(record) or _clean(record.get("partner_name")) + tax_id = _customer_tax_id(record) + if not name: + return None + + # Never create/update a fiscal customer with an Odoo sale order reference + # (S00265, S00277, ...). That reference belongs to the purchase process. + if source_system == "odoo" and _looks_like_odoo_sale_order_reference(name): + return None + + # Jasmin customer parties are fiscal/commercial customers by definition. + # Odoo res.partner can also contain people/contacts, so require VAT, an + # explicit company flag, or a clear legal-company name before creating a + # fiscal customer in ClientFlow. + is_company = bool(record.get("is_company")) or _clean(record.get("company_type")) == "company" + if source_system == "odoo" and not (tax_id or is_company or _looks_like_company_name(name)): + return None + + external_key = _external_customer_key(record, source_system=source_system) + metadata = { + "seeded_by_reconciliation": True, + "source_system": source_system, + "external_customer_key": external_key, + "raw_customer_payload": record, + } + data = { + "name": name, + "tax_id": tax_id, + "email": _customer_email(record), + "phone": _clean(_first(record, "phone", "telephone", "mobile", "phoneNumber")), + "street_name": _clean(_first(record, "streetName", "street", "address", "street_name")), + "postal_zone": _clean(_first(record, "postalZone", "zip", "postal_code", "postalCode")), + "city_name": _clean(_first(record, "cityName", "city")), + "country": _country_code(_first(record, "country", "country_id", "countryCode")), + "metadata": metadata, + } + if source_system == "jasmin": + data["jasmin_customer_party_key"] = external_key + data["jasmin_customer_id"] = _clean(_first(record, "id", "uuid")) or None + return data + + +def _upsert_fiscal_customer_from_external_record(record: Dict[str, Any], *, source_system: str) -> Optional[str]: + data = _customer_seed_data_from_record(record, source_system=source_system) + if not data: + return None + try: + customer = upsert_customer(data) + return str(customer.get("id") or "") or None + except Exception: + # Customer seeding must never block candidate creation. A document can + # still be reviewed manually if the fiscal customer import fails. + return None + + +def _already_linked_to_opportunity(*, source_system: str, external_id: str) -> bool: + if not external_id: + return False + try: + with engine.begin() as conn: + value = conn.execute(text(""" + SELECT EXISTS ( + SELECT 1 FROM commercial_documents + WHERE system = :source_system + AND external_id = :external_id + AND opportunity_id IS NOT NULL + ) + """), {"source_system": source_system, "external_id": external_id}).scalar() + return bool(value) + except Exception: + # commercial_documents may not exist yet on very old deployments. The + # sync must not fail because of the optional local cache. + return False + + +def _jasmin_external_type(record: Dict[str, Any], default_type: str) -> str: + text_blob = " ".join( + _clean(_first(record, key)) + for key in ("documentType", "documentTypeKey", "documentKind", "naturalKey", "documentNumber") + ).lower() + if "proforma" in text_blob or "pró-forma" in text_blob or "pro-forma" in text_blob: + return JASMIN_PROFORMA_TYPE + if "invoice" in text_blob or "fatura" in text_blob or "factura" in text_blob or "ft" in text_blob.split(): + return JASMIN_INVOICE_TYPE + return default_type + + +def _jasmin_amount(record: Dict[str, Any]) -> Optional[str]: + return _decimal_or_none(_first(record, "payableAmount", "totalAmount", "total", "grossAmount", "amount")) + + +def _jasmin_candidate_from_record(record: Dict[str, Any], *, default_type: str) -> Optional[Dict[str, Any]]: + external_id = _external_id(record, fallback_prefix="jasmin") + if _already_linked_to_opportunity(source_system="jasmin", external_id=external_id): + return None + external_type = _jasmin_external_type(record, default_type) + document_number = _document_number(record) or external_id + customer_name = _customer_name(record) + amount = _jasmin_amount(record) + date_value = _date_or_none(_first(record, "documentDate", "date", "creationDate", "postingDate")) + label = { + JASMIN_QUOTATION_TYPE: "Orçamento Jasmin sem oportunidade", + JASMIN_PROFORMA_TYPE: "Pró-forma Jasmin sem oportunidade", + JASMIN_INVOICE_TYPE: "Fatura Jasmin sem oportunidade", + }.get(external_type, "Documento Jasmin sem oportunidade") + suggested_action = { + JASMIN_QUOTATION_TYPE: "SEND_PROFORMA", + JASMIN_PROFORMA_TYPE: "CONFIRM_PAYMENT", + JASMIN_INVOICE_TYPE: "CONFIRM_PAYMENT", + }.get(external_type, "REVIEW_MANUALLY") + priority = "alta" if external_type in {JASMIN_PROFORMA_TYPE, JASMIN_INVOICE_TYPE} else "normal" + customer_id = _upsert_fiscal_customer_from_external_record(record, source_system="jasmin") + return { + "source_system": "jasmin", + "external_type": external_type, + "external_id": external_id, + "title": f"{label} · {document_number}", + "description": "Documento encontrado via API Jasmin. Operador deve ligar a oportunidade existente, criar oportunidade ou ignorar.", + "priority": priority, + "suggested_action": suggested_action, + "confidence": 0.75 if customer_name else 0.55, + "customer_id": customer_id, + "customer_name": customer_name, + "customer_email": _customer_email(record), + "customer_tax_id": _customer_tax_id(record), + "document_number": document_number, + "document_date": date_value, + "amount": amount, + "currency": _currency(record), + "payload": {"source": "jasmin_api", "record": record}, + "idempotency_key": f"api-sync:jasmin:{external_type}:{external_id}", + } + + +async def sync_jasmin_reconciliation_candidates(*, limit: int = 100, days: int = 3) -> Dict[str, Any]: + """Fetch recent Jasmin quotations/invoices and stage unlinked candidates. + + `days` is intentionally enforced locally after the API call. Some Jasmin + tenants accept OData `$filter`, while others return broader result sets. + The reconciliation queue must only show the recent window requested by the + operator, never historic invoices just because they are first in the API + default order. + """ + if not bool(settings.jasmin_enabled): + return {"source": "jasmin", "enabled": False, "seen": 0, "created_or_updated": 0, "skipped": "JASMIN_ENABLED=false"} + + from app.jasmin_client import JasminClient + + client = JasminClient() + seen = 0 + created = 0 + errors: List[str] = [] + limit = min(max(int(limit or 100), 1), 500) + days = max(int(days or 3), 1) + since = _recent_window_start(days).isoformat() + + async def _fetch(kind: str) -> List[Dict[str, Any]]: + # Fetch a wider first page per document family because final `limit` is + # global after merging quotations and invoices. Ask Jasmin for recent + # documents first, then still validate the date locally. + top = min(max(limit * 2, 20), 100) + odata_filter = f"documentDate ge {since}" + try: + if kind == "quotation": + data = await client.list_quotations(top=top, skip=0, filter=odata_filter, orderby="documentDate desc") + else: + data = await client.list_invoices(top=top, skip=0, filter=odata_filter, orderby="documentDate desc") + return _as_list(data) + except Exception as exc: + errors.append(f"{kind} filtered: {exc}") + try: + # Compatibility fallback for tenants where OData filtering over + # documentDate is not supported. Local filtering still applies. + if kind == "quotation": + data = await client.list_quotations(top=top, skip=0, orderby="documentDate desc") + else: + data = await client.list_invoices(top=top, skip=0, orderby="documentDate desc") + return _as_list(data) + except Exception as fallback_exc: + errors.append(f"{kind}: {fallback_exc}") + return [] + + staged: List[tuple[str, Dict[str, Any]]] = [] + for default_type, records in [ + (JASMIN_QUOTATION_TYPE, await _fetch("quotation")), + (JASMIN_INVOICE_TYPE, await _fetch("invoice")), + ]: + for record in records: + if not _within_days(record, days=days): + continue + staged.append((default_type, record)) + + staged.sort(key=lambda item: _date_sort_key(_record_date(item[1])), reverse=True) + + for default_type, record in staged[:limit]: + seen += 1 + candidate = _jasmin_candidate_from_record(record, default_type=default_type) + if not candidate: + continue + upsert_reconciliation_item(**candidate) + created += 1 + + return { + "source": "jasmin", + "enabled": True, + "seen": seen, + "created_or_updated": created, + "days": days, + "since": since, + "errors": errors, + } + + +def _odoo_partner_name(value: Any) -> str: + if isinstance(value, (list, tuple)) and value: + return _clean(value[1] if len(value) > 1 else value[0]) + return _clean(value) + + +def _odoo_currency(value: Any) -> str: + if isinstance(value, (list, tuple)) and value: + return _clean(value[1] if len(value) > 1 else value[0]) or "EUR" + return _clean(value) or "EUR" + + +def _odoo_m2o_id(value: Any) -> Optional[int]: + if isinstance(value, (list, tuple)) and value: + try: + return int(value[0]) + except (TypeError, ValueError): + return None + try: + return int(value) + except (TypeError, ValueError): + return None + + +def _odoo_m2o_name(value: Any) -> str: + if isinstance(value, (list, tuple)) and value: + return _clean(value[1] if len(value) > 1 else value[0]) + return _clean(value) + + +def _odoo_search_read_safe(client: Any, model: str, domain: List[Any], fields: List[str], *, limit: int = 100, order: str = "id asc") -> List[Dict[str, Any]]: + try: + rows = client.search_read(model, domain=domain, fields=fields, limit=limit, order=order) + return [dict(row) for row in rows] + except Exception: + return [] + + +def _odoo_fetch_sale_lines(client: Any, sale_id: int) -> List[Dict[str, Any]]: + if not sale_id: + return [] + base_fields = ["id", "order_id", "product_id", "name", "product_uom_qty", "price_unit", "price_total"] + extended_fields = base_fields + ["qty_delivered", "qty_invoiced"] + rows = _odoo_search_read_safe(client, "sale.order.line", [["order_id", "=", sale_id]], extended_fields, limit=200, order="id asc") + if not rows: + rows = _odoo_search_read_safe(client, "sale.order.line", [["order_id", "=", sale_id]], base_fields, limit=200, order="id asc") + normalized = [] + for row in rows: + normalized.append({ + "id": row.get("id"), + "product_id": _odoo_m2o_id(row.get("product_id")), + "product_name": _odoo_m2o_name(row.get("product_id")) or _clean(row.get("name")), + "description": _clean(row.get("name")), + "qty_ordered": row.get("product_uom_qty"), + "qty_delivered": row.get("qty_delivered"), + "qty_invoiced": row.get("qty_invoiced"), + "price_unit": row.get("price_unit"), + "price_total": row.get("price_total"), + }) + return normalized + + +def _odoo_fetch_pickings(client: Any, sale_name: str) -> List[Dict[str, Any]]: + if not sale_name: + return [] + fields = ["id", "name", "state", "origin", "picking_type_id", "scheduled_date", "date_done"] + rows = _odoo_search_read_safe(client, "stock.picking", [["origin", "ilike", sale_name]], fields, limit=100, order="id desc") + return [ + { + "id": row.get("id"), + "name": row.get("name"), + "state": row.get("state"), + "origin": row.get("origin"), + "type": _odoo_m2o_name(row.get("picking_type_id")), + "scheduled_date": row.get("scheduled_date"), + "date_done": row.get("date_done"), + } + for row in rows + ] + + +def _odoo_fetch_productions(client: Any, sale_name: str) -> List[Dict[str, Any]]: + if not sale_name: + return [] + fields = ["id", "name", "state", "origin", "product_id", "product_qty", "date_start", "date_finished"] + rows = _odoo_search_read_safe(client, "mrp.production", [["origin", "ilike", sale_name]], fields, limit=100, order="id desc") + return [ + { + "id": row.get("id"), + "name": row.get("name"), + "state": row.get("state"), + "origin": row.get("origin"), + "product_name": _odoo_m2o_name(row.get("product_id")), + "qty": row.get("product_qty"), + "date_start": row.get("date_start"), + "date_finished": row.get("date_finished"), + } + for row in rows + ] + + +def _odoo_is_outgoing_picking(picking: Dict[str, Any]) -> bool: + name = _clean(picking.get("name")).upper() + picking_type = _clean(picking.get("type")).lower() + if "/OUT/" in name or name.startswith("WH/OUT"): + return True + return any(keyword in picking_type for keyword in ["delivery", "outgoing", "entrega", "expedi", "saída", "saida"]) + + +def _odoo_derive_fulfilment(record: Dict[str, Any]) -> Dict[str, Any]: + lines = record.get("order_lines") if isinstance(record.get("order_lines"), list) else [] + pickings = record.get("pickings") if isinstance(record.get("pickings"), list) else [] + productions = record.get("productions") if isinstance(record.get("productions"), list) else [] + outgoing = [p for p in pickings if _odoo_is_outgoing_picking(p)] or pickings + delivery_done = bool(outgoing) and all(_clean(p.get("state")) == "done" for p in outgoing) + delivery_ready = any(_clean(p.get("state")) == "assigned" for p in outgoing) + production_active = any(_clean(p.get("state")) in {"progress", "to_close", "confirmed"} for p in productions) + invoice_status = _clean(record.get("invoice_status")) + has_uninvoiced_lines = False + for line in lines: + ordered = line.get("qty_ordered") or 0 + invoiced = line.get("qty_invoiced") + try: + if invoiced is not None and float(invoiced) < float(ordered or 0): + has_uninvoiced_lines = True + except (TypeError, ValueError): + pass + invoice_pending = invoice_status in {"to invoice", "no"} or has_uninvoiced_lines + physical_status = "order_created" + label = "Venda criada" + stage = "ODOO_ORDER_CREATED" + if delivery_done: + physical_status = "shipped" + label = "Entrega concluída no Odoo" + stage = "SHIPMENT_CREATED" + elif delivery_ready: + physical_status = "ready_to_ship" + label = "Pronta para despacho" + stage = "READY_TO_SHIP" + elif production_active: + physical_status = "in_production" + label = "Em produção/preparação" + stage = "IN_PRODUCTION" + return { + "physical_status": physical_status, + "label": label, + "stage": stage, + "delivery_done": delivery_done, + "delivery_ready": delivery_ready, + "invoice_pending": invoice_pending, + "invoice_status": invoice_status, + "next_action": "SEND_INVOICE" if invoice_pending else "REVIEW_MANUALLY", + "outgoing_pickings": outgoing, + "lines": lines, + "productions": productions, + } + + +def _odoo_candidate_from_record(record: Dict[str, Any]) -> Dict[str, Any]: + external_id = _clean(_first(record, "id", "name")) + order_name = _clean(_first(record, "name", "client_order_ref")) or f"Odoo #{external_id}" + customer_name = _clean(record.get("partner_name")) or _odoo_partner_name(record.get("partner_id")) + state = _clean(record.get("state")) + invoice_status = _clean(record.get("invoice_status")) + amount = _decimal_or_none(record.get("amount_total")) + fulfilment = record.get("fulfilment") if isinstance(record.get("fulfilment"), dict) else _odoo_derive_fulfilment(record) + invoice_pending = bool(fulfilment.get("invoice_pending")) or invoice_status in {"to invoice", "no"} + customer_id = _upsert_fiscal_customer_from_external_record(record, source_system="odoo") + return { + "source_system": "odoo", + "external_type": ODOO_SALE_ORDER_TYPE, + "external_id": external_id or order_name, + "title": f"Venda Odoo sem oportunidade · {order_name}", + "description": "Venda/encomenda encontrada via API Odoo. Operador deve ligar a oportunidade existente, criar oportunidade ou ignorar.", + "priority": "alta" if state in {"sale", "done"} and invoice_pending else "normal", + "suggested_action": "SEND_INVOICE" if invoice_pending else "REVIEW_MANUALLY", + "confidence": 0.70 if customer_name else 0.50, + "customer_id": customer_id, + "customer_name": customer_name, + "customer_email": _customer_email(record), + "customer_tax_id": _customer_tax_id(record), + "document_number": order_name, + "document_date": _date_or_none(record.get("date_order") or record.get("create_date")), + "amount": amount, + "currency": _odoo_currency(record.get("currency_id")), + "payload": {"source": "odoo_api", "record": record}, + "idempotency_key": f"api-sync:odoo:sale_order:{external_id or order_name}", + } + + +def sync_odoo_reconciliation_candidates(*, limit: int = 100, days: int = 3) -> Dict[str, Any]: + """Fetch recent Odoo sale orders and stage unlinked candidates.""" + if not bool(settings.odoo_enabled): + return {"source": "odoo", "enabled": False, "seen": 0, "created_or_updated": 0, "skipped": "ODOO_ENABLED=false"} + + from app.odoo_client import OdooClient + + client = OdooClient() + days = max(int(days or 3), 1) + since = _recent_window_start(days).strftime("%Y-%m-%d") + domain = [["date_order", ">=", since], ["state", "in", ["sale", "done"]]] + fields = [ + "id", + "name", + "partner_id", + "amount_total", + "currency_id", + "date_order", + "state", + "invoice_status", + "client_order_ref", + "create_date", + "order_line", + ] + records = client.search_read("sale.order", domain=domain, fields=fields, limit=limit, order="date_order desc") + + # sale.order does not reliably expose partner VAT/NIF in all Odoo editions. + # Fetch it from res.partner and enrich the staging record before matching. + partner_ids: List[int] = [] + for record in records: + partner = record.get("partner_id") + if isinstance(partner, (list, tuple)) and partner: + try: + partner_ids.append(int(partner[0])) + except (TypeError, ValueError): + pass + partner_vat_by_id: Dict[int, str] = {} + partner_email_by_id: Dict[int, str] = {} + partner_name_by_id: Dict[int, str] = {} + if partner_ids: + partners = client.search_read( + "res.partner", + domain=[["id", "in", sorted(set(partner_ids))]], + fields=["id", "vat", "email", "name"], + limit=len(set(partner_ids)), + order="id asc", + ) + for partner in partners: + try: + partner_id = int(partner.get("id")) + except (TypeError, ValueError): + continue + partner_vat_by_id[partner_id] = _normalize_tax_id(partner.get("vat")) + partner_email_by_id[partner_id] = _clean(partner.get("email")) + partner_name_by_id[partner_id] = _clean(partner.get("name")) + + seen = 0 + created = 0 + for record in records: + partner = record.get("partner_id") + if isinstance(partner, (list, tuple)) and partner: + try: + partner_id = int(partner[0]) + record["partner_vat"] = partner_vat_by_id.get(partner_id, "") + record["partner_email"] = partner_email_by_id.get(partner_id, "") + record["partner_name"] = partner_name_by_id.get(partner_id, "") + record["partner_external_id"] = partner_id + except (TypeError, ValueError): + pass + try: + sale_id = int(record.get("id")) + except (TypeError, ValueError): + sale_id = 0 + sale_name = _clean(record.get("name")) + record["order_lines"] = _odoo_fetch_sale_lines(client, sale_id) + record["pickings"] = _odoo_fetch_pickings(client, sale_name) + record["productions"] = _odoo_fetch_productions(client, sale_name) + record["fulfilment"] = _odoo_derive_fulfilment(record) + seen += 1 + candidate = _odoo_candidate_from_record(record) + upsert_reconciliation_item(**candidate) + created += 1 + return {"source": "odoo", "enabled": True, "seen": seen, "created_or_updated": created, "days": max(int(days), 1), "since": since} + + +def _packlink_candidate_from_record(record: Dict[str, Any]) -> Dict[str, Any]: + external_id = _external_id(record, fallback_prefix="packlink") + reference = _clean(_first(record, "reference", "shipmentReference", "id", "tracking_number")) or external_id + recipient = record.get("to") if isinstance(record.get("to"), dict) else {} + customer_name = _customer_name(record) or _clean(_first(recipient, "name", "contactName", "company")) + email = _customer_email(record) or _clean(_first(recipient, "email")) + amount = _decimal_or_none(_first(record, "price", "totalPrice", "amount")) + customer_id = _upsert_fiscal_customer_from_external_record(record, source_system="packlink") + return { + "source_system": "packlink", + "external_type": PACKLINK_SHIPMENT_TYPE, + "external_id": external_id, + "title": f"Envio Packlink sem oportunidade · {reference}", + "description": "Envio/tracking encontrado via API Packlink. Operador deve ligar a oportunidade existente ou ignorar.", + "priority": "normal", + "suggested_action": "REVIEW_MANUALLY", + "confidence": 0.55, + "customer_id": customer_id, + "customer_name": customer_name, + "customer_email": email, + "customer_tax_id": _customer_tax_id(record), + "document_number": reference, + "document_date": _date_or_none(_first(record, "createdAt", "created_at", "date")), + "amount": amount, + "currency": _currency(record), + "payload": {"source": "packlink_api", "record": record}, + "idempotency_key": f"api-sync:packlink:shipment:{external_id}", + } + + +async def sync_packlink_reconciliation_candidates(*, limit: int = 100, days: int = 3) -> Dict[str, Any]: + """Fetch recent Packlink shipments if the API/list endpoint is available.""" + if not bool(settings.packlink_enabled): + return {"source": "packlink", "enabled": False, "seen": 0, "created_or_updated": 0, "skipped": "PACKLINK_ENABLED=false"} + + from app.packlink_client import PacklinkClient + + client = PacklinkClient() + try: + if hasattr(client, "list_shipments"): + data = await client.list_shipments(limit=limit) + else: + data = await client._request("GET", "/shipments", params={"limit": limit}) # type: ignore[attr-defined] + except Exception as exc: + return {"source": "packlink", "enabled": True, "seen": 0, "created_or_updated": 0, "errors": [str(exc)]} + + days = max(int(days or 3), 1) + records = [] + for record in _as_list(data): + if _within_days(record, days=days): + records.append(record) + if len(records) >= int(limit): + break + created = 0 + for record in records: + upsert_reconciliation_item(**_packlink_candidate_from_record(record)) + created += 1 + return {"source": "packlink", "enabled": True, "seen": len(records), "created_or_updated": created, "days": days} + + +async def sync_jasmin_fiscal_customers_for_reconciliation(*, limit: int = 200) -> Dict[str, Any]: + """Seed ClientFlow fiscal customers from Jasmin before document matching.""" + if not bool(settings.jasmin_enabled): + return {"source": "jasmin_customers", "enabled": False, "seen": 0, "created_or_updated": 0, "skipped": "JASMIN_ENABLED=false"} + from app.jasmin_client import JasminClient + + client = JasminClient() + seen = 0 + created = 0 + errors: List[str] = [] + try: + data = await client.list_customers_odata(top=min(max(int(limit or 200), 1), 100), skip=0) + records = _as_list(data) + except Exception as exc: + errors.append(str(exc)) + records = [] + for record in records[: max(int(limit or 200), 1)]: + seen += 1 + if _upsert_fiscal_customer_from_external_record(record, source_system="jasmin"): + created += 1 + return {"source": "jasmin_customers", "enabled": True, "seen": seen, "created_or_updated": created, "errors": errors} + + +def _odoo_model_field_names(client: Any, model: str, errors: List[str]) -> Optional[set]: + """Return available Odoo model fields, or None if discovery is unavailable. + + Odoo installations can differ by edition/module set. In the production + instance that triggered v4.9.25.2, res.partner did not expose ``mobile``; + asking search_read for that optional field made the entire customer seeding + phase fail even though quotations/sales could still sync. + """ + try: + result = client.execute_kw(model, "fields_get", [], {"attributes": ["type"]}) + except Exception as exc: + errors.append(f"Odoo {model}.fields_get falhou; a usar campos compatíveis por defeito: {exc}") + return None + if isinstance(result, dict): + return set(result.keys()) + return None + + +def _odoo_partner_fields_for_available_schema(available_fields: Optional[set]) -> List[str]: + requested = [ + "id", + "name", + "vat", + "email", + "phone", + "mobile", + "street", + "zip", + "city", + "country_id", + "is_company", + "company_type", + ] + if available_fields is None: + # Most compatible fallback: do not request mobile unless we confirmed it + # exists. This prevents "Invalid field 'mobile' on 'res.partner'". + return [field for field in requested if field != "mobile"] + return [field for field in requested if field in available_fields] + + +def _odoo_partner_domains_for_available_schema(available_fields: Optional[set]) -> List[List[Any]]: + domains: List[List[Any]] = [] + if available_fields is None or {"is_company", "vat"}.issubset(available_fields): + domains.append(["|", ["is_company", "=", True], ["vat", "!=", False]]) + elif "vat" in available_fields: + domains.append([["vat", "!=", False]]) + + if available_fields is None or "customer_rank" in available_fields: + domains.append([["customer_rank", ">", 0]]) + + # Final fallback: fetch recent partners and let _customer_seed_data... filter + # people/contacts out before creating fiscal customers. + domains.append([]) + return domains + + +def sync_odoo_fiscal_customers_for_reconciliation(*, limit: int = 200) -> Dict[str, Any]: + """Seed ClientFlow fiscal customers from Odoo companies/partners. + + v4.9.25.2 makes this phase schema-compatible with Odoo installs that do + not have optional partner fields such as ``mobile``. + """ + if not bool(settings.odoo_enabled): + return {"source": "odoo_customers", "enabled": False, "seen": 0, "created_or_updated": 0, "skipped": "ODOO_ENABLED=false"} + from app.odoo_client import OdooClient + + client = OdooClient() + errors: List[str] = [] + available_fields = _odoo_model_field_names(client, "res.partner", errors) + fields = _odoo_partner_fields_for_available_schema(available_fields) + domains = _odoo_partner_domains_for_available_schema(available_fields) + ignored_optional_fields = [] + if available_fields is not None: + ignored_optional_fields = [field for field in ("mobile",) if field not in available_fields] + + records: List[Dict[str, Any]] = [] + for domain in domains: + try: + records = client.search_read("res.partner", domain=domain, fields=fields, limit=limit, order="write_date desc") + break + except Exception as exc: + errors.append(str(exc)) + records = [] + seen = 0 + created = 0 + for record in records[: max(int(limit or 200), 1)]: + seen += 1 + record["partner_external_id"] = record.get("id") + if _upsert_fiscal_customer_from_external_record(record, source_system="odoo"): + created += 1 + return { + "source": "odoo_customers", + "enabled": True, + "seen": seen, + "created_or_updated": created, + "errors": errors, + "ignored_optional_fields": ignored_optional_fields, + } + + +async def sync_external_fiscal_customers_for_reconciliation(*, limit: int = 200) -> Dict[str, Any]: + """Phase 1 of reconciliation: create/update fiscal customers first.""" + jasmin_result = await sync_jasmin_fiscal_customers_for_reconciliation(limit=limit) + odoo_result = await asyncio.to_thread(sync_odoo_fiscal_customers_for_reconciliation, limit=limit) + results = [jasmin_result, odoo_result] + return { + "seen": sum(int(r.get("seen") or 0) for r in results), + "created_or_updated": sum(int(r.get("created_or_updated") or 0) for r in results), + "results": results, + } + + +async def sync_all_external_reconciliation_candidates(*, limit: int = 100, days: int = 3) -> Dict[str, Any]: + """Run the ordered reconciliation pipeline. + + v4.9.25 adds Phase 0: enrich open ClientFlow opportunities with fiscal + customers before trying to match Jasmin/Odoo documents. This makes the + rest of the pipeline use stronger keys (NIF/fiscal name/customer id) + instead of relying on contact names. + + Phase 1 seeds fiscal customers from Jasmin/Odoo. Phase 2 stages Jasmin + quotations/proformas/invoices. Phase 3 stages Odoo sales/orders. Phase 4 + stages shipment evidence. Process cards are then reconstructed by the + reconciliation service from customer identity + purchase-operation anchors. + """ + try: + from app.fiscal_enrichment_service import enrich_open_opportunities + enrichment_result = await asyncio.to_thread( + enrich_open_opportunities, + limit=max(min(limit, 200), 50), + apply_safe=True, + mode="pre_reconciliation", + ) + except Exception as exc: + enrichment_result = {"seen": 0, "enriched": 0, "suggested": 0, "auto_applied": 0, "errors": [str(exc)]} + + customer_result = await sync_external_fiscal_customers_for_reconciliation(limit=max(limit, 200)) + jasmin_result = await sync_jasmin_reconciliation_candidates(limit=limit, days=days) + # Odoo client is synchronous XML-RPC; run it in a thread to keep async routes responsive. + odoo_result = await asyncio.to_thread(sync_odoo_reconciliation_candidates, limit=limit, days=days) + packlink_result = await sync_packlink_reconciliation_candidates(limit=limit, days=days) + document_results = [jasmin_result, odoo_result, packlink_result] + return { + "seen": sum(int(r.get("seen") or 0) for r in document_results), + "created_or_updated": sum(int(r.get("created_or_updated") or 0) for r in document_results), + "enrichment_seen": int(enrichment_result.get("seen") or 0), + "enrichment_suggested": int(enrichment_result.get("suggested") or 0), + "enrichment_auto_applied": int(enrichment_result.get("auto_applied") or 0), + "enrichment_result": enrichment_result, + "customer_seen": int(customer_result.get("seen") or 0), + "customers_created_or_updated": int(customer_result.get("created_or_updated") or 0), + "customer_results": customer_result.get("results") or [], + "results": document_results, + } diff --git a/app/fiscal_enrichment_service.py b/app/fiscal_enrichment_service.py new file mode 100644 index 0000000..06b9dbb --- /dev/null +++ b/app/fiscal_enrichment_service.py @@ -0,0 +1,1267 @@ +"""Fiscal enrichment worker for ClientFlow opportunities. + +v4.9.25 adds a small, autonomous layer that runs before reconciliation: + +- read open opportunities without a fiscal customer; +- use local data/cache first; +- consult the external company/contact lookup API when enabled; +- create an auditable suggestion; +- auto-associate only very strong, non-conflicting matches. + +The service is intentionally conservative. It enriches the fiscal identity used +by Jasmin/Odoo reconciliation; it does not create/close opportunities or alter +commercial stages. +""" +from __future__ import annotations + +import hashlib +import json +import re +import time +import urllib.error +import urllib.parse +import urllib.request +from datetime import datetime, timezone +from typing import Any, Dict, Iterable, List, Optional, Tuple + +from sqlalchemy import text + +from app.config import settings +from app.db import engine +from app.commercial_service import ( + get_customer_by_tax_id, + link_customer_to_opportunity, + normalize_fiscal_name, + normalize_tax_id, + upsert_customer, +) +from app.opportunity_service import ensure_opportunity_schema, get_opportunity, list_opportunities +from app.email_identity_extraction_service import extract_identity_for_opportunity, is_plausible_company_mention + + +CACHE_SOURCE = "informa_pipeline_api" +DEFAULT_AUTO_THRESHOLD = 95.0 +PUBLIC_EMAIL_DOMAINS = { + "gmail.com", "googlemail.com", "hotmail.com", "hotmail.pt", "outlook.com", + "outlook.pt", "live.com", "msn.com", "icloud.com", "me.com", "mac.com", + "yahoo.com", "yahoo.pt", "sapo.pt", "mail.com", "proton.me", "protonmail.com", + "aol.com", "gmx.com", "gmx.net", "uol.com.br", +} +VERY_STRONG_MATCH_TYPES = { + "nif_exato", + "email_exato", + "contacto_email_exato", + "email_principal_exato", + "email_exato_empresa_inferida", + "email_exato_empresa_associada", + "empresa_email_principal_exato", + "email_identity_company_internal", +} +STRONG_MATCH_TYPES = VERY_STRONG_MATCH_TYPES | { + "nome_exato", + "email_principal_dominio", + "website_dominio", + "contacto_email_dominio", +} +MEDIUM_MATCH_TYPES = STRONG_MATCH_TYPES | {"prefixo_nome", "parte_nome", "dominio", "email_dominio", "contacto"} +DOMAIN_ONLY_MATCH_TYPES = {"email_principal_dominio", "email_dominio_empresa_associada", "contacto_email_dominio", "dominio", "email_dominio", "website_dominio"} + +_SCHEMA_READY = False + + +def _clean(value: Any) -> str: + return str(value or "").strip() + + +def _json(value: Any) -> str: + return json.dumps(value or {}, ensure_ascii=False, default=str) + + +def _float(value: Any, default: float = 0.0) -> float: + try: + return float(value) + except (TypeError, ValueError): + return default + + +def _now_iso() -> str: + return datetime.now(timezone.utc).isoformat() + + +def _sha256_payload(value: Any) -> str: + raw = json.dumps(value or {}, ensure_ascii=False, sort_keys=True, default=str) + return hashlib.sha256(raw.encode("utf-8")).hexdigest() + + +def _domain_from_email(email: Any) -> str: + email_value = _clean(email).lower() + if "@" not in email_value: + return "" + return normalize_domain(email_value.rsplit("@", 1)[1]) + + +def normalize_domain(value: Any) -> str: + raw = _clean(value).lower() + if not raw: + return "" + raw = raw.replace("https://", "").replace("http://", "") + raw = raw.split("/", 1)[0].split("?", 1)[0].split("#", 1)[0] + raw = raw.strip(". ") + if raw.startswith("www."): + raw = raw[4:] + return raw + + +def is_public_email_domain(domain: Any) -> bool: + return normalize_domain(domain) in PUBLIC_EMAIL_DOMAINS + + +def ensure_fiscal_enrichment_schema() -> None: + """Create the cache/suggestion/audit tables used by the worker.""" + global _SCHEMA_READY + if _SCHEMA_READY: + return + ensure_opportunity_schema() + with engine.begin() as conn: + conn.execute(text(""" + CREATE TABLE IF NOT EXISTS external_company_cache ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + source TEXT NOT NULL DEFAULT 'informa_pipeline_api', + lookup_type TEXT NOT NULL, + lookup_value TEXT NOT NULL, + external_id TEXT, + nif TEXT, + legal_name TEXT, + normalized_name TEXT, + cae TEXT, + address TEXT, + postcode TEXT, + city TEXT, + district TEXT, + country TEXT, + website TEXT, + phone TEXT, + email TEXT, + duns TEXT, + status TEXT, + score NUMERIC(5,2), + match_type TEXT, + raw_payload JSONB NOT NULL DEFAULT '{}'::jsonb, + payload_hash TEXT, + fetched_at TIMESTAMPTZ NOT NULL DEFAULT now(), + expires_at TIMESTAMPTZ + ) + """)) + for stmt in [ + "ALTER TABLE external_company_cache ADD COLUMN IF NOT EXISTS payload_hash TEXT", + "ALTER TABLE external_company_cache ADD COLUMN IF NOT EXISTS expires_at TIMESTAMPTZ", + "CREATE UNIQUE INDEX IF NOT EXISTS ux_external_company_cache_lookup ON external_company_cache(source, lookup_type, lookup_value)", + "CREATE INDEX IF NOT EXISTS idx_external_company_cache_nif ON external_company_cache(nif) WHERE nif IS NOT NULL AND nif <> ''", + "CREATE INDEX IF NOT EXISTS idx_external_company_cache_name ON external_company_cache(normalized_name)", + ]: + conn.execute(text(stmt)) + + conn.execute(text(""" + CREATE TABLE IF NOT EXISTS fiscal_customer_suggestions ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + opportunity_id UUID REFERENCES opportunities(id) ON DELETE CASCADE, + reconciliation_item_id UUID, + suggested_customer_id UUID REFERENCES customers(id) ON DELETE SET NULL, + suggested_nif TEXT, + suggested_name TEXT NOT NULL, + source TEXT NOT NULL DEFAULT 'informa_pipeline_api', + lookup_type TEXT, + lookup_value TEXT, + match_type TEXT, + confidence NUMERIC(5,2) NOT NULL DEFAULT 0, + reason TEXT, + status TEXT NOT NULL DEFAULT 'pending', + auto_applied BOOLEAN NOT NULL DEFAULT FALSE, + raw_payload JSONB NOT NULL DEFAULT '{}'::jsonb, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + resolved_at TIMESTAMPTZ, + resolved_by TEXT + ) + """)) + for stmt in [ + "ALTER TABLE fiscal_customer_suggestions ADD COLUMN IF NOT EXISTS suggested_customer_id UUID REFERENCES customers(id) ON DELETE SET NULL", + "ALTER TABLE fiscal_customer_suggestions ADD COLUMN IF NOT EXISTS auto_applied BOOLEAN NOT NULL DEFAULT FALSE", + "ALTER TABLE fiscal_customer_suggestions ADD COLUMN IF NOT EXISTS lookup_type TEXT", + "ALTER TABLE fiscal_customer_suggestions ADD COLUMN IF NOT EXISTS lookup_value TEXT", + "ALTER TABLE fiscal_customer_suggestions ADD COLUMN IF NOT EXISTS resolved_by TEXT", + "CREATE INDEX IF NOT EXISTS idx_fiscal_customer_suggestions_opp ON fiscal_customer_suggestions(opportunity_id, status)", + "CREATE INDEX IF NOT EXISTS idx_fiscal_customer_suggestions_status ON fiscal_customer_suggestions(status, created_at DESC)", + "CREATE UNIQUE INDEX IF NOT EXISTS ux_fiscal_customer_suggestion_opp_nif_pending ON fiscal_customer_suggestions(opportunity_id, suggested_nif) WHERE suggested_nif IS NOT NULL AND suggested_nif <> '' AND status = 'pending'", + ]: + conn.execute(text(stmt)) + + conn.execute(text(""" + CREATE TABLE IF NOT EXISTS fiscal_enrichment_runs ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + mode TEXT NOT NULL DEFAULT 'incremental', + status TEXT NOT NULL DEFAULT 'finished', + seen INTEGER NOT NULL DEFAULT 0, + enriched INTEGER NOT NULL DEFAULT 0, + suggested INTEGER NOT NULL DEFAULT 0, + auto_applied INTEGER NOT NULL DEFAULT 0, + skipped INTEGER NOT NULL DEFAULT 0, + errors JSONB NOT NULL DEFAULT '[]'::jsonb, + payload JSONB NOT NULL DEFAULT '{}'::jsonb, + started_at TIMESTAMPTZ NOT NULL DEFAULT now(), + finished_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + """)) + _SCHEMA_READY = True + + +def _external_enabled() -> bool: + return bool(getattr(settings, "external_company_lookup_enabled", False)) and bool(_clean(getattr(settings, "external_company_lookup_base_url", ""))) + + +def _api_base_url() -> str: + return _clean(getattr(settings, "external_company_lookup_base_url", "")).rstrip("/") + + +def _api_key() -> str: + return _clean(getattr(settings, "external_company_lookup_api_key", "")) + + +def _timeout() -> float: + return max(float(getattr(settings, "external_company_lookup_timeout", 10) or 10), 1.0) + + +def _auto_threshold() -> float: + return float(getattr(settings, "external_company_lookup_auto_threshold", DEFAULT_AUTO_THRESHOLD) or DEFAULT_AUTO_THRESHOLD) + + +def _http_json(method: str, path: str, *, params: Optional[Dict[str, Any]] = None, payload: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: + if not _external_enabled(): + return {"_skipped": "EXTERNAL_COMPANY_LOOKUP_ENABLED=false"} + url = _api_base_url() + path + if params: + query = urllib.parse.urlencode({k: v for k, v in params.items() if v not in (None, "")}, doseq=True) + if query: + url += "?" + query + headers = {"Accept": "application/json"} + if _api_key(): + headers["X-API-Key"] = _api_key() + data = None + if payload is not None: + data = json.dumps(payload, ensure_ascii=False).encode("utf-8") + headers["Content-Type"] = "application/json" + request = urllib.request.Request(url, data=data, method=method.upper(), headers=headers) + try: + with urllib.request.urlopen(request, timeout=_timeout()) as response: # noqa: S310 - configured internal API endpoint + raw = response.read().decode("utf-8") + return json.loads(raw) if raw else {} + except urllib.error.HTTPError as exc: + if exc.code == 404: + return {"_not_found": True, "status_code": 404} + body = "" + try: + body = exc.read().decode("utf-8")[:500] + except Exception: + body = "" + return {"_error": f"HTTP {exc.code}", "body": body} + except Exception as exc: + return {"_error": str(exc)} + + +def _company_from_contact_item(item: Dict[str, Any]) -> Optional[Dict[str, Any]]: + empresa = item.get("empresa") + if not isinstance(empresa, dict) or not empresa: + return None + company = dict(empresa) + company["score"] = _float(item.get("confidence"), _float(company.get("score"), 0.0)) + company["match_type"] = _clean(item.get("match_type") or item.get("empresa_resolution_type") or company.get("match_type")) + company["empresa_resolution_type"] = item.get("empresa_resolution_type") + company["contacto_payload"] = item.get("contacto") or {} + return company + + +def _best_company_from_response(data: Dict[str, Any]) -> Optional[Dict[str, Any]]: + if not isinstance(data, dict) or data.get("_error") or data.get("_skipped"): + return None + if data.get("nome") or data.get("nif"): + company = dict(data) + company.setdefault("score", 100.0 if data.get("nif") else 0.0) + company.setdefault("match_type", "nif_exato" if data.get("nif") else "") + return company + contacts = data.get("contactos") + if isinstance(contacts, list) and contacts: + candidates = [_company_from_contact_item(x) for x in contacts if isinstance(x, dict)] + candidates = [x for x in candidates if x] + if candidates: + return sorted(candidates, key=lambda c: _float(c.get("score")), reverse=True)[0] + companies = data.get("empresas") or data.get("matches") + if isinstance(companies, list) and companies: + candidates = [x for x in companies if isinstance(x, dict)] + if candidates: + return sorted(candidates, key=lambda c: _float(c.get("score")), reverse=True)[0] + return None + + +def _company_to_customer_data(company: Dict[str, Any]) -> Dict[str, Any]: + return { + "name": _clean(company.get("nome") or company.get("legal_name") or company.get("name")), + "tax_id": normalize_tax_id(company.get("nif") or company.get("tax_id")), + "email": _clean(company.get("email_principal") or company.get("email")), + "phone": _clean(company.get("telefone") or company.get("phone")), + "street_name": _clean(company.get("morada") or company.get("address")), + "postal_zone": _clean(company.get("codigo_postal") or company.get("postcode")), + "city_name": _clean(company.get("localidade") or company.get("city")), + "country": _clean(company.get("pais") or company.get("country") or "PT") or "PT", + "metadata": { + "fiscal_enrichment_source": CACHE_SOURCE, + "external_company_id": company.get("id"), + "cae": company.get("cae"), + "duns": company.get("duns"), + "website": company.get("website"), + "district": company.get("distrito") or company.get("district"), + "einforma_estado": company.get("einforma_estado"), + "last_fiscal_enrichment_at": _now_iso(), + }, + } + + +def _cache_lookup(lookup_type: str, lookup_value: str) -> Optional[Dict[str, Any]]: + ensure_fiscal_enrichment_schema() + with engine.begin() as conn: + row = conn.execute(text(""" + SELECT raw_payload + FROM external_company_cache + WHERE source = :source AND lookup_type = :lookup_type AND lookup_value = :lookup_value + AND (expires_at IS NULL OR expires_at > now()) + ORDER BY fetched_at DESC + LIMIT 1 + """), {"source": CACHE_SOURCE, "lookup_type": lookup_type, "lookup_value": lookup_value}).mappings().first() + if row and isinstance(row.get("raw_payload"), dict): + return dict(row["raw_payload"]) + return None + + +def _cache_company(lookup_type: str, lookup_value: str, company: Dict[str, Any]) -> None: + ensure_fiscal_enrichment_schema() + if not lookup_value or not company: + return + normalized_name = normalize_fiscal_name(company.get("nome") or company.get("legal_name") or company.get("name")) + params = { + "source": CACHE_SOURCE, + "lookup_type": lookup_type, + "lookup_value": lookup_value, + "external_id": _clean(company.get("id")), + "nif": normalize_tax_id(company.get("nif") or company.get("tax_id")) or None, + "legal_name": _clean(company.get("nome") or company.get("legal_name") or company.get("name")) or None, + "normalized_name": normalized_name or None, + "cae": _clean(company.get("cae")) or None, + "address": _clean(company.get("morada") or company.get("address")) or None, + "postcode": _clean(company.get("codigo_postal") or company.get("postcode")) or None, + "city": _clean(company.get("localidade") or company.get("city")) or None, + "district": _clean(company.get("distrito") or company.get("district")) or None, + "country": _clean(company.get("pais") or company.get("country") or "PT") or "PT", + "website": _clean(company.get("website")) or None, + "phone": _clean(company.get("telefone") or company.get("phone")) or None, + "email": _clean(company.get("email_principal") or company.get("email")) or None, + "duns": _clean(company.get("duns")) or None, + "status": _clean(company.get("einforma_estado") or company.get("status")) or None, + "score": _float(company.get("score"), 0.0), + "match_type": _clean(company.get("match_type")) or None, + "raw_payload": _json(company), + "payload_hash": _sha256_payload(company), + } + with engine.begin() as conn: + conn.execute(text(""" + INSERT INTO external_company_cache ( + source, lookup_type, lookup_value, external_id, nif, legal_name, normalized_name, + cae, address, postcode, city, district, country, website, phone, email, duns, + status, score, match_type, raw_payload, payload_hash, fetched_at, expires_at + ) VALUES ( + :source, :lookup_type, :lookup_value, :external_id, :nif, :legal_name, :normalized_name, + :cae, :address, :postcode, :city, :district, :country, :website, :phone, :email, :duns, + :status, :score, :match_type, CAST(:raw_payload AS JSONB), :payload_hash, now(), now() + interval '30 days' + ) + ON CONFLICT (source, lookup_type, lookup_value) + DO UPDATE SET + external_id = EXCLUDED.external_id, + nif = EXCLUDED.nif, + legal_name = EXCLUDED.legal_name, + normalized_name = EXCLUDED.normalized_name, + cae = EXCLUDED.cae, + address = EXCLUDED.address, + postcode = EXCLUDED.postcode, + city = EXCLUDED.city, + district = EXCLUDED.district, + country = EXCLUDED.country, + website = EXCLUDED.website, + phone = EXCLUDED.phone, + email = EXCLUDED.email, + duns = EXCLUDED.duns, + status = EXCLUDED.status, + score = EXCLUDED.score, + match_type = EXCLUDED.match_type, + raw_payload = EXCLUDED.raw_payload, + payload_hash = EXCLUDED.payload_hash, + fetched_at = now(), + expires_at = EXCLUDED.expires_at + """), params) + + + +def batch_lookup_companies(*, nifs: Iterable[str] = (), names: Iterable[str] = (), emails: Iterable[str] = (), domains: Iterable[str] = (), websites: Iterable[str] = (), limit_per_query: int = 5) -> Dict[str, Any]: + """Use POST /empresas/batch for large offline enrichment/reconciliation runs. + + The incremental worker prefers single lookups so it can stop as soon as it + finds a strong match. Batch lookup is exposed for future nightly jobs and + tests/manual diagnostics. + """ + payload = { + "nifs": [normalize_tax_id(x) for x in nifs if normalize_tax_id(x)], + "names": [_clean(x) for x in names if _clean(x)], + "emails": [_clean(x).lower() for x in emails if _clean(x)], + "domains": [normalize_domain(x) for x in domains if normalize_domain(x) and not is_public_email_domain(x)], + "websites": [_clean(x) for x in websites if _clean(x)], + "include_contactos": False, + "limit_per_query": max(min(int(limit_per_query or 5), 20), 1), + } + payload = {k: v for k, v in payload.items() if v not in ([], "", None)} + if not any(k in payload for k in ("nifs", "names", "emails", "domains", "websites")): + return {"total_inputs": 0, "results": {}} + return _http_json("POST", "/empresas/batch", payload=payload) + +def _lookup_external_by_nif(nif: str) -> Optional[Dict[str, Any]]: + nif = normalize_tax_id(nif) + if not nif: + return None + cached = _cache_lookup("nif", nif) + if cached: + return cached + data = _http_json("GET", f"/empresas/nif/{urllib.parse.quote(nif)}", params={"include_contactos": "false"}) + company = _best_company_from_response(data) + if company: + company.setdefault("score", 100.0) + company.setdefault("match_type", "nif_exato") + _cache_company("nif", nif, company) + return company + + +def _lookup_external_by_email(email: str) -> Optional[Dict[str, Any]]: + email = _clean(email).lower() + if not email or "@" not in email: + return None + cached = _cache_lookup("email", email) + if cached: + return cached + data = _http_json("GET", "/contactos/search", params={"email": email, "limit": 10}) + company = _best_company_from_response(data) + if company: + _cache_company("email", email, company) + return company + domain = _domain_from_email(email) + if domain and not is_public_email_domain(domain): + return _lookup_external_by_domain(domain) + return None + + +def _lookup_external_by_domain(domain: str) -> Optional[Dict[str, Any]]: + domain = normalize_domain(domain) + if not domain or is_public_email_domain(domain): + return None + cached = _cache_lookup("domain", domain) + if cached: + return cached + data = _http_json("GET", f"/empresas/domain/{urllib.parse.quote(domain)}", params={"limit": 10, "include_contactos": "false"}) + company = _best_company_from_response(data) + if company: + _cache_company("domain", domain, company) + return company + data = _http_json("GET", "/empresas/website", params={"domain": domain, "limit": 10, "include_contactos": "false"}) + company = _best_company_from_response(data) + if company: + _cache_company("domain", domain, company) + return company + + +def _lookup_external_by_name(name: str) -> Optional[Dict[str, Any]]: + name = _clean(name) + if len(name) < 3: + return None + normalized = normalize_fiscal_name(name) + cached = _cache_lookup("name", normalized) + if cached: + return cached + data = _http_json("GET", "/empresas/search", params={"q": name, "limit": 10, "include_contactos": "false"}) + company = _best_company_from_response(data) + if company: + _cache_company("name", normalized, company) + return company + + +def _source_signals_from_opportunity(opportunity: Dict[str, Any]) -> List[Tuple[str, str]]: + metadata = opportunity.get("metadata") if isinstance(opportunity.get("metadata"), dict) else {} + signals: List[Tuple[str, str]] = [] + for key in ("customer_tax_id", "tax_id", "nif"): + value = normalize_tax_id(metadata.get(key) or opportunity.get(key)) + if value: + signals.append(("nif", value)) + email = _clean(opportunity.get("customer_email") or metadata.get("customer_email") or metadata.get("email")).lower() + if email and "@" in email: + signals.append(("email", email)) + domain = _domain_from_email(email) + if domain and not is_public_email_domain(domain): + signals.append(("domain", domain)) + for key in ("website", "url"): + domain = normalize_domain(metadata.get(key) or opportunity.get(key)) + if domain and not is_public_email_domain(domain): + signals.append(("domain", domain)) + for value in (opportunity.get("linked_customer_tax_id"),): + nif = normalize_tax_id(value) + if nif: + signals.append(("nif", nif)) + name = _clean(opportunity.get("customer_name") or metadata.get("customer_name") or opportunity.get("title")) + if name: + signals.append(("name", name)) + # stable de-dup preserving order + result: List[Tuple[str, str]] = [] + seen = set() + for kind, value in signals: + key = (kind, value.casefold()) + if value and key not in seen: + seen.add(key) + result.append((kind, value)) + return result + + + +def _company_from_customer_row(row: Dict[str, Any], *, match_type: str = "email_identity_company_internal", score: float = 96.0) -> Dict[str, Any]: + """Represent an existing ClientFlow customer as a company candidate.""" + return { + "id": _clean(row.get("id")), + "nome": _clean(row.get("name")), + "nif": normalize_tax_id(row.get("tax_id")), + "email_principal": _clean(row.get("email")), + "telefone": _clean(row.get("phone")), + "morada": _clean(row.get("street_name")), + "codigo_postal": _clean(row.get("postal_zone")), + "localidade": _clean(row.get("city_name")), + "pais": _clean(row.get("country") or "Portugal"), + "score": score, + "match_type": match_type, + "clientflow_customer_id": _clean(row.get("id")), + "source": "clientflow_internal_identity", + } + + + + +def _identity_company_mentions(identity: Optional[Dict[str, Any]]) -> List[str]: + if not identity or not isinstance(identity.get("company_mentions"), list): + return [] + out: List[str] = [] + for value in identity.get("company_mentions") or []: + cleaned = _clean(value) + if cleaned and is_plausible_company_mention(cleaned): + out.append(cleaned) + return out + + +def _normalized_identity_mentions(identity: Optional[Dict[str, Any]]) -> List[str]: + return [normalize_fiscal_name(x) for x in _identity_company_mentions(identity) if normalize_fiscal_name(x)] + + +def _company_name_matches_mention(mention_norm: str, candidate_norm: str) -> bool: + if not mention_norm or not candidate_norm: + return False + if len(mention_norm) < 4 or len(candidate_norm) < 4: + return False + if mention_norm == candidate_norm: + return True + # Substring matching is useful for variants like "Dietimport S.A" vs + # "DIETIMPORT, S.A.", but dangerous for tiny tokens such as "pt". + if len(mention_norm) >= 5 and len(candidate_norm) >= 5: + if mention_norm in candidate_norm or candidate_norm in mention_norm: + return True + mention_tokens = {t for t in mention_norm.split() if len(t) >= 5 and t not in {"unipessoal", "limitada"}} + candidate_tokens = {t for t in candidate_norm.split() if len(t) >= 5 and t not in {"unipessoal", "limitada"}} + return bool(mention_tokens & candidate_tokens) + +def _find_internal_customer_by_identity(identity: Optional[Dict[str, Any]], opportunity: Dict[str, Any]) -> Optional[Dict[str, Any]]: + """Prefer explicit company evidence found in the email body/signature. + + This prevents a weak external domain match from winning when the email text + itself mentions a company already known by ClientFlow/Odoo/Jasmin, e.g. + Dietimport S.A. in the legal disclaimer. + """ + if not identity: + return None + mentions = _identity_company_mentions(identity) + normalized_mentions = _normalized_identity_mentions(identity) + domain = normalize_domain(identity.get("domain") or _domain_from_email(opportunity.get("customer_email"))) + if not normalized_mentions and not domain: + return None + + with engine.begin() as conn: + rows = conn.execute(text(""" + SELECT id::text, name, tax_id, email, phone, street_name, postal_zone, city_name, country, metadata + FROM customers + WHERE (tax_id IS NOT NULL AND tax_id <> '') + OR (email IS NOT NULL AND email <> '') + OR name IS NOT NULL + ORDER BY updated_at DESC + LIMIT 2000 + """)).mappings().all() + + best: Optional[Dict[str, Any]] = None + best_score = 0.0 + for row in rows: + name_norm = normalize_fiscal_name(row.get("name")) + email_domain = _domain_from_email(row.get("email")) + score = 0.0 + for mention in normalized_mentions: + if mention and name_norm: + if mention == name_norm: + score = max(score, 98.0) + elif _company_name_matches_mention(mention, name_norm): + score = max(score, 92.0) + if domain and email_domain and domain == email_domain: + score += 3.0 if score else 70.0 + # same address reinforces an explicit company mention + if score >= 90 and _clean(identity.get("address")) and _clean(row.get("street_name")): + if normalize_fiscal_name(row.get("street_name")) in normalize_fiscal_name(identity.get("address")): + score += 2.0 + if score > best_score: + best_score = min(score, 99.0) + best = dict(row) + if not best or best_score < 90: + return None + return _company_from_customer_row(best, score=best_score) + + +def _identity_company_conflict(identity: Optional[Dict[str, Any]], company: Dict[str, Any]) -> bool: + """Detect when an external candidate conflicts with explicit email identity.""" + if not identity: + return False + mentions = _identity_company_mentions(identity) + normalized_mentions = _normalized_identity_mentions(identity) + if not normalized_mentions: + return False + candidate_name = normalize_fiscal_name(company.get("nome") or company.get("legal_name") or company.get("name")) + if not candidate_name: + return False + for mention in normalized_mentions: + if _company_name_matches_mention(mention, candidate_name): + return False + return True + + +def _identity_mentions_match_name(identity: Optional[Dict[str, Any]], name: Any) -> bool: + if not identity or not _clean(name): + return False + candidate_name = normalize_fiscal_name(name) + mentions = _identity_company_mentions(identity) + for mention in mentions: + mention_norm = normalize_fiscal_name(mention) + if _company_name_matches_mention(mention_norm, candidate_name): + return True + return False + + +def _identity_conflicts_with_linked_customer(identity: Optional[Dict[str, Any]], opportunity: Dict[str, Any]) -> bool: + linked_name = _clean(opportunity.get("linked_customer_name")) + mentions = _identity_company_mentions(identity) + if not linked_name or not mentions: + return False + return not _identity_mentions_match_name(identity, linked_name) + + +def email_identity_review_for_opportunity(opportunity_id: str, *, refresh: bool = False) -> Dict[str, Any]: + """Return the operator-facing identity review for an opportunity. + + This is read-only except when refresh=True, where it stores a new extraction. + It intentionally runs even if a fiscal customer is already linked so the UI + can show cases like: email mentions Dietimport S.A. but the opportunity is + linked to Fmrl - Imobiliária S.A. + """ + ensure_fiscal_enrichment_schema() + opportunity = get_opportunity(opportunity_id) + if not opportunity: + return {"ok": False, "reason": "opportunity_not_found"} + try: + identity = extract_identity_for_opportunity( + opportunity_id, + refresh=refresh, + use_llm=bool(getattr(settings, "email_identity_extraction_use_llm", True)), + ) + except Exception as exc: + return {"ok": False, "reason": f"email_identity_extraction_failed: {exc}"} + + valid_mentions = _identity_company_mentions(identity) + identity_company = _find_internal_customer_by_identity(identity, opportunity) if valid_mentions else None + linked_customer_id = _clean(opportunity.get("linked_customer_id") or opportunity.get("local_customer_id")) + suggested_customer_id = _clean(identity_company.get("clientflow_customer_id")) if identity_company else "" + conflict = _identity_conflicts_with_linked_customer(identity, opportunity) if valid_mentions else False + if conflict and suggested_customer_id and linked_customer_id and suggested_customer_id == linked_customer_id: + conflict = False + + return { + "ok": True, + "opportunity_id": opportunity_id, + "identity": identity, + "valid_company_mentions": valid_mentions, + "linked_customer_id": linked_customer_id, + "linked_customer_name": opportunity.get("linked_customer_name"), + "linked_customer_tax_id": opportunity.get("linked_customer_tax_id"), + "suggested_internal_customer": identity_company, + "conflict": conflict, + "status": "conflict" if conflict else ("suggestion" if identity_company else "identity_only"), + } + + +def assist_email_identity_enrichment(opportunity_id: str, *, refresh: bool = True, apply_safe: bool = False) -> Dict[str, Any]: + """Use extracted email identity to create an assisted fiscal suggestion. + + v4.9.26.4 deliberately defaults to apply_safe=False. The goal is to show + evidence and create pending suggestions for the operator, not to auto-link + fiscal customers based only on an LLM extraction. + """ + ensure_fiscal_enrichment_schema() + review = email_identity_review_for_opportunity(opportunity_id, refresh=refresh) + if not review.get("ok"): + return {"seen": 0, "suggested": 0, "auto_applied": 0, "skipped": 1, "reason": review.get("reason")} + + identity = review.get("identity") or {} + opportunity = get_opportunity(opportunity_id) or {} + valid_mentions = _identity_company_mentions(identity) + identity_company = review.get("suggested_internal_customer") if valid_mentions else None + linked_customer_id = _clean(review.get("linked_customer_id")) + conflict = bool(review.get("conflict")) if valid_mentions else False + + event_type = "email_identity_review" + event_note = "Identidade extraída do email para revisão fiscal assistida." + suggestion = None + + if identity_company: + confidence = _confidence_for_company(identity_company, lookup_type="email_identity") + suggested_customer_id = _clean(identity_company.get("clientflow_customer_id")) or None + + # v4.9.26.5 / v4.9.26.4.1: + # If the extracted identity points to the same fiscal customer already + # linked to the opportunity, do not create another pending suggestion. + # The operator needs a validation event, not duplicate work. + if suggested_customer_id and linked_customer_id and suggested_customer_id == linked_customer_id and not conflict: + with engine.begin() as conn: + conn.execute(text(""" + INSERT INTO opportunity_events (id, opportunity_id, event_type, note, payload, created_by) + VALUES (gen_random_uuid(), CAST(:opportunity_id AS UUID), 'email_identity_matches_current_fiscal_customer', :note, CAST(:payload AS JSONB), 'email_identity_assisted_enrichment') + """), { + "opportunity_id": opportunity_id, + "note": f"Identidade extraída confirma o cliente fiscal atual: {identity_company.get('nome')} / {identity_company.get('nif') or 'sem NIF'}.", + "payload": _json({"identity": identity, "customer_id": suggested_customer_id, "conflict": False}), + }) + conn.execute(text(""" + UPDATE opportunities + SET metadata = COALESCE(metadata, '{}'::jsonb) || jsonb_build_object( + 'last_email_identity_validation', jsonb_build_object( + 'status', 'matches_current_fiscal_customer', + 'customer_id', CAST(:customer_id AS TEXT), + 'customer_name', CAST(:customer_name AS TEXT), + 'customer_tax_id', CAST(:customer_tax_id AS TEXT), + 'validated_at', now() + ) + ), + updated_at = now() + WHERE id = CAST(:opportunity_id AS UUID) + """), { + "opportunity_id": opportunity_id, + "customer_id": suggested_customer_id, + "customer_name": identity_company.get("nome"), + "customer_tax_id": identity_company.get("nif"), + }) + return { + "seen": 1, "suggested": 0, "auto_applied": 0, "skipped": 1, + "identity_used": True, "conflict": False, "suggestion_id": None, + "status": "email_identity_matches_current_fiscal_customer", + } + + status = "pending" + auto_applied = False + if apply_safe and not conflict and not linked_customer_id and suggested_customer_id and _should_auto_apply(opportunity, identity_company, confidence=confidence): + link_customer_to_opportunity(suggested_customer_id, opportunity_id) + status = "accepted" + auto_applied = True + suggestion = _upsert_suggestion( + opportunity_id, + identity_company, + lookup_type="email_identity", + lookup_value=", ".join(identity.get("company_mentions") or []), + confidence=confidence, + status=status, + suggested_customer_id=suggested_customer_id, + auto_applied=auto_applied, + ) + event_type = "email_identity_fiscal_conflict" if conflict else "email_identity_fiscal_suggestion" + event_note = ( + f"Possível conflito fiscal: email menciona {', '.join(identity.get('company_mentions') or [])}; " + f"cliente atual {review.get('linked_customer_name') or '—'}." + if conflict else + f"Sugestão fiscal por identidade extraída do email: {identity_company.get('nome')} / {identity_company.get('nif') or 'sem NIF'}." + ) + with engine.begin() as conn: + conn.execute(text(""" + INSERT INTO opportunity_events (id, opportunity_id, event_type, note, payload, created_by) + VALUES (gen_random_uuid(), CAST(:opportunity_id AS UUID), :event_type, :note, CAST(:payload AS JSONB), 'email_identity_assisted_enrichment') + """), { + "opportunity_id": opportunity_id, + "event_type": event_type, + "note": event_note, + "payload": _json({"identity": identity, "suggestion_id": suggestion.get("id") if suggestion else None, "conflict": conflict}), + }) + return { + "seen": 1, "suggested": 1, "auto_applied": 1 if auto_applied else 0, "skipped": 0, + "identity_used": True, "conflict": conflict, "suggestion_id": suggestion.get("id") if suggestion else None, + "status": event_type, + } + + # No internal customer match. If a company was explicitly mentioned, ask the + # external lookup by name and create a pending suggestion, still assisted. + mentions = valid_mentions + for mention in mentions[:3]: + try: + company = _lookup_external_by_name(mention) + except Exception: + company = None + if not company: + continue + confidence = _apply_identity_confidence_guard(company, identity=identity, confidence=_confidence_for_company(company, lookup_type="name")) + if confidence < 75: + continue + suggestion = _upsert_suggestion( + opportunity_id, + company, + lookup_type="email_identity_name", + lookup_value=mention, + confidence=confidence, + status="pending", + suggested_customer_id=None, + auto_applied=False, + ) + with engine.begin() as conn: + conn.execute(text(""" + INSERT INTO opportunity_events (id, opportunity_id, event_type, note, payload, created_by) + VALUES (gen_random_uuid(), CAST(:opportunity_id AS UUID), 'email_identity_external_suggestion', :note, CAST(:payload AS JSONB), 'email_identity_assisted_enrichment') + """), { + "opportunity_id": opportunity_id, + "note": f"Sugestão externa por empresa extraída do email: {mention}.", + "payload": _json({"identity": identity, "suggestion_id": suggestion.get("id") if suggestion else None}), + }) + return {"seen": 1, "suggested": 1, "auto_applied": 0, "skipped": 0, "identity_used": True, "suggestion_id": suggestion.get("id") if suggestion else None, "status": "email_identity_external_suggestion"} + + with engine.begin() as conn: + conn.execute(text(""" + INSERT INTO opportunity_events (id, opportunity_id, event_type, note, payload, created_by) + VALUES (gen_random_uuid(), CAST(:opportunity_id AS UUID), :event_type, :note, CAST(:payload AS JSONB), 'email_identity_assisted_enrichment') + """), { + "opportunity_id": opportunity_id, + "event_type": event_type, + "note": event_note, + "payload": _json({"identity": identity, "conflict": conflict}), + }) + return {"seen": 1, "suggested": 0, "auto_applied": 0, "skipped": 0, "identity_used": bool(identity), "status": "identity_only"} + + +def _apply_identity_confidence_guard(company: Dict[str, Any], *, identity: Optional[Dict[str, Any]], confidence: float) -> float: + """Lower confidence when the endpoint only matched a domain and the email mentions another company.""" + match_type = _clean(company.get("match_type") or company.get("empresa_resolution_type")) + adjusted = float(confidence or 0.0) + if match_type in DOMAIN_ONLY_MATCH_TYPES: + adjusted = min(adjusted, 85.0) + if _identity_company_conflict(identity, company): + adjusted = min(adjusted, 70.0) + company["identity_conflict"] = True + company["identity_company_mentions"] = identity.get("company_mentions") if identity else [] + return adjusted + +def _lookup_company_for_signal(kind: str, value: str) -> Optional[Dict[str, Any]]: + if kind == "nif": + return _lookup_external_by_nif(value) + if kind == "email": + return _lookup_external_by_email(value) + if kind == "domain": + return _lookup_external_by_domain(value) + if kind == "name": + return _lookup_external_by_name(value) + return None + + +def _confidence_for_company(company: Dict[str, Any], *, lookup_type: str) -> float: + score = _float(company.get("score"), 0.0) + match_type = _clean(company.get("match_type") or company.get("empresa_resolution_type")) + if match_type in VERY_STRONG_MATCH_TYPES: + return max(score, 95.0) + if match_type in STRONG_MATCH_TYPES: + return max(score, 85.0) + if match_type in MEDIUM_MATCH_TYPES: + return max(score, 65.0) + if lookup_type == "nif" and normalize_tax_id(company.get("nif")): + return max(score, 100.0) + if score: + return score + return 50.0 if lookup_type == "name" else 70.0 + + +def _reason_for_company(company: Dict[str, Any], *, lookup_type: str, lookup_value: str) -> str: + match_type = _clean(company.get("match_type") or company.get("empresa_resolution_type") or "match") + name = _clean(company.get("nome") or company.get("legal_name") or company.get("name")) + nif = normalize_tax_id(company.get("nif") or company.get("tax_id")) + return f"{match_type} por {lookup_type}={lookup_value}; empresa={name}; nif={nif or '—'}" + + +def _upsert_suggestion( + opportunity_id: str, + company: Dict[str, Any], + *, + lookup_type: str, + lookup_value: str, + confidence: float, + status: str = "pending", + suggested_customer_id: Optional[str] = None, + auto_applied: bool = False, +) -> Dict[str, Any]: + ensure_fiscal_enrichment_schema() + name = _clean(company.get("nome") or company.get("legal_name") or company.get("name")) + nif = normalize_tax_id(company.get("nif") or company.get("tax_id")) + reason = _reason_for_company(company, lookup_type=lookup_type, lookup_value=lookup_value) + match_type = _clean(company.get("match_type") or company.get("empresa_resolution_type")) + params = { + "opportunity_id": opportunity_id, + "suggested_customer_id": suggested_customer_id, + "suggested_nif": nif or None, + "suggested_name": name, + "source": CACHE_SOURCE, + "lookup_type": lookup_type, + "lookup_value": lookup_value, + "match_type": match_type, + "confidence": confidence, + "reason": reason, + "status": status, + "auto_applied": auto_applied, + "raw_payload": _json(company), + } + with engine.begin() as conn: + row = conn.execute(text(""" + INSERT INTO fiscal_customer_suggestions ( + opportunity_id, suggested_customer_id, suggested_nif, suggested_name, source, + lookup_type, lookup_value, match_type, confidence, reason, status, auto_applied, + raw_payload, updated_at, resolved_at, resolved_by + ) VALUES ( + CAST(:opportunity_id AS UUID), CAST(:suggested_customer_id AS UUID), :suggested_nif, :suggested_name, :source, + :lookup_type, :lookup_value, :match_type, :confidence, :reason, :status, :auto_applied, + CAST(:raw_payload AS JSONB), now(), CASE WHEN :status <> 'pending' THEN now() ELSE NULL END, + CASE WHEN :status <> 'pending' THEN 'fiscal_enrichment_worker' ELSE NULL END + ) + ON CONFLICT (opportunity_id, suggested_nif) + WHERE suggested_nif IS NOT NULL AND suggested_nif <> '' AND status = 'pending' + DO UPDATE SET + suggested_customer_id = COALESCE(EXCLUDED.suggested_customer_id, fiscal_customer_suggestions.suggested_customer_id), + suggested_name = EXCLUDED.suggested_name, + lookup_type = EXCLUDED.lookup_type, + lookup_value = EXCLUDED.lookup_value, + match_type = EXCLUDED.match_type, + confidence = GREATEST(EXCLUDED.confidence, fiscal_customer_suggestions.confidence), + reason = EXCLUDED.reason, + raw_payload = EXCLUDED.raw_payload, + updated_at = now() + RETURNING id::text, opportunity_id::text, suggested_customer_id::text, suggested_nif, + suggested_name, source, lookup_type, lookup_value, match_type, confidence, + reason, status, auto_applied, raw_payload, created_at, updated_at, resolved_at + """), params).mappings().first() + return dict(row or {}) + + +def _has_conflicting_customer(opportunity: Dict[str, Any], suggested_tax_id: str) -> bool: + linked_tax_id = normalize_tax_id(opportunity.get("linked_customer_tax_id")) + return bool(linked_tax_id and suggested_tax_id and linked_tax_id != suggested_tax_id) + + +def _should_auto_apply(opportunity: Dict[str, Any], company: Dict[str, Any], *, confidence: float) -> bool: + if opportunity.get("linked_customer_id") or opportunity.get("local_customer_id"): + return False + tax_id = normalize_tax_id(company.get("nif") or company.get("tax_id")) + name = _clean(company.get("nome") or company.get("legal_name") or company.get("name")) + if not (tax_id and name): + return False + if _has_conflicting_customer(opportunity, tax_id): + return False + match_type = _clean(company.get("match_type") or company.get("empresa_resolution_type")) + if match_type in DOMAIN_ONLY_MATCH_TYPES: + return False + return confidence >= _auto_threshold() and match_type in VERY_STRONG_MATCH_TYPES + + +def apply_fiscal_suggestion(suggestion_id: str, *, actor: str = "operator") -> Dict[str, Any]: + """Accept a suggestion and link/create the fiscal customer idempotently.""" + ensure_fiscal_enrichment_schema() + with engine.begin() as conn: + row = conn.execute(text(""" + SELECT * FROM fiscal_customer_suggestions + WHERE id = CAST(:id AS UUID) + LIMIT 1 + """), {"id": suggestion_id}).mappings().first() + if not row: + return {"applied": False, "reason": "suggestion_not_found"} + suggestion = dict(row) + company = suggestion.get("raw_payload") if isinstance(suggestion.get("raw_payload"), dict) else {} + opportunity_id = str(suggestion.get("opportunity_id") or "") + opportunity = get_opportunity(opportunity_id) + if not opportunity: + return {"applied": False, "reason": "opportunity_not_found"} + if opportunity.get("linked_customer_id") and normalize_tax_id(opportunity.get("linked_customer_tax_id")) != normalize_tax_id(suggestion.get("suggested_nif")): + return {"applied": False, "reason": "opportunity_has_conflicting_customer"} + customer = upsert_customer(_company_to_customer_data(company)) + customer_id = str(customer.get("id") or "") + if not customer_id: + return {"applied": False, "reason": "customer_upsert_failed"} + link_customer_to_opportunity(customer_id, opportunity_id) + with engine.begin() as conn: + conn.execute(text(""" + UPDATE fiscal_customer_suggestions + SET status = 'accepted', suggested_customer_id = CAST(:customer_id AS UUID), + auto_applied = COALESCE(auto_applied, FALSE), resolved_at = now(), resolved_by = :actor, + updated_at = now() + WHERE id = CAST(:id AS UUID) + """), {"id": suggestion_id, "customer_id": customer_id, "actor": actor}) + conn.execute(text(""" + INSERT INTO opportunity_events (id, opportunity_id, event_type, note, payload, created_by) + VALUES (gen_random_uuid(), CAST(:opportunity_id AS UUID), 'fiscal_enrichment_applied', :note, CAST(:payload AS JSONB), :actor) + """), { + "opportunity_id": opportunity_id, + "note": f"Cliente fiscal associado por enriquecimento: {customer.get('name')} / {customer.get('tax_id') or 'sem NIF'}", + "payload": _json({"suggestion_id": suggestion_id, "customer_id": customer_id, "source": CACHE_SOURCE}), + "actor": actor, + }) + return {"applied": True, "customer_id": customer_id, "opportunity_id": opportunity_id} + + +def reject_fiscal_suggestion(suggestion_id: str, *, actor: str = "operator", reason: str = "rejected_by_operator") -> Dict[str, Any]: + ensure_fiscal_enrichment_schema() + with engine.begin() as conn: + row = conn.execute(text(""" + UPDATE fiscal_customer_suggestions + SET status = 'rejected', reason = COALESCE(reason, '') || ' | ' || :reason, + resolved_at = now(), resolved_by = :actor, updated_at = now() + WHERE id = CAST(:id AS UUID) + RETURNING id::text + """), {"id": suggestion_id, "actor": actor, "reason": reason}).mappings().first() + return {"rejected": bool(row)} + + +def list_fiscal_suggestions_for_opportunity(opportunity_id: str, *, limit: int = 5) -> List[Dict[str, Any]]: + ensure_fiscal_enrichment_schema() + with engine.begin() as conn: + rows = conn.execute(text(""" + SELECT + s.id::text, + s.opportunity_id::text, + s.suggested_customer_id::text, + COALESCE(NULLIF(s.suggested_nif, ''), c.tax_id) AS suggested_nif, + COALESCE(NULLIF(s.suggested_name, ''), c.name) AS suggested_name, + s.source, + s.lookup_type, + s.lookup_value, + s.match_type, + s.confidence, + s.reason, + s.status, + s.auto_applied, + s.created_at, + s.updated_at, + s.resolved_at + FROM fiscal_customer_suggestions s + LEFT JOIN customers c ON c.id = s.suggested_customer_id + WHERE s.opportunity_id = CAST(:opportunity_id AS UUID) + ORDER BY CASE s.status WHEN 'pending' THEN 0 WHEN 'accepted' THEN 1 ELSE 2 END, + s.confidence DESC, s.updated_at DESC + LIMIT :limit + """), {"opportunity_id": opportunity_id, "limit": int(limit)}).mappings().all() + return [dict(row) for row in rows] + + +def enrich_opportunity(opportunity_id: str, *, apply_safe: bool = True) -> Dict[str, Any]: + """Enrich a single opportunity with a fiscal customer suggestion/association.""" + ensure_fiscal_enrichment_schema() + opportunity = get_opportunity(opportunity_id) + if not opportunity: + return {"seen": 0, "enriched": 0, "suggested": 0, "auto_applied": 0, "skipped": 1, "reason": "opportunity_not_found"} + if opportunity.get("linked_customer_id") or opportunity.get("local_customer_id"): + return {"seen": 1, "enriched": 0, "suggested": 0, "auto_applied": 0, "skipped": 1, "reason": "already_has_fiscal_customer"} + + errors: List[str] = [] + identity: Optional[Dict[str, Any]] = None + if bool(getattr(settings, "email_identity_extraction_enabled", True)): + try: + identity = extract_identity_for_opportunity( + opportunity_id, + refresh=False, + use_llm=bool(getattr(settings, "email_identity_extraction_use_llm", True)), + ) + except Exception as exc: + errors.append(f"email_identity_extraction_failed: {exc}") + identity = None + + identity_company = _find_internal_customer_by_identity(identity, opportunity) + if identity_company: + confidence = _confidence_for_company(identity_company, lookup_type="email_identity") + existing_customer_id = _clean(identity_company.get("clientflow_customer_id")) or None + auto_applied = False + status = "pending" + if apply_safe and existing_customer_id and _should_auto_apply(opportunity, identity_company, confidence=confidence): + link_customer_to_opportunity(existing_customer_id, opportunity_id) + status = "accepted" + auto_applied = True + suggestion = _upsert_suggestion( + opportunity_id, + identity_company, + lookup_type="email_identity", + lookup_value=", ".join(identity.get("company_mentions") or []) if identity else "", + confidence=confidence, + status=status, + suggested_customer_id=existing_customer_id, + auto_applied=auto_applied, + ) + if auto_applied: + with engine.begin() as conn: + conn.execute(text(""" + INSERT INTO opportunity_events (id, opportunity_id, event_type, note, payload, created_by) + VALUES (gen_random_uuid(), CAST(:opportunity_id AS UUID), 'email_identity_fiscal_auto_applied', :note, CAST(:payload AS JSONB), 'email_identity_extraction_service') + """), { + "opportunity_id": opportunity_id, + "note": f"Cliente fiscal auto-associado por empresa explícita no email: {identity_company.get('nome')} / {identity_company.get('nif') or 'sem NIF'}", + "payload": _json({"suggestion_id": suggestion.get("id"), "identity": identity, "confidence": confidence}), + }) + return {"seen": 1, "enriched": 1, "suggested": 1, "auto_applied": 1, "skipped": 0, "customer_id": existing_customer_id, "suggestion_id": suggestion.get("id"), "identity_used": True} + return {"seen": 1, "enriched": 0, "suggested": 1, "auto_applied": 0, "skipped": 0, "suggestion_id": suggestion.get("id"), "confidence": confidence, "errors": errors, "identity_used": True} + + for lookup_type, lookup_value in _source_signals_from_opportunity(opportunity): + try: + company = _lookup_company_for_signal(lookup_type, lookup_value) + except Exception as exc: # keep worker resilient + errors.append(str(exc)) + continue + if not company: + continue + name = _clean(company.get("nome") or company.get("legal_name") or company.get("name")) + tax_id = normalize_tax_id(company.get("nif") or company.get("tax_id")) + if not name: + continue + confidence = _confidence_for_company(company, lookup_type=lookup_type) + confidence = _apply_identity_confidence_guard(company, identity=identity, confidence=confidence) + match_type = _clean(company.get("match_type") or company.get("empresa_resolution_type")) + # v4.9.25.1: keep the enrichment queue operationally clean. + # Fuzzy/name-only matches below 75 or explicit approximate-name matches + # are too noisy for the normal workflow and should not create pending + # suggestions. They can still be inspected by querying the external + # source directly when needed. + if match_type == "nome_aproximado" or confidence < 75.0: + continue + existing_customer_id: Optional[str] = None + if tax_id: + existing = get_customer_by_tax_id(tax_id) + if existing: + existing_customer_id = str(existing.get("id") or "") or None + auto_applied = False + status = "pending" + if apply_safe and _should_auto_apply(opportunity, company, confidence=confidence): + customer = upsert_customer(_company_to_customer_data(company)) + existing_customer_id = str(customer.get("id") or "") or existing_customer_id + if existing_customer_id: + link_customer_to_opportunity(existing_customer_id, opportunity_id) + status = "accepted" + auto_applied = True + suggestion = _upsert_suggestion( + opportunity_id, + company, + lookup_type=lookup_type, + lookup_value=lookup_value, + confidence=confidence, + status=status, + suggested_customer_id=existing_customer_id, + auto_applied=auto_applied, + ) + if auto_applied: + with engine.begin() as conn: + conn.execute(text(""" + INSERT INTO opportunity_events (id, opportunity_id, event_type, note, payload, created_by) + VALUES (gen_random_uuid(), CAST(:opportunity_id AS UUID), 'fiscal_enrichment_auto_applied', :note, CAST(:payload AS JSONB), 'fiscal_enrichment_worker') + """), { + "opportunity_id": opportunity_id, + "note": f"Cliente fiscal auto-associado: {name} / {tax_id}", + "payload": _json({"suggestion_id": suggestion.get("id"), "lookup_type": lookup_type, "lookup_value": lookup_value, "confidence": confidence}), + }) + return {"seen": 1, "enriched": 1, "suggested": 1, "auto_applied": 1, "skipped": 0, "customer_id": existing_customer_id, "suggestion_id": suggestion.get("id")} + return {"seen": 1, "enriched": 0, "suggested": 1, "auto_applied": 0, "skipped": 0, "suggestion_id": suggestion.get("id"), "confidence": confidence, "errors": errors} + + return {"seen": 1, "enriched": 0, "suggested": 0, "auto_applied": 0, "skipped": 1, "reason": "no_company_match", "errors": errors} + + +def _open_opportunities_without_fiscal_customer(*, limit: int = 100) -> List[Dict[str, Any]]: + # list_opportunities carries linked_customer_id and contact fields and uses + # the same query path as the UI. Filter in Python to avoid duplicating a + # large SQL projection here. + opportunities = list_opportunities(status="open", limit=max(int(limit or 100), 1)) + return [o for o in opportunities if not (o.get("linked_customer_id") or o.get("local_customer_id"))] + + +def enrich_open_opportunities(*, limit: int = 100, apply_safe: bool = True, mode: str = "incremental") -> Dict[str, Any]: + """Worker entrypoint: enrich open opportunities missing fiscal customer.""" + ensure_fiscal_enrichment_schema() + started = time.time() + seen = enriched = suggested = auto_applied = skipped = 0 + errors: List[str] = [] + opportunities = _open_opportunities_without_fiscal_customer(limit=limit) + for opportunity in opportunities[: int(limit or 100)]: + result = enrich_opportunity(str(opportunity.get("id")), apply_safe=apply_safe) + seen += int(result.get("seen") or 0) + enriched += int(result.get("enriched") or 0) + suggested += int(result.get("suggested") or 0) + auto_applied += int(result.get("auto_applied") or 0) + skipped += int(result.get("skipped") or 0) + errors.extend(str(e) for e in (result.get("errors") or []) if e) + summary = { + "mode": mode, + "enabled": _external_enabled(), + "seen": seen, + "enriched": enriched, + "suggested": suggested, + "auto_applied": auto_applied, + "skipped": skipped, + "errors": errors[:10], + "duration_seconds": round(time.time() - started, 3), + } + with engine.begin() as conn: + conn.execute(text(""" + INSERT INTO fiscal_enrichment_runs (mode, status, seen, enriched, suggested, auto_applied, skipped, errors, payload, finished_at) + VALUES (:mode, 'finished', :seen, :enriched, :suggested, :auto_applied, :skipped, CAST(:errors AS JSONB), CAST(:payload AS JSONB), now()) + """), { + "mode": mode, + "seen": seen, + "enriched": enriched, + "suggested": suggested, + "auto_applied": auto_applied, + "skipped": skipped, + "errors": _json(errors[:10]), + "payload": _json(summary), + }) + return summary + + +def fiscal_enrichment_summary() -> Dict[str, Any]: + ensure_fiscal_enrichment_schema() + with engine.begin() as conn: + row = conn.execute(text(""" + SELECT + (SELECT COUNT(*) FROM opportunities WHERE status = 'open' AND local_customer_id IS NULL)::int AS open_without_fiscal_customer, + (SELECT COUNT(*) FROM fiscal_customer_suggestions WHERE status = 'pending')::int AS pending_suggestions, + (SELECT COUNT(*) FROM fiscal_customer_suggestions WHERE status = 'accepted' AND auto_applied = TRUE)::int AS auto_applied_suggestions, + (SELECT MAX(finished_at) FROM fiscal_enrichment_runs)::text AS last_run_at + """)).mappings().first() + return dict(row or {}) diff --git a/app/integration_outbox_service.py b/app/integration_outbox_service.py new file mode 100644 index 0000000..ef110c7 --- /dev/null +++ b/app/integration_outbox_service.py @@ -0,0 +1,592 @@ +import json +import os +from typing import Any, Dict, List, Optional + +from sqlalchemy import text + +from app.db import engine + + +def _json(value: Any) -> str: + return json.dumps(value or {}, ensure_ascii=False) + + +def create_outbox_item( + *, + business_event_id: str, + target_system: str, + action_type: str, + payload: Dict[str, Any], + idempotency_key: str, +) -> Optional[str]: + sql = text(""" + INSERT INTO integration_outbox ( + business_event_id, + target_system, + action_type, + payload, + status, + retry_count, + idempotency_key + ) + VALUES ( + CAST(:business_event_id AS UUID), + :target_system, + :action_type, + CAST(:payload AS JSONB), + 'pending', + 0, + :idempotency_key + ) + ON CONFLICT (idempotency_key) DO NOTHING + RETURNING id::text + """) + + with engine.begin() as conn: + row = conn.execute(sql, { + "business_event_id": business_event_id, + "target_system": target_system, + "action_type": action_type, + "payload": _json(payload), + "idempotency_key": idempotency_key, + }).fetchone() + + return row[0] if row else None + + +def build_outbox_specs( + *, + business_event_id: str, + event_type: str, + task: Dict[str, Any], + payload: Dict[str, Any], +) -> List[Dict[str, Any]]: + """Constrói ações de outbox para integrações externas ativas. + + A integração CRM externa antiga foi removida. O pipeline comercial passa a viver no + ClientFlow: oportunidades, produtos, financeiro e encomendas são entidades + internas. Esta função já não cria novos itens para o CRM externo antigo. + """ + base_payload = { + "business_event_id": business_event_id, + "event_type": event_type, + "task_id": task.get("id"), + "conversation_id": task.get("conversation_id"), + "contact_id": task.get("contact_id"), + "action_code": task.get("action_code"), + "route": task.get("route"), + "action": task.get("action"), + "note": task.get("note"), + "event_payload": payload or {}, + } + + if event_type == "invoice_sent": + return [ + { + "target_system": "chatwoot", + "action_type": "add_private_note", + "payload": {**base_payload, "note": "Fatura marcada como enviada no ClientFlow."}, + }, + { + "target_system": "mautic", + "action_type": "add_tag", + "payload": {**base_payload, "tag": "invoice_sent"}, + }, + ] + + if event_type == "payment_confirmed": + return [ + { + "target_system": "mautic", + "action_type": "add_tag", + "payload": {**base_payload, "tag": "payment_confirmed"}, + }, + { + "target_system": "mautic", + "action_type": "remove_tag", + "payload": {**base_payload, "tag": "proforma_unpaid"}, + }, + ] + + if event_type == "order_prepared": + return [ + { + "target_system": "chatwoot", + "action_type": "add_private_note", + "payload": {**base_payload, "note": "Encomenda preparada no ClientFlow."}, + }, + ] + + if event_type == "shipment_validated": + return [ + { + "target_system": "chatwoot", + "action_type": "add_private_note", + "payload": {**base_payload, "note": "Envio validado no ClientFlow."}, + }, + { + "target_system": "mautic", + "action_type": "add_tag", + "payload": {**base_payload, "tag": "shipment_validated"}, + }, + ] + + return [] + +def create_outbox_for_business_event( + *, + business_event_id: str, + event_type: str, + task: Dict[str, Any], + payload: Optional[Dict[str, Any]] = None, +) -> List[str]: + specs = build_outbox_specs( + business_event_id=business_event_id, + event_type=event_type, + task=task, + payload=payload or {}, + ) + + created_ids: List[str] = [] + + for spec in specs: + idempotency_key = ":".join([ + "outbox", + business_event_id, + spec["target_system"], + spec["action_type"], + ]) + + outbox_id = create_outbox_item( + business_event_id=business_event_id, + target_system=spec["target_system"], + action_type=spec["action_type"], + payload=spec["payload"], + idempotency_key=idempotency_key, + ) + + if outbox_id: + created_ids.append(outbox_id) + + return created_ids + + +def list_outbox( + *, + status: Optional[str] = None, + target_system: Optional[str] = None, + limit: int = 100, +) -> List[Dict[str, Any]]: + where = [] + params: Dict[str, Any] = {"limit": limit} + + if status: + where.append("status = :status") + params["status"] = status + + if target_system: + where.append("target_system = :target_system") + params["target_system"] = target_system + + where_sql = "" + if where: + where_sql = "WHERE " + " AND ".join(where) + + sql = text(f""" + SELECT + id::text, + business_event_id::text, + target_system, + action_type, + payload, + status, + retry_count, + idempotency_key, + last_error, + created_at, + updated_at, + sent_at, + locked_at, + lock_owner, + ignored_at + FROM integration_outbox + {where_sql} + ORDER BY created_at DESC + LIMIT :limit + """) + + with engine.begin() as conn: + rows = conn.execute(sql, params).mappings().all() + + return [dict(row) for row in rows] + + +def get_outbox_item(outbox_id: str) -> Optional[Dict[str, Any]]: + sql = text(""" + SELECT + id::text, + business_event_id::text, + target_system, + action_type, + payload, + status, + retry_count, + idempotency_key, + last_error, + created_at, + updated_at, + sent_at, + locked_at, + lock_owner, + ignored_at + FROM integration_outbox + WHERE id = CAST(:outbox_id AS UUID) + LIMIT 1 + """) + + with engine.begin() as conn: + row = conn.execute(sql, {"outbox_id": outbox_id}).mappings().first() + + return dict(row) if row else None + + +def list_pending_outbox( + limit: int = 50, + target_system: Optional[str] = None, +) -> List[Dict[str, Any]]: + return list_outbox( + status="pending", + target_system=target_system, + limit=limit, + ) + + +def claim_pending_outbox( + *, + limit: int = 50, + target_system: Optional[str] = None, + lock_owner: str = "worker", +) -> List[Dict[str, Any]]: + """Claim pending outbox rows atomically for one worker. + + This prevents two systemd timers/workers from processing the same pending + integration item at the same time. PostgreSQL SKIP LOCKED lets concurrent + workers take different rows without blocking each other. + """ + sql = text(""" + WITH picked AS ( + SELECT id + FROM integration_outbox + WHERE status = 'pending' + AND (:target_system IS NULL OR target_system = :target_system) + ORDER BY created_at ASC + FOR UPDATE SKIP LOCKED + LIMIT :limit + ) + UPDATE integration_outbox io + SET + status = 'processing', + locked_at = now(), + lock_owner = :lock_owner, + updated_at = now(), + last_error = NULL + FROM picked + WHERE io.id = picked.id + RETURNING + io.id::text, + io.business_event_id::text, + io.target_system, + io.action_type, + io.payload, + io.status, + io.retry_count, + io.idempotency_key, + io.last_error, + io.created_at, + io.updated_at, + io.sent_at, + io.locked_at, + io.lock_owner + """) + with engine.begin() as conn: + rows = conn.execute(sql, { + "limit": int(limit), + "target_system": target_system, + "lock_owner": str(lock_owner or "worker")[:120], + }).mappings().all() + return [dict(row) for row in rows] + + + +def outbox_stale_minutes() -> int: + """Configured threshold for stuck processing rows.""" + raw = os.getenv("OUTBOX_STALE_PROCESSING_MINUTES", "30").strip() + try: + return max(1, int(raw)) + except ValueError: + return 30 + + +def recover_stale_processing_outbox( + *, + stale_minutes: Optional[int] = None, + mode: Optional[str] = None, + limit: int = 100, + actor: str = "system", +) -> List[Dict[str, Any]]: + """Recover or expose outbox items left in processing too long. + + Modes: + - manual_only: mark rows as ``stale`` so the operator can decide; + - mark_failed: mark rows as ``failed`` with a stale-processing reason; + - retry_pending: return rows to ``pending`` so the worker retries them. + """ + stale_minutes = int(stale_minutes or outbox_stale_minutes()) + mode = str(mode or os.getenv("OUTBOX_STALE_RECOVERY_MODE", "manual_only")).strip().lower() + if mode not in {"manual_only", "mark_failed", "retry_pending"}: + mode = "manual_only" + + if mode == "retry_pending": + new_status = "pending" + retry_sql = "retry_count = retry_count + 1," + error = f"Processing stale há mais de {stale_minutes} minutos; reposto para pending por {actor}." + elif mode == "mark_failed": + new_status = "failed" + retry_sql = "retry_count = retry_count + 1," + error = f"Processing stale há mais de {stale_minutes} minutos; marcado failed por {actor}." + else: + new_status = "stale" + retry_sql = "" + error = f"Processing stale há mais de {stale_minutes} minutos; requer revisão manual." + + sql = text(f""" + WITH picked AS ( + SELECT id + FROM integration_outbox + WHERE status = 'processing' + AND locked_at IS NOT NULL + AND locked_at < now() - (:stale_minutes * interval '1 minute') + ORDER BY locked_at ASC + FOR UPDATE SKIP LOCKED + LIMIT :limit + ) + UPDATE integration_outbox io + SET + status = :new_status, + {retry_sql} + locked_at = NULL, + lock_owner = NULL, + last_error = :error, + updated_at = now() + FROM picked + WHERE io.id = picked.id + RETURNING + io.id::text, + io.business_event_id::text, + io.target_system, + io.action_type, + io.payload, + io.status, + io.retry_count, + io.idempotency_key, + io.last_error, + io.created_at, + io.updated_at, + io.sent_at, + io.locked_at, + io.lock_owner, + io.ignored_at + """) + + with engine.begin() as conn: + rows = conn.execute(sql, { + "stale_minutes": stale_minutes, + "limit": int(limit), + "new_status": new_status, + "error": error[:2000], + }).mappings().all() + + recovered = [dict(row) for row in rows] + if recovered: + try: + from app.operator_audit_service import record_operator_action_best_effort + for item in recovered: + record_operator_action_best_effort( + action="outbox_stale_recovered", + entity_type="outbox", + entity_id=item.get("id"), + actor=actor, + payload={ + "mode": mode, + "stale_minutes": stale_minutes, + "new_status": item.get("status"), + "target_system": item.get("target_system"), + "action_type": item.get("action_type"), + }, + ) + except Exception as exc: + print(f"ClientFlow stale outbox audit failed: {exc}", flush=True) + + return recovered + +def mark_outbox_sent(outbox_id: str) -> None: + with engine.begin() as conn: + conn.execute(text(""" + UPDATE integration_outbox + SET + status = 'sent', + sent_at = now(), + locked_at = NULL, + lock_owner = NULL, + updated_at = now(), + last_error = NULL + WHERE id = CAST(:outbox_id AS UUID) + """), {"outbox_id": outbox_id}) + + +def mark_outbox_failed(outbox_id: str, error: str) -> None: + with engine.begin() as conn: + conn.execute(text(""" + UPDATE integration_outbox + SET + status = 'failed', + retry_count = retry_count + 1, + locked_at = NULL, + lock_owner = NULL, + last_error = :error, + updated_at = now() + WHERE id = CAST(:outbox_id AS UUID) + """), { + "outbox_id": outbox_id, + "error": error[:2000], + }) + + +def set_outbox_status( + *, + outbox_id: str, + status: str, + error: str | None = None, +) -> None: + """Atualiza o estado de um item da outbox com semântica operacional clara.""" + allowed = {"pending", "processing", "sent", "failed", "blocked", "dry_run", "ignored", "cancelled", "stale"} + if status not in allowed: + raise ValueError(f"Estado inválido: {status}") + + status_defaults = { + "failed": "Marcado manualmente como failed.", + "blocked": "Bloqueado por configuração ou pré-condição.", + "dry_run": "Validado em OUTBOX_DRY_RUN=true; nenhuma integração real foi executada.", + "ignored": "Ignorado manualmente.", + "cancelled": "Cancelado manualmente.", + "stale": "Processing preso; requer revisão ou reprocessamento manual.", + } + + if status == "sent": + sql = text(""" + UPDATE integration_outbox + SET + status = 'sent', + sent_at = COALESCE(sent_at, now()), + locked_at = NULL, + lock_owner = NULL, + last_error = NULL, + updated_at = now() + WHERE id = CAST(:outbox_id AS UUID) + """) + params = {"outbox_id": outbox_id} + + elif status == "pending": + sql = text(""" + UPDATE integration_outbox + SET + status = 'pending', + sent_at = NULL, + locked_at = NULL, + lock_owner = NULL, + ignored_at = NULL, + last_error = NULL, + updated_at = now() + WHERE id = CAST(:outbox_id AS UUID) + """) + params = {"outbox_id": outbox_id} + + elif status == "processing": + sql = text(""" + UPDATE integration_outbox + SET + status = 'processing', + locked_at = now(), + lock_owner = COALESCE(:error, 'worker'), + updated_at = now() + WHERE id = CAST(:outbox_id AS UUID) + """) + params = {"outbox_id": outbox_id, "error": error} + + else: + ignored_at_sql = "ignored_at = now()," if status == "ignored" else "ignored_at = ignored_at," + retry_sql = "retry_count = retry_count + 1," if status == "failed" else "" + sql = text(f""" + UPDATE integration_outbox + SET + status = :status, + {retry_sql} + sent_at = NULL, + locked_at = NULL, + lock_owner = NULL, + {ignored_at_sql} + last_error = :error, + updated_at = now() + WHERE id = CAST(:outbox_id AS UUID) + """) + params = { + "outbox_id": outbox_id, + "status": status, + "error": (error or status_defaults.get(status) or "Estado atualizado.")[:2000], + } + + with engine.begin() as conn: + conn.execute(sql, params) + + +def mark_outbox_dry_run(outbox_id: str, message: str | None = None) -> None: + set_outbox_status( + outbox_id=outbox_id, + status="dry_run", + error=message or "OUTBOX_DRY_RUN=true; ação não executada na integração externa.", + ) + + +def mark_outbox_blocked(outbox_id: str, message: str | None = None) -> None: + set_outbox_status( + outbox_id=outbox_id, + status="blocked", + error=message or "Integração desativada ou configuração incompleta.", + ) + +def get_outbox_item(outbox_id: str) -> Optional[Dict[str, Any]]: + sql = text(""" + SELECT + id::text, + business_event_id::text, + target_system, + action_type, + payload, + status, + retry_count, + idempotency_key, + last_error, + created_at, + updated_at, + sent_at, + locked_at, + lock_owner, + ignored_at + FROM integration_outbox + WHERE id = CAST(:outbox_id AS UUID) + """) + + with engine.begin() as conn: + row = conn.execute(sql, {"outbox_id": outbox_id}).mappings().first() + + return dict(row) if row else None diff --git a/app/integrations/__init__.py b/app/integrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/jasmin_backfill_service.py b/app/jasmin_backfill_service.py new file mode 100644 index 0000000..73d166c --- /dev/null +++ b/app/jasmin_backfill_service.py @@ -0,0 +1,733 @@ +"""Operator helpers to reimport Jasmin document details into opportunities. + +Used by both CLI backfills and the opportunity detail UI. The function is +idempotent because the underlying reconciliation import upserts commercial +documents, document lines and opportunity items by Jasmin identifiers. +""" +from __future__ import annotations + +from decimal import Decimal, InvalidOperation +from typing import Any, Dict, Iterable, List, Optional +import json + +from sqlalchemy import text + +from app.db import engine +from app.reconciliation_service import ( + _apply_jasmin_documents_to_opportunity, # noqa: PLC2701 - deliberate operator maintenance helper + _jasmin_document_lines_from_item, # noqa: PLC2701 + _jasmin_document_totals, # noqa: PLC2701 +) + + +def _as_text(value: Any) -> str: + return str(value or "").strip() + + +def _json(value: Any) -> str: + return json.dumps(value, ensure_ascii=False, default=str) + + +def _money_value(value: Any) -> Any: + if isinstance(value, dict): + for key in ("amount", "baseAmount", "reportingAmount", "value"): + if value.get(key) not in (None, ""): + return value.get(key) + return None + return value + + +def _decimal_or_none(value: Any) -> Optional[str]: + value = _money_value(value) + if value in (None, ""): + return None + try: + return str(Decimal(str(value).replace(",", ".")).quantize(Decimal("0.01"))) + except (InvalidOperation, ValueError): + return None + + +def _ids_from_metadata(metadata: Any) -> List[str]: + if not isinstance(metadata, dict): + return [] + ids: List[str] = [] + for key in ("created_from_reconciliation_item_id", "reconciliation_item_id"): + value = metadata.get(key) + if value: + ids.append(str(value)) + for key in ("item_ids", "reconciliation_item_ids"): + value = metadata.get(key) + if isinstance(value, list): + ids.extend(str(v) for v in value if v) + return list(dict.fromkeys(ids)) + + +def _record_from_jasmin_item(item: Dict[str, Any]) -> Dict[str, Any]: + payload = item.get("payload") if isinstance(item.get("payload"), dict) else {} + record = payload.get("record") if isinstance(payload.get("record"), dict) else {} + return record + + +def _field_text(record: Dict[str, Any], *keys: str) -> str: + for key in keys: + value = record.get(key) + if value not in (None, ""): + return _as_text(value) + return "" + + +def _field_bool(record: Dict[str, Any], *keys: str) -> bool: + for key in keys: + value = record.get(key) + if isinstance(value, bool): + return value + if isinstance(value, str) and value.strip().lower() in {"true", "1", "yes", "sim"}: + return True + if isinstance(value, (int, float)) and value == 1: + return True + return False + + +def jasmin_document_lifecycle(item: Dict[str, Any]) -> Dict[str, Any]: + """Classify a Jasmin reconciliation item before linking/importing it. + + The Jasmin API stores lifecycle information inside payload.record, not in + reconciliation_items.status (which is ClientFlow resolution status such as + linked/rejected). The UI must not import an old/closed quotation when a newer + open quotation exists for the same fiscal customer. + """ + record = _record_from_jasmin_item(item) + status_desc = _field_text(record, "documentStatusDescription", "statusDescription", "documentLineStatusDescription") + status_code = _field_text(record, "documentStatus", "status") + line_status = " ".join( + _field_text(line, "documentLineStatusDescription", "lineStatusDescription", "statusDescription") + for line in (record.get("documentLines") or []) + if isinstance(line, dict) + ).strip() + text_blob = " ".join([status_desc, status_code, line_status]).casefold() + is_deleted = _field_bool(record, "isDeleted", "deleted") + is_draft = _field_bool(record, "isDraft") + is_completed = _field_bool(record, "statusWasCompleted", "wasCompleted", "completed") + closed_terms = { + "closed", "close", "fechado", "fechada", "completed", "complete", "concluido", "concluído", + "concluida", "concluída", "converted", "convertido", "convertida", "cancelled", "canceled", + "cancelado", "cancelada", "anulado", "anulada", "void", "deleted", "apagado", "apagada", + } + open_terms = {"open", "aberto", "aberta", "active", "ativo", "ativa"} + has_closed_term = any(term in text_blob for term in closed_terms) + has_open_term = any(term in text_blob for term in open_terms) + # In Jasmin quotations observed in production, documentStatus=1 + + # documentStatusDescription=Open means valid/open. Keep code 1 as a weak + # positive only when no closed term exists. + code_open = status_code.strip() == "1" + invalid_reasons: List[str] = [] + if is_deleted: + invalid_reasons.append("deleted") + if is_completed: + invalid_reasons.append("completed") + if has_closed_term: + invalid_reasons.append("closed_status") + if is_draft: + invalid_reasons.append("draft") + is_open = (has_open_term or code_open) and not invalid_reasons + is_valid = is_open and _as_text(item.get("external_type")) in {"jasmin_quotation", "jasmin_proforma"} + if not is_valid and not invalid_reasons: + invalid_reasons.append("not_open_or_not_quotation") + label = status_desc or ("Open" if is_open else "Unknown") + return { + "status_label": label, + "status_code": status_code, + "line_status_label": line_status, + "is_open": bool(is_open), + "is_valid_candidate": bool(is_valid), + "invalid_reason": ",".join(dict.fromkeys(invalid_reasons)), + } + + +def _jasmin_candidate_sort_tuple(item: Dict[str, Any]) -> tuple: + lifecycle = item.get("lifecycle") if isinstance(item.get("lifecycle"), dict) else jasmin_document_lifecycle(item) + valid_rank = 1 if lifecycle.get("is_valid_candidate") else 0 + external_type = _as_text(item.get("external_type")) + type_rank = {"jasmin_quotation": 3, "jasmin_proforma": 2, "jasmin_invoice": 1}.get(external_type, 0) + match_score = int(item.get("match_score") or 0) + record = _record_from_jasmin_item(item) + series_number = 0 + try: + series_number = int(record.get("seriesNumber") or 0) + except Exception: + series_number = 0 + doc_date = _as_text(item.get("document_date") or record.get("documentDate") or record.get("postingDate") or item.get("updated_at")) + return (valid_rank, type_rank, doc_date, series_number, match_score, _as_text(item.get("updated_at"))) + + +def _jasmin_item_recency_tuple(item: Dict[str, Any]) -> tuple: + """Comparable recency key for Jasmin documents. + + Jasmin quotations are naturally ordered by document date and series number. + We use this to prevent an old still-open quotation from being offered as a + replacement when the opportunity already has a newer Jasmin document. + """ + record = _record_from_jasmin_item(item) + series_number = 0 + try: + series_number = int(record.get("seriesNumber") or item.get("series_number") or 0) + except Exception: + series_number = 0 + doc_date = _as_text( + item.get("document_date") + or record.get("documentDate") + or record.get("postingDate") + or item.get("created_at") + or item.get("updated_at") + )[:10] + return (doc_date, series_number, _as_text(item.get("document_number") or item.get("external_id"))) + + +def _commercial_doc_as_jasmin_item(doc: Dict[str, Any]) -> Dict[str, Any]: + payload = doc.get("payload") if isinstance(doc.get("payload"), dict) else {} + record = payload.get("record") if isinstance(payload.get("record"), dict) else payload + return { + "external_type": f"jasmin_{doc.get('document_kind') or 'document'}", + "external_id": doc.get("external_id"), + "document_number": doc.get("document_number"), + "document_date": doc.get("document_date"), + "created_at": doc.get("created_at"), + "updated_at": doc.get("updated_at"), + "payload": {"record": record} if isinstance(record, dict) else {}, + } + + +def load_jasmin_backfill_opportunity(opportunity_id: Optional[str] = None, document_number: Optional[str] = None) -> Optional[Dict[str, Any]]: + with engine.begin() as conn: + if opportunity_id: + row = conn.execute(text(""" + SELECT id::text, title, value_amount, product_interest, local_customer_id::text, + customer_name, customer_email, metadata + FROM opportunities + WHERE id = CAST(:id AS UUID) + LIMIT 1 + """), {"id": opportunity_id}).mappings().first() + return dict(row) if row else None + if document_number: + row = conn.execute(text(""" + SELECT id::text, title, value_amount, product_interest, local_customer_id::text, + customer_name, customer_email, metadata + FROM opportunities + WHERE metadata->>'document_number' = :document_number + OR title ILIKE '%' || :document_number || '%' + ORDER BY updated_at DESC + LIMIT 1 + """), {"document_number": document_number}).mappings().first() + return dict(row) if row else None + return None + + +def load_jasmin_candidate_items_for_opportunity(opportunity: Dict[str, Any]) -> List[Dict[str, Any]]: + metadata = opportunity.get("metadata") if isinstance(opportunity.get("metadata"), dict) else {} + ids = _ids_from_metadata(metadata) + external_id = _as_text(metadata.get("external_id")) + document_number = _as_text(metadata.get("document_number")) + opportunity_id = _as_text(opportunity.get("id")) + + with engine.begin() as conn: + rows = conn.execute(text(""" + SELECT id::text, source_system, external_type, external_id, title, description, + status, priority, suggested_action, confidence, opportunity_id::text, + customer_id::text, customer_name, customer_email, customer_tax_id, + document_number, document_date, amount, currency, payload, + resolution_note, created_at, updated_at, resolved_at + FROM reconciliation_items + WHERE source_system = 'jasmin' + AND ( + opportunity_id = CAST(:opportunity_id AS UUID) + OR (CAST(:ids AS TEXT[]) IS NOT NULL AND id::text = ANY(CAST(:ids AS TEXT[]))) + OR (CAST(:external_id AS TEXT) <> '' AND external_id = CAST(:external_id AS TEXT)) + OR (CAST(:document_number AS TEXT) <> '' AND document_number = CAST(:document_number AS TEXT)) + OR (CAST(:document_number AS TEXT) <> '' AND payload::text ILIKE '%' || CAST(:document_number AS TEXT) || '%') + ) + ORDER BY updated_at DESC, created_at DESC + """), { + "opportunity_id": opportunity_id, + "ids": ids or [], + "external_id": external_id, + "document_number": document_number, + }).mappings().all() + + seen = set() + result: List[Dict[str, Any]] = [] + for row in rows: + item = dict(row) + item_id = item.get("id") + if item_id in seen: + continue + seen.add(item_id) + result.append(item) + return result + + +async def _fetch_jasmin_detail_async(item: Dict[str, Any]) -> Optional[Dict[str, Any]]: + external_type = _as_text(item.get("external_type")) + external_id = _as_text(item.get("external_id")) + if not external_id: + return None + from app.jasmin_client import JasminClient + + client = JasminClient() + if external_type == "jasmin_quotation": + return await client.get_quotation(external_id) + if external_type == "jasmin_invoice": + return await client.get_invoice(external_id) + if external_type == "jasmin_proforma": + try: + return await client.get_quotation(external_id) + except Exception: + return None + return None + + +async def enrich_jasmin_item_with_detail(item: Dict[str, Any], *, fetch_detail: bool = True) -> Dict[str, Any]: + if not fetch_detail: + return item + existing_lines = _jasmin_document_lines_from_item(item) + if existing_lines: + return item + try: + detail = await _fetch_jasmin_detail_async(item) + except Exception as exc: + item = dict(item) + payload = item.get("payload") if isinstance(item.get("payload"), dict) else {} + item["payload"] = {**payload, "detail_fetch_error": f"{type(exc).__name__}: {exc}"} + return item + if not isinstance(detail, dict): + return item + + payload = item.get("payload") if isinstance(item.get("payload"), dict) else {} + enriched = dict(item) + enriched["payload"] = { + **payload, + "record": detail, + "detail_source": "jasmin_api", + "previous_record": payload.get("record"), + } + + record_number = detail.get("documentNumber") or detail.get("naturalKey") or detail.get("number") + if record_number and not enriched.get("document_number"): + enriched["document_number"] = record_number + total = detail.get("payableAmount") or detail.get("totalAmount") or detail.get("grossAmount") or detail.get("amount") + if total and not enriched.get("amount"): + enriched["amount"] = _decimal_or_none(total) or total + return enriched + + +def summary_for_jasmin_items(items: Iterable[Dict[str, Any]]) -> List[Dict[str, Any]]: + result = [] + for item in items: + lifecycle = jasmin_document_lifecycle(item) + result.append({ + "id": item.get("id"), + "external_type": item.get("external_type"), + "external_id": item.get("external_id"), + "document_number": item.get("document_number"), + "amount": item.get("amount"), + "totals": _jasmin_document_totals(item), + "lines": len(_jasmin_document_lines_from_item(item)), + "payload_keys": list((item.get("payload") or {}).keys()) if isinstance(item.get("payload"), dict) else [], + "lifecycle": lifecycle, + }) + return result + + +def jasmin_backfill_post_import_summary(opportunity_id: str) -> Dict[str, Any]: + with engine.begin() as conn: + opportunity = conn.execute(text(""" + SELECT id::text, title, value_amount, product_interest, metadata + FROM opportunities + WHERE id = CAST(:id AS UUID) + """), {"id": opportunity_id}).mappings().first() + docs = conn.execute(text(""" + SELECT id::text, document_kind, document_number, amount, total_amount, currency, document_date + FROM commercial_documents + WHERE opportunity_id = CAST(:id AS UUID) + ORDER BY created_at DESC + """), {"id": opportunity_id}).mappings().all() + items = conn.execute(text(""" + SELECT product_name, quantity, unit_price, total_price, jasmin_sales_item + FROM opportunity_items + WHERE opportunity_id = CAST(:id AS UUID) + ORDER BY created_at + """), {"id": opportunity_id}).mappings().all() + lines = conn.execute(text(""" + SELECT cdl.description, cdl.quantity, cdl.unit_price, cdl.total_amount, cdl.jasmin_sales_item + FROM commercial_document_lines cdl + JOIN commercial_documents cd ON cd.id = cdl.document_id + WHERE cd.opportunity_id = CAST(:id AS UUID) + ORDER BY cdl.line_index + """), {"id": opportunity_id}).mappings().all() + return { + "opportunity": dict(opportunity or {}), + "documents": [dict(r) for r in docs], + "opportunity_items": [dict(r) for r in items], + "document_lines": [dict(r) for r in lines], + } + + +async def backfill_jasmin_opportunity_details_async( + *, + opportunity_id: Optional[str] = None, + document_number: Optional[str] = None, + fetch_detail: bool = True, + actor: str = "operator_ui_reimport", + dry_run: bool = False, +) -> Dict[str, Any]: + opportunity = load_jasmin_backfill_opportunity(opportunity_id=opportunity_id, document_number=document_number) + if not opportunity: + return {"ok": False, "error": "opportunity_not_found"} + + items = load_jasmin_candidate_items_for_opportunity(opportunity) + enriched_items = [await enrich_jasmin_item_with_detail(item, fetch_detail=fetch_detail) for item in items] + result: Dict[str, Any] = { + "ok": bool(enriched_items), + "opportunity_id": opportunity.get("id"), + "title": opportunity.get("title"), + "candidate_items": summary_for_jasmin_items(enriched_items), + "dry_run": dry_run, + } + if not enriched_items: + result["error"] = "no_jasmin_reconciliation_items_found" + return result + if dry_run: + return result + + with engine.begin() as conn: + import_result = _apply_jasmin_documents_to_opportunity( + conn, + enriched_items, + str(opportunity["id"]), + actor=actor, + ) + result["import_result"] = import_result + result["summary"] = jasmin_backfill_post_import_summary(str(opportunity["id"])) + return result + + +def _email_domain(email: Any) -> str: + text_value = _as_text(email).lower().strip().strip(';') + if "@" not in text_value: + return "" + return text_value.rsplit("@", 1)[-1].strip() + + +def find_jasmin_document_candidates_for_opportunity(opportunity_id: str, *, limit: int = 5) -> List[Dict[str, Any]]: + """Find Jasmin documents that probably belong to an opportunity but are not imported yet. + + This is deliberately conservative and prioritizes fiscal identity (NIF/customer_id) + over weak name/domain matching. It is used in the opportunity detail UI before + offering to create a new Jasmin quotation, so operators can associate an existing + quotation instead of duplicating it. + """ + with engine.begin() as conn: + opp = conn.execute(text(""" + SELECT + o.id::text, + o.customer_name, + o.customer_email, + o.local_customer_id::text, + c.name AS fiscal_customer_name, + c.tax_id AS fiscal_customer_tax_id, + c.email AS fiscal_customer_email + FROM opportunities o + LEFT JOIN customers c ON c.id = o.local_customer_id + WHERE o.id = CAST(:id AS UUID) + LIMIT 1 + """), {"id": opportunity_id}).mappings().first() + if not opp: + return [] + + fiscal_tax_id = _as_text(opp.get("fiscal_customer_tax_id")) + fiscal_customer_id = _as_text(opp.get("local_customer_id")) + fiscal_name = _as_text(opp.get("fiscal_customer_name")) + fiscal_email = _as_text(opp.get("fiscal_customer_email") or opp.get("customer_email")).lower() + domain = _email_domain(fiscal_email) + + current_docs = conn.execute(text(""" + SELECT + id::text, document_number, external_id, document_kind, document_date, + total_amount, amount, payload, created_at, updated_at + FROM commercial_documents + WHERE opportunity_id = CAST(:opportunity_id AS UUID) + AND system = 'jasmin' + AND document_kind IN ('quotation', 'proforma') + ORDER BY document_date DESC NULLS LAST, created_at DESC + """), {"opportunity_id": opportunity_id}).mappings().all() + + rows = conn.execute(text(""" + SELECT + ri.id::text, + ri.source_system, + ri.external_type, + ri.external_id, + ri.title, + ri.status, + ri.opportunity_id::text, + ri.customer_id::text, + ri.customer_name, + ri.customer_email, + ri.customer_tax_id, + ri.document_number, + ri.document_date, + ri.amount, + ri.currency, + ri.payload, + ri.created_at, + ri.updated_at, + CASE + WHEN CAST(:fiscal_customer_id AS TEXT) <> '' AND ri.customer_id::text = CAST(:fiscal_customer_id AS TEXT) THEN 100 + WHEN CAST(:fiscal_tax_id AS TEXT) <> '' AND ri.customer_tax_id = CAST(:fiscal_tax_id AS TEXT) THEN 98 + WHEN CAST(:fiscal_tax_id AS TEXT) <> '' AND ri.payload::text ILIKE '%' || CAST(:fiscal_tax_id AS TEXT) || '%' THEN 96 + WHEN CAST(:fiscal_email AS TEXT) <> '' AND LOWER(COALESCE(ri.customer_email, '')) = CAST(:fiscal_email AS TEXT) THEN 90 + WHEN CAST(:domain AS TEXT) <> '' AND ri.payload::text ILIKE '%' || CAST(:domain AS TEXT) || '%' THEN 78 + WHEN CAST(:fiscal_name AS TEXT) <> '' AND LOWER(COALESCE(ri.customer_name, '')) = LOWER(CAST(:fiscal_name AS TEXT)) THEN 75 + ELSE 0 + END AS match_score, + CASE + WHEN CAST(:fiscal_customer_id AS TEXT) <> '' AND ri.customer_id::text = CAST(:fiscal_customer_id AS TEXT) THEN 'customer_id' + WHEN CAST(:fiscal_tax_id AS TEXT) <> '' AND ri.customer_tax_id = CAST(:fiscal_tax_id AS TEXT) THEN 'nif' + WHEN CAST(:fiscal_tax_id AS TEXT) <> '' AND ri.payload::text ILIKE '%' || CAST(:fiscal_tax_id AS TEXT) || '%' THEN 'payload_nif' + WHEN CAST(:fiscal_email AS TEXT) <> '' AND LOWER(COALESCE(ri.customer_email, '')) = CAST(:fiscal_email AS TEXT) THEN 'email' + WHEN CAST(:domain AS TEXT) <> '' AND ri.payload::text ILIKE '%' || CAST(:domain AS TEXT) || '%' THEN 'domain_payload' + WHEN CAST(:fiscal_name AS TEXT) <> '' AND LOWER(COALESCE(ri.customer_name, '')) = LOWER(CAST(:fiscal_name AS TEXT)) THEN 'exact_name' + ELSE 'none' + END AS match_reason + FROM reconciliation_items ri + WHERE ri.source_system = 'jasmin' + AND ri.external_type IN ('jasmin_quotation', 'jasmin_proforma', 'jasmin_invoice') + AND (ri.opportunity_id IS NULL OR ri.opportunity_id = CAST(:opportunity_id AS UUID)) + AND NOT EXISTS ( + SELECT 1 FROM commercial_documents cd + WHERE cd.opportunity_id = CAST(:opportunity_id AS UUID) + AND ( + (ri.document_number IS NOT NULL AND cd.document_number = ri.document_number) + OR (ri.external_id IS NOT NULL AND cd.external_id = ri.external_id) + ) + ) + AND ( + (CAST(:fiscal_customer_id AS TEXT) <> '' AND ri.customer_id::text = CAST(:fiscal_customer_id AS TEXT)) + OR (CAST(:fiscal_tax_id AS TEXT) <> '' AND ri.customer_tax_id = CAST(:fiscal_tax_id AS TEXT)) + OR (CAST(:fiscal_tax_id AS TEXT) <> '' AND ri.payload::text ILIKE '%' || CAST(:fiscal_tax_id AS TEXT) || '%') + OR (CAST(:fiscal_email AS TEXT) <> '' AND LOWER(COALESCE(ri.customer_email, '')) = CAST(:fiscal_email AS TEXT)) + OR (CAST(:domain AS TEXT) <> '' AND ri.payload::text ILIKE '%' || CAST(:domain AS TEXT) || '%') + OR (CAST(:fiscal_name AS TEXT) <> '' AND LOWER(COALESCE(ri.customer_name, '')) = LOWER(CAST(:fiscal_name AS TEXT))) + ) + ORDER BY match_score DESC, ri.document_date DESC NULLS LAST, ri.updated_at DESC + LIMIT :limit + """), { + "opportunity_id": opportunity_id, + "fiscal_customer_id": fiscal_customer_id, + "fiscal_tax_id": fiscal_tax_id, + "fiscal_email": fiscal_email, + "domain": domain, + "fiscal_name": fiscal_name, + "limit": int(limit), + }).mappings().all() + + current_doc_items = [_commercial_doc_as_jasmin_item(dict(row)) for row in current_docs] + current_latest_key = max((_jasmin_item_recency_tuple(item) for item in current_doc_items), default=None) + current_doc_numbers = { + _as_text(row.get("document_number")) + for row in current_docs + if _as_text(row.get("document_number")) + } + current_external_ids = { + _as_text(row.get("external_id")) + for row in current_docs + if _as_text(row.get("external_id")) + } + + candidates: List[Dict[str, Any]] = [] + ignored: List[Dict[str, Any]] = [] + hidden_old_count = 0 + for row in rows: + item = dict(row) + item_doc_number = _as_text(item.get("document_number")) + item_external_id = _as_text(item.get("external_id")) + if item_doc_number in current_doc_numbers or item_external_id in current_external_ids: + continue + + totals = _jasmin_document_totals(item) + lines = _jasmin_document_lines_from_item(item) + lifecycle = jasmin_document_lifecycle(item) + + # If the opportunity already has a Jasmin quotation/proforma, do not show + # older/equal still-open quotations as actionable candidates. They caused + # operators to import ORC.137 after ORC.158 already existed. Newer valid + # documents still appear as replacement candidates. + recency_key = _jasmin_item_recency_tuple(item) + if current_latest_key and recency_key <= current_latest_key: + hidden_old_count += 1 + continue + + item["totals"] = totals + item["line_count"] = len(lines) + item["lifecycle"] = lifecycle + item["jasmin_status_label"] = lifecycle.get("status_label") + item["jasmin_status_code"] = lifecycle.get("status_code") + item["is_valid_candidate"] = lifecycle.get("is_valid_candidate") + item["invalid_reason"] = lifecycle.get("invalid_reason") + if lifecycle.get("is_valid_candidate"): + candidates.append(item) + else: + ignored.append(item) + candidates.sort(key=_jasmin_candidate_sort_tuple, reverse=True) + ignored.sort(key=_jasmin_candidate_sort_tuple, reverse=True) + + result = (candidates + ignored)[: int(limit)] + if hidden_old_count and result: + result[0]["hidden_older_candidates_count"] = hidden_old_count + return result + + +async def replace_jasmin_document_for_opportunity_async( + *, + opportunity_id: str, + item_id: str, + actor: str = "operator_replace_jasmin_document", + dry_run: bool = False, +) -> Dict[str, Any]: + """Remove current imported Jasmin quotation/proforma details and import a valid candidate. + + This is intended for operator repair when an old/closed quotation was linked + by mistake. It only removes ClientFlow imported Jasmin artifacts from this + opportunity; it does not delete anything in Jasmin. + """ + with engine.begin() as conn: + current_docs = conn.execute(text(""" + SELECT id::text, document_number, external_id, document_kind, status, total_amount + FROM commercial_documents + WHERE opportunity_id = CAST(:opportunity_id AS UUID) + AND system = 'jasmin' + AND document_kind IN ('quotation', 'proforma') + ORDER BY created_at DESC + """), {"opportunity_id": opportunity_id}).mappings().all() + current_items = conn.execute(text(""" + SELECT id::text, product_name, total_price, metadata + FROM opportunity_items + WHERE opportunity_id = CAST(:opportunity_id AS UUID) + AND (metadata->>'source_system' = 'jasmin' OR status = 'JASMIN_IMPORTED') + ORDER BY created_at + """), {"opportunity_id": opportunity_id}).mappings().all() + + if dry_run: + return { + "ok": True, + "dry_run": True, + "would_remove_documents": [dict(r) for r in current_docs], + "would_remove_items": [dict(r) for r in current_items], + "candidate_item_id": item_id, + } + + # Validate candidate before deleting anything. + with engine.begin() as conn: + candidate = conn.execute(text(""" + SELECT id::text, source_system, external_type, payload, document_number, opportunity_id::text + FROM reconciliation_items + WHERE id = CAST(:item_id AS UUID) + LIMIT 1 + """), {"item_id": item_id}).mappings().first() + if not candidate: + return {"ok": False, "error": "jasmin_item_not_found"} + lifecycle = jasmin_document_lifecycle(dict(candidate)) + if candidate.get("source_system") != "jasmin" or not lifecycle.get("is_valid_candidate"): + return {"ok": False, "error": "jasmin_candidate_not_open_or_valid", "lifecycle": lifecycle} + + with engine.begin() as conn: + conn.execute(text(""" + DELETE FROM commercial_documents + WHERE opportunity_id = CAST(:opportunity_id AS UUID) + AND system = 'jasmin' + AND document_kind IN ('quotation', 'proforma') + """), {"opportunity_id": opportunity_id}) + conn.execute(text(""" + DELETE FROM opportunity_items + WHERE opportunity_id = CAST(:opportunity_id AS UUID) + AND (metadata->>'source_system' = 'jasmin' OR status = 'JASMIN_IMPORTED') + """), {"opportunity_id": opportunity_id}) + conn.execute(text(""" + UPDATE reconciliation_items + SET opportunity_id = NULL, + status = CASE WHEN status = 'linked' THEN 'open' ELSE status END, + updated_at = now(), + resolution_note = COALESCE(resolution_note, '') || '\nDesassociado por substituição de orçamento Jasmin em ClientFlow.' + WHERE source_system = 'jasmin' + AND opportunity_id = CAST(:opportunity_id AS UUID) + AND id <> CAST(:item_id AS UUID) + """), {"opportunity_id": opportunity_id, "item_id": item_id}) + + result = await link_and_import_jasmin_candidate_async(opportunity_id=opportunity_id, item_id=item_id, actor=actor) + result["removed_documents"] = len(current_docs) + result["removed_items"] = len(current_items) + return result + + +async def link_and_import_jasmin_candidate_async( + *, + opportunity_id: str, + item_id: str, + actor: str = "operator_ui_link_existing_jasmin", +) -> Dict[str, Any]: + """Link an existing Jasmin reconciliation item to an opportunity and import its details.""" + with engine.begin() as conn: + item = conn.execute(text(""" + SELECT id::text, source_system, external_type, opportunity_id::text, document_number, payload + FROM reconciliation_items + WHERE id = CAST(:item_id AS UUID) + LIMIT 1 + """), {"item_id": item_id}).mappings().first() + if not item: + return {"ok": False, "error": "jasmin_item_not_found"} + if item.get("source_system") != "jasmin": + return {"ok": False, "error": "item_is_not_jasmin"} + lifecycle = jasmin_document_lifecycle(item) + if not lifecycle.get("is_valid_candidate"): + return { + "ok": False, + "error": "jasmin_candidate_not_open_or_valid", + "status": lifecycle.get("status_label"), + "reason": lifecycle.get("invalid_reason"), + } + current_opportunity_id = _as_text(item.get("opportunity_id")) + if current_opportunity_id and current_opportunity_id != opportunity_id: + return {"ok": False, "error": "jasmin_item_already_linked_to_other_opportunity", "linked_opportunity_id": current_opportunity_id} + conn.execute(text(""" + UPDATE reconciliation_items + SET opportunity_id = CAST(:opportunity_id AS UUID), + status = 'linked', + resolved_at = COALESCE(resolved_at, now()), + updated_at = now(), + resolution_note = COALESCE(resolution_note, 'Associado a oportunidade pela ficha comercial'), + payload = COALESCE(payload, '{}'::jsonb) || jsonb_build_object( + 'manual_opportunity_link', jsonb_build_object( + 'actor', CAST(:actor AS TEXT), + 'opportunity_id', CAST(:opportunity_id AS TEXT), + 'linked_at', now() + ) + ) + WHERE id = CAST(:item_id AS UUID) + """), {"item_id": item_id, "opportunity_id": opportunity_id, "actor": actor}) + conn.execute(text(""" + INSERT INTO opportunity_events (id, opportunity_id, event_type, note, payload, created_by) + VALUES (gen_random_uuid(), CAST(:opportunity_id AS UUID), 'jasmin_existing_document_linked', :note, CAST(:payload AS JSONB), :actor) + """), { + "opportunity_id": opportunity_id, + "note": "Documento Jasmin existente associado à oportunidade.", + "payload": _json({"reconciliation_item_id": item_id, "document_number": item.get("document_number")}), + "actor": actor, + }) + + result = await backfill_jasmin_opportunity_details_async( + opportunity_id=opportunity_id, + fetch_detail=True, + actor=actor, + dry_run=False, + ) + result["linked_item_id"] = item_id + return result diff --git a/app/jasmin_client.py b/app/jasmin_client.py new file mode 100644 index 0000000..7288212 --- /dev/null +++ b/app/jasmin_client.py @@ -0,0 +1,234 @@ +"""Cliente REST mínimo para Cegid Jasmin. + +Validado contra Jasmin 3.02: +- OAuth Client Credentials em identity.primaverabss.com/connect/token; +- endpoints em https://my.jasminsoftware.com/api/{account}/{subscription}; +- OData com $top máximo 100; +- invoice from quotation exige body JSON vazio ({}) para evitar HTTP 411. +""" + +from __future__ import annotations + +import time +from dataclasses import dataclass +from typing import Any, Dict, List, Optional + +import httpx + +from app.config import settings + + +class JasminError(RuntimeError): + pass + + +@dataclass(frozen=True) +class JasminConfig: + base_url: str + token_url: str + account: str + subscription: str + client_id: str + client_secret: str + scope: str = "application" + + +_TOKEN_CACHE: Dict[str, Any] = {"access_token": "", "expires_at": 0.0} + + +def get_jasmin_config() -> JasminConfig: + cfg = JasminConfig( + base_url=(settings.jasmin_base_url or "https://my.jasminsoftware.com").rstrip("/"), + token_url=(settings.jasmin_token_url or "https://identity.primaverabss.com/connect/token").strip(), + account=(settings.jasmin_account or "").strip(), + subscription=(settings.jasmin_subscription or "").strip(), + client_id=(settings.jasmin_client_id or "").strip(), + client_secret=(settings.jasmin_client_secret or "").strip(), + scope=(settings.jasmin_scope or "application").strip() or "application", + ) + missing = [name for name, value in { + "JASMIN_ACCOUNT": cfg.account, + "JASMIN_SUBSCRIPTION": cfg.subscription, + "JASMIN_CLIENT_ID": cfg.client_id, + "JASMIN_CLIENT_SECRET": cfg.client_secret, + }.items() if not value] + if missing: + raise JasminError("Configuração Jasmin incompleta: " + ", ".join(missing)) + return cfg + + +class JasminClient: + def __init__(self, config: Optional[JasminConfig] = None, timeout: float = 30.0): + self.config = config or get_jasmin_config() + self.timeout = timeout + + @property + def api_root(self) -> str: + return f"{self.config.base_url}/api/{self.config.account}/{self.config.subscription}" + + async def get_token(self, *, force_refresh: bool = False) -> str: + now = time.time() + if not force_refresh and _TOKEN_CACHE.get("access_token") and float(_TOKEN_CACHE.get("expires_at") or 0) > now + 60: + return str(_TOKEN_CACHE["access_token"]) + + data = {"grant_type": "client_credentials", "scope": self.config.scope} + async with httpx.AsyncClient(timeout=self.timeout) as client: + response = await client.post( + self.config.token_url, + data=data, + auth=(self.config.client_id, self.config.client_secret), + headers={"Content-Type": "application/x-www-form-urlencoded", "Accept": "application/json"}, + ) + if response.status_code >= 400: + raise JasminError(f"OAuth Jasmin falhou {response.status_code}: {response.text}") + payload = response.json() + token = payload.get("access_token") + if not token: + raise JasminError(f"OAuth Jasmin não devolveu access_token: {payload}") + expires_in = int(payload.get("expires_in") or 3600) + _TOKEN_CACHE.update({"access_token": token, "expires_at": now + expires_in}) + return str(token) + + async def _request( + self, + method: str, + path: str, + *, + params: Optional[Dict[str, Any]] = None, + json_body: Any = None, + headers: Optional[Dict[str, str]] = None, + ) -> Any: + token = await self.get_token() + url = f"{self.api_root}/{path.lstrip('/')}" + request_headers = { + "Authorization": f"Bearer {token}", + "Accept": "application/json", + } + if json_body is not None: + request_headers["Content-Type"] = "application/json" + if headers: + request_headers.update(headers) + async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client: + response = await client.request( + method.upper(), + url, + params=params, + json=json_body, + headers=request_headers, + ) + if response.status_code >= 400: + detail: Any = response.text + try: + detail = response.json() + except Exception: + pass + raise JasminError(f"Jasmin error {response.status_code} em {method.upper()} {path}: {detail}") + if not response.content: + return None + try: + return response.json() + except Exception: + return response.text + + + async def _raw_request( + self, + method: str, + path: str, + *, + accept: str = "application/pdf", + json_body: Any = None, + ) -> tuple[bytes, str]: + """Pedido autenticado para respostas binárias, usado por PDFs Jasmin.""" + token = await self.get_token() + url = f"{self.api_root}/{path.lstrip('/')}" + headers = {"Authorization": f"Bearer {token}", "Accept": accept} + if json_body is not None: + headers["Content-Type"] = "application/json" + async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client: + response = await client.request(method.upper(), url, json=json_body, headers=headers) + if response.status_code >= 400: + detail: Any = response.text + try: + detail = response.json() + except Exception: + pass + raise JasminError(f"Jasmin error {response.status_code} em {method.upper()} {path}: {detail}") + return response.content, response.headers.get("content-type", accept) + + async def print_quotation_pdf(self, quotation_id: str) -> tuple[bytes, str]: + return await self._raw_request("GET", f"/sales/quotations/{quotation_id}/print", accept="application/pdf") + + async def print_invoice_pdf(self, invoice_id: str) -> tuple[bytes, str]: + return await self._raw_request("GET", f"/billing/invoices/{invoice_id}/print", accept="application/pdf") + + async def get_versions(self) -> Dict[str, Any]: + return await self._request("GET", "/businessCore/productInfos/getVersions") + + async def list_customers_odata(self, *, top: int = 100, skip: int = 0) -> Dict[str, Any]: + top = min(max(int(top), 1), 100) + return await self._request("GET", "/salesCore/customerParties/extension/odata", params={"$top": top, "$skip": int(skip)}) + + async def get_customer_by_tax_id(self, tax_id: str) -> Any: + return await self._request("GET", f"/salesCore/customerParties/getCustomerByCompanyTaxId/{tax_id}") + + async def get_customer_by_party_key(self, party_key: str) -> Dict[str, Any]: + return await self._request("GET", f"/salesCore/customerParties/{party_key}") + + async def create_customer(self, payload: Dict[str, Any]) -> str: + result = await self._request("POST", "/salesCore/customerParties", json_body=payload) + return str(result).strip('"') + + async def list_sales_items(self, *, top: int = 100, skip: int = 0) -> Dict[str, Any]: + top = min(max(int(top), 1), 100) + return await self._request("GET", "/salesCore/salesItems/extension/odata", params={"$top": top, "$skip": int(skip)}) + + async def get_sales_item(self, item_key: str) -> Dict[str, Any]: + return await self._request("GET", f"/salesCore/salesItems/{item_key}") + + async def create_quotation(self, payload: Dict[str, Any]) -> str: + result = await self._request("POST", "/sales/quotations", json_body=payload) + return str(result).strip('"') + + async def get_quotation(self, quotation_id: str) -> Dict[str, Any]: + return await self._request("GET", f"/sales/quotations/{quotation_id}") + + async def list_quotations( + self, + *, + top: int = 100, + skip: int = 0, + filter: Optional[str] = None, + orderby: Optional[str] = None, + ) -> Dict[str, Any]: + top = min(max(int(top), 1), 100) + params: Dict[str, Any] = {"$top": top, "$skip": int(skip)} + if filter: + params["$filter"] = filter + if orderby: + params["$orderby"] = orderby + return await self._request("GET", "/sales/quotations/odata", params=params) + + async def create_invoice_from_quotation(self, quotation_id: str) -> str: + # Body vazio é obrigatório no tenant testado; sem body devolve HTTP 411. + result = await self._request("POST", f"/billing/invoices/fromQuotation/{quotation_id}", json_body={}) + return str(result).strip('"') + + async def get_invoice(self, invoice_id: str) -> Dict[str, Any]: + return await self._request("GET", f"/billing/invoices/{invoice_id}") + + async def list_invoices( + self, + *, + top: int = 100, + skip: int = 0, + filter: Optional[str] = None, + orderby: Optional[str] = None, + ) -> Dict[str, Any]: + top = min(max(int(top), 1), 100) + params: Dict[str, Any] = {"$top": top, "$skip": int(skip)} + if filter: + params["$filter"] = filter + if orderby: + params["$orderby"] = orderby + return await self._request("GET", "/billing/invoices/odata", params=params) diff --git a/app/jasmin_service.py b/app/jasmin_service.py new file mode 100644 index 0000000..a7aae8d --- /dev/null +++ b/app/jasmin_service.py @@ -0,0 +1,638 @@ +"""Serviço Jasmin para ClientFlow. + +Camada de negócio validada com testes reais: +- NIF pesquisado sem prefixo PT; +- cliente novo criado sem enviar campos opcionais vazios; +- orçamento ORC/ORC2026 criado via POST /sales/quotations; +- fatura criada via POST /billing/invoices/fromQuotation/{id} com body {}. +""" + +from __future__ import annotations + +import json +import os +import re +import uuid +from datetime import date +from decimal import Decimal +from typing import Any, Dict, List, Optional, Tuple + +from sqlalchemy import text + +from app.commercial_service import ( + add_document_lines, + create_commercial_document, + find_invoice_for_parent, + get_customer_by_tax_id as get_local_customer_by_tax_id, + get_customer_for_opportunity, + link_customer_to_opportunity, + get_commercial_document, + get_latest_active_quotation, + list_commercial_documents, + mark_document_status, + update_commercial_document_details, + normalize_tax_id, + upsert_customer, +) +from app.config import settings +from app.db import engine +from app.integration_outbox_service import create_outbox_item +from app.jasmin_client import JasminClient, JasminError +from app.operation_service import register_operation_action +from app.opportunity_service import get_opportunity, set_opportunity_stage +from app.product_service import list_opportunity_items + + +class JasminPayloadError(ValueError): + pass + + +def _env(name: str, default: str = "") -> str: + return os.getenv(name, default).strip() + + +def _setting(name: str, default: str = "") -> str: + return str(getattr(settings, name, "") or default).strip() + + +def _compact(value: Any, limit: int = 180) -> str: + value = re.sub(r"\s+", " ", str(value or "")).strip() + return value[:limit].strip() + + +def _metadata_dict(row: Dict[str, Any]) -> Dict[str, Any]: + value = row.get("metadata") or {} + if isinstance(value, dict): + return value + try: + return json.loads(value) + except Exception: + return {} + + +def _without_empty(payload: Dict[str, Any]) -> Dict[str, Any]: + return {k: v for k, v in payload.items() if v not in (None, "", [], {})} + + +def _as_decimal(value: Any, default: str = "0") -> Decimal: + try: + if value is None or str(value).strip() == "": + return Decimal(default) + return Decimal(str(value).replace(",", ".").strip()) + except Exception: + return Decimal(default) + + +def _money_amount(value: Any) -> Optional[str]: + if value is None: + return None + if isinstance(value, dict): + for key in ("amount", "value", "baseAmount", "reportingAmount"): + if value.get(key) is not None: + return str(_as_decimal(value.get(key))) + return None + return str(_as_decimal(value)) + + +def _first_present(data: Dict[str, Any], *keys: str) -> Any: + for key in keys: + if data.get(key) not in (None, ""): + return data.get(key) + return None + + +def _safe_int(value: Any) -> Optional[int]: + try: + if value is None or str(value).strip() == "": + return None + return int(value) + except Exception: + return None + + +def _normalize_doc_details(raw: Dict[str, Any]) -> Dict[str, Any]: + """Extrai campos estáveis de uma resposta de documento Jasmin.""" + if not isinstance(raw, dict): + return {"payload": {"jasmin_details": raw}} + document_type = _first_present(raw, "documentType", "documentTypeKey") + serie = _first_present(raw, "serie", "serieKey") + series_number = _safe_int(_first_present(raw, "seriesNumber", "number")) + document_number = _first_present(raw, "documentNumber", "naturalKey", "documentNo") + if not document_number and document_type and serie and series_number is not None: + document_number = f"{document_type} {serie}/{series_number}" + total_amount = ( + _money_amount(raw.get("payableAmount")) + or _money_amount(raw.get("totalAmount")) + or _money_amount(raw.get("payableAmountAmount")) + or _money_amount(raw.get("amount")) + ) + tax_amount = _money_amount(raw.get("taxExclusiveAmount")) or _money_amount(raw.get("taxAmount")) + currency = _first_present(raw, "currency", "currencyKey") + return { + "company": _first_present(raw, "company", "companyKey"), + "document_type": document_type, + "serie": serie, + "series_number": series_number, + "document_number": document_number, + "amount": total_amount, + "tax_amount": tax_amount, + "total_amount": total_amount, + "currency": currency, + "document_date": str(_first_present(raw, "documentDate", "postingDate") or "")[:10] or None, + "due_date": str(_first_present(raw, "dueDate") or "")[:10] or None, + "payload": {"jasmin_details": raw}, + } + + +def _find_first(metadata: Dict[str, Any], keys: List[str]) -> str: + for key in keys: + value = metadata.get(key) + if isinstance(value, dict): + # permitir metadata.customer.tax_id, metadata.billing.address, etc. + for nested in keys: + if nested in value and value[nested]: + return _compact(value[nested]) + elif value: + return _compact(value) + return "" + + +def split_pt_address(address: str) -> Tuple[str, str, str]: + clean = re.sub(r"\s+", " ", str(address or "").replace("\n", " ")).strip() + if not clean: + return "", "", "" + match = re.search(r"\b(\d{4}-\d{3}|\d{4})\b\s*(.*)$", clean) + if not match: + return clean, "", "" + return clean[: match.start()].strip(" ,-"), match.group(1), _compact(match.group(2), 80) + + +def extract_customer_data_from_opportunity(opportunity: Dict[str, Any]) -> Dict[str, Any]: + metadata = _metadata_dict(opportunity) + customer_meta = metadata.get("customer") if isinstance(metadata.get("customer"), dict) else {} + billing_meta = metadata.get("billing") if isinstance(metadata.get("billing"), dict) else {} + merged = {**metadata, **customer_meta, **billing_meta} + + name = _compact( + merged.get("name") + or merged.get("customer_name") + or merged.get("company_name") + or opportunity.get("customer_name") + ) + tax_id = normalize_tax_id( + merged.get("tax_id") + or merged.get("nif") + or merged.get("vat") + or merged.get("companyTaxID") + or merged.get("customer_tax_id") + ) + street = _compact( + merged.get("street_name") + or merged.get("streetName") + or merged.get("address") + or merged.get("customer_address") + or merged.get("billing_address") + ) + parsed_street, parsed_zip, parsed_city = split_pt_address(street) + postal_zone = _compact(merged.get("postal_zone") or merged.get("postalZone") or merged.get("zip") or merged.get("postal_code") or parsed_zip) + city_name = _compact(merged.get("city_name") or merged.get("cityName") or merged.get("city") or parsed_city) + if parsed_street: + street = parsed_street + + return _without_empty({ + "name": name, + "tax_id": tax_id, + "email": _compact(merged.get("email") or merged.get("electronicMail") or opportunity.get("customer_email")), + "phone": _compact(merged.get("phone") or merged.get("telephone") or opportunity.get("customer_phone")), + "street_name": street, + "postal_zone": postal_zone, + "city_name": city_name, + "country": _compact(merged.get("country") or settings.jasmin_default_country or "PT", 2).upper(), + "metadata": {"source": "opportunity", "opportunity_id": str(opportunity.get("id"))}, + }) + + +def build_jasmin_customer_payload(customer: Dict[str, Any]) -> Dict[str, Any]: + tax_id = normalize_tax_id(customer.get("tax_id")) + if not tax_id: + raise JasminPayloadError("NIF em falta para criar cliente Jasmin") + name = _compact(customer.get("name"), 120) + if not name: + raise JasminPayloadError("Nome do cliente em falta") + payload = { + "partyKey": customer.get("jasmin_customer_party_key") or f"CF{tax_id}", + "name": name, + "companyTaxID": tax_id, + "electronicMail": _compact(customer.get("email"), 120), + "telephone": _compact(customer.get("phone"), 40), + "streetName": _compact(customer.get("street_name"), 160), + "postalZone": _compact(customer.get("postal_zone"), 20), + "cityName": _compact(customer.get("city_name"), 80), + "country": _compact(customer.get("country") or settings.jasmin_default_country or "PT", 2).upper(), + "customerGroup": settings.jasmin_default_customer_group, + "priceList": settings.jasmin_default_price_list, + "paymentMethod": settings.jasmin_default_payment_method, + "paymentTerm": settings.jasmin_default_payment_term, + "partyTaxSchema": settings.jasmin_default_party_tax_schema, + "deliveryTerm": settings.jasmin_default_delivery_term, + "currency": settings.jasmin_default_currency, + "endCustomer": True, + "oneTimeCustomer": False, + "isPerson": False, + } + # Jasmin rejeitou ElectronicMail/Telephone vazios nos testes. Omitir campos opcionais vazios. + return _without_empty(payload) + + +async def find_or_create_customer_for_opportunity(opportunity_id: str) -> Dict[str, Any]: + opportunity = get_opportunity(opportunity_id) + if not opportunity: + raise JasminPayloadError(f"Oportunidade não encontrada: {opportunity_id}") + + linked_customer = get_customer_for_opportunity(opportunity_id) + if linked_customer: + data = {**linked_customer, **{"metadata": {"source": "linked_customer", "opportunity_id": opportunity_id}}} + else: + data = extract_customer_data_from_opportunity(opportunity) + + tax_id = normalize_tax_id(data.get("tax_id")) + if not tax_id: + raise JasminPayloadError("NIF do cliente em falta. Associe um cliente à oportunidade ou preencha metadata.customer_tax_id/metadata.nif.") + + local = get_local_customer_by_tax_id(tax_id) + if local and local.get("jasmin_customer_party_key"): + try: + link_customer_to_opportunity(local["id"], opportunity_id) + except Exception: + pass + return local + + client = JasminClient() + result = await client.get_customer_by_tax_id(tax_id) + # Quando não existe, Jasmin devolve lista com {message: ...}; quando existe devolve objeto. + if isinstance(result, dict) and result.get("customerPartyKey"): + customer = upsert_customer({ + **data, + "name": result.get("customerName") or data.get("name"), + "tax_id": tax_id, + "jasmin_customer_party_key": result.get("customerPartyKey"), + "jasmin_customer_id": result.get("id"), + "metadata": {"jasmin_lookup": result}, + }) + try: + link_customer_to_opportunity(customer["id"], opportunity_id) + except Exception: + pass + return customer + + if not data.get("name"): + raise JasminPayloadError("Cliente não existe no Jasmin e falta nome para criar.") + + payload = build_jasmin_customer_payload(data) + jasmin_id = await client.create_customer(payload) + # Confirmar por NIF para obter customerPartyKey final. + confirm = await client.get_customer_by_tax_id(tax_id) + party_key = payload.get("partyKey") + customer_name = data.get("name") + if isinstance(confirm, dict): + party_key = confirm.get("customerPartyKey") or party_key + customer_name = confirm.get("customerName") or customer_name + customer = upsert_customer({ + **data, + "name": customer_name, + "tax_id": tax_id, + "jasmin_customer_party_key": party_key, + "jasmin_customer_id": jasmin_id, + "metadata": {"jasmin_created": {"id": jasmin_id, "payload": payload, "confirm": confirm}}, + }) + try: + link_customer_to_opportunity(customer["id"], opportunity_id) + except Exception: + pass + return customer + + +def _line_metadata(item: Dict[str, Any]) -> Dict[str, Any]: + value = item.get("metadata") or {} + if isinstance(value, dict): + return value + try: + return json.loads(value) + except Exception: + return {} + + +def build_quotation_lines(opportunity_id: str) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]: + items = list_opportunity_items(opportunity_id) + active = [item for item in items if str(item.get("status") or "").upper() not in {"REJECTED", "CANCELLED"}] + if not active: + raise JasminPayloadError("A oportunidade não tem linhas de produto ativas.") + + document_lines: List[Dict[str, Any]] = [] + local_lines: List[Dict[str, Any]] = [] + today = date.today().isoformat() + for item in active: + meta = _line_metadata(item) + jasmin_item = _compact( + meta.get("jasmin_sales_item") + or meta.get("jasmin_item_key") + or meta.get("salesItem") + or item.get("jasmin_sales_item") + or settings.jasmin_default_sales_item + or item.get("sku") + ) + if not jasmin_item: + raise JasminPayloadError(f"Linha sem salesItem Jasmin: {item.get('product_name')}") + unit = _compact(meta.get("unit") or meta.get("jasmin_unit") or settings.jasmin_default_unit or "UN", 12) + tax_schema = _compact(meta.get("itemTaxSchema") or meta.get("jasmin_item_tax_schema") or settings.jasmin_default_item_tax_schema or "NORMAL", 40) + quantity = _as_decimal(item.get("quantity"), "1") + unit_price = _as_decimal(item.get("unit_price"), "0") + description = _compact(item.get("description") or item.get("product_name") or jasmin_item, 180) + document_lines.append({ + "description": description, + "quantity": float(quantity), + "unit": unit, + "unitPrice": {"amount": float(unit_price)}, + "itemTaxSchema": tax_schema, + "salesItem": jasmin_item, + "documentLineStatus": 1, + "deliveryDate": today, + }) + local_lines.append({ + "opportunity_item_id": item.get("id"), + "local_product_id": item.get("product_id"), + "jasmin_sales_item": jasmin_item, + "description": description, + "quantity": str(quantity), + "unit": unit, + "unit_price": str(unit_price), + "tax_schema": tax_schema, + "total_amount": str(_as_decimal(item.get("total_price"), str(quantity * unit_price))), + "payload": {"opportunity_item": item, "jasmin_line": document_lines[-1]}, + }) + return document_lines, local_lines + + +def validate_opportunity_for_quotation(opportunity_id: str) -> List[str]: + """Valida dados antes de criar outbox/orçamento Jasmin. + + Esta validação evita a sensação de "cliquei e depois falhou no worker" para + problemas previsíveis: cliente sem NIF, sem ficha local, ou linhas sem artigo + Jasmin. O worker continua a validar novamente ao montar o payload. + """ + errors: List[str] = [] + customer = get_customer_for_opportunity(opportunity_id) + if not customer: + errors.append("Associar uma ficha de cliente à oportunidade.") + else: + if not _compact(customer.get("name")): + errors.append("Cliente sem nome fiscal.") + if not normalize_tax_id(customer.get("tax_id")): + errors.append("Cliente sem NIF válido.") + # Se o cliente ainda não existe no Jasmin, a criação precisa de morada. + if not _compact(customer.get("jasmin_customer_party_key")): + if not _compact(customer.get("street_name")): + errors.append("Cliente sem morada fiscal.") + if not _compact(customer.get("postal_zone")): + errors.append("Cliente sem código postal.") + if not _compact(customer.get("city_name")): + errors.append("Cliente sem cidade.") + + try: + items = list_opportunity_items(opportunity_id) + except Exception as exc: + return [f"Não foi possível ler produtos da oportunidade: {exc}"] + + active = [item for item in items if str(item.get("status") or "").upper() not in {"REJECTED", "CANCELLED"}] + if not active: + errors.append("Adicionar pelo menos um produto ativo à oportunidade.") + + for item in active: + name = _compact(item.get("product_name") or item.get("sku") or "Produto") + jasmin_item = _compact(item.get("jasmin_sales_item")) + if not jasmin_item: + errors.append(f"Produto '{name}' sem Artigo Jasmin.") + if _as_decimal(item.get("unit_price"), "0") <= 0: + errors.append(f"Produto '{name}' com preço unitário zero ou inválido.") + + return errors + + +def assert_opportunity_ready_for_quotation(opportunity_id: str) -> None: + errors = validate_opportunity_for_quotation(opportunity_id) + if errors: + raise JasminPayloadError("Não é possível criar orçamento:\n- " + "\n- ".join(errors)) + + +def build_quotation_payload(opportunity_id: str, customer: Dict[str, Any]) -> Tuple[Dict[str, Any], List[Dict[str, Any]]]: + document_lines, local_lines = build_quotation_lines(opportunity_id) + today = date.today().isoformat() + customer_party_key = customer.get("jasmin_customer_party_key") + if not customer_party_key: + raise JasminPayloadError("Cliente local sem jasmin_customer_party_key") + + payload = { + "documentType": settings.jasmin_quotation_type, + "serie": settings.jasmin_quotation_serie, + "seriesNumber": 0, + "company": settings.jasmin_company_key, + "documentDate": today, + "postingDate": today, + "buyerCustomerParty": customer_party_key, + "buyerCustomerPartyName": customer.get("name"), + "exchangeRate": 1, + "discount": 0, + "paymentMethod": settings.jasmin_default_payment_method, + "paymentTerm": settings.jasmin_default_payment_term, + "currency": settings.jasmin_default_currency, + "deliveryTerm": settings.jasmin_default_delivery_term, + "priceList": settings.jasmin_default_price_list, + "remarks": f"Orçamento criado via API ClientFlow para oportunidade {opportunity_id}.", + "documentLines": document_lines, + } + missing = [name for name in ["documentType", "serie", "company"] if not payload.get(name)] + if missing: + raise JasminPayloadError("Configuração Jasmin em falta: " + ", ".join(missing)) + return payload, local_lines + + +async def create_quotation_for_opportunity(opportunity_id: str) -> Dict[str, Any]: + customer = await find_or_create_customer_for_opportunity(opportunity_id) + payload, local_lines = build_quotation_payload(opportunity_id, customer) + quotation_id = await JasminClient().create_quotation(payload) + + amount = sum(_as_decimal(line.get("total_amount"), "0") for line in local_lines) + doc = create_commercial_document( + document_kind="quotation", + customer_id=customer.get("id"), + opportunity_id=opportunity_id, + system="jasmin", + external_id=quotation_id, + company=payload.get("company"), + document_type=payload.get("documentType"), + serie=payload.get("serie"), + customer_party_key=customer.get("jasmin_customer_party_key"), + status="created", + amount=str(amount), + total_amount=str(amount), + currency=payload.get("currency") or "EUR", + payload={"jasmin_payload": payload, "jasmin_id": quotation_id}, + document_date=payload.get("documentDate"), + ) + try: + details = await JasminClient().get_quotation(quotation_id) + doc = update_commercial_document_details(doc["id"], _normalize_doc_details(details)) or doc + except Exception as exc: + doc = update_commercial_document_details(doc["id"], {"payload": {"jasmin_detail_warning": str(exc)}}) or doc + add_document_lines(doc["id"], local_lines) + + try: + register_operation_action( + opportunity_id, + "jasmin_quotation", + external_id=quotation_id, + external_name=f"{payload.get('documentType')} {payload.get('serie')}", + payload={"commercial_document_id": doc.get("id"), "customer_party_key": customer.get("jasmin_customer_party_key")}, + created_by="jasmin_service", + ) + except Exception: + # operation_links é compatibilidade visual; não deve falhar o fluxo principal. + pass + set_opportunity_stage(opportunity_id, "QUOTE_SENT", note="Orçamento Jasmin criado via ClientFlow.", created_by="jasmin_service") + return {"customer": customer, "quotation": doc, "quotation_id": quotation_id, "payload": payload} + + +async def convert_latest_quotation_to_invoice(opportunity_id: str) -> Dict[str, Any]: + quotation = get_latest_active_quotation(opportunity_id) + if not quotation or not quotation.get("external_id"): + raise JasminPayloadError("Não existe orçamento Jasmin ativo para converter.") + existing_invoice = find_invoice_for_parent(quotation["id"]) + if existing_invoice: + raise JasminPayloadError(f"Este orçamento já tem fatura associada: {existing_invoice.get('external_id')}") + + invoice_id = await JasminClient().create_invoice_from_quotation(str(quotation["external_id"])) + invoice_doc = create_commercial_document( + document_kind="invoice", + customer_id=quotation.get("customer_id"), + opportunity_id=opportunity_id, + system="jasmin", + external_id=invoice_id, + company=quotation.get("company"), + customer_party_key=quotation.get("customer_party_key"), + status="issued", + amount=quotation.get("amount"), + total_amount=quotation.get("total_amount") or quotation.get("amount"), + currency=quotation.get("currency") or "EUR", + parent_document_id=quotation.get("id"), + payload={"from_quotation": quotation, "jasmin_invoice_id": invoice_id}, + document_date=date.today().isoformat(), + ) + try: + details = await JasminClient().get_invoice(invoice_id) + invoice_doc = update_commercial_document_details(invoice_doc["id"], _normalize_doc_details(details)) or invoice_doc + except Exception as exc: + invoice_doc = update_commercial_document_details(invoice_doc["id"], {"payload": {"jasmin_detail_warning": str(exc)}}) or invoice_doc + mark_document_status(quotation["id"], "converted", {"invoice_document_id": invoice_doc.get("id"), "invoice_id": invoice_id}) + + try: + register_operation_action( + opportunity_id, + "jasmin_invoice", + external_id=invoice_id, + external_name="Fatura Jasmin", + payload={"commercial_document_id": invoice_doc.get("id"), "parent_quotation_id": quotation.get("external_id")}, + created_by="jasmin_service", + ) + except Exception: + pass + set_opportunity_stage(opportunity_id, "INVOICED", note="Orçamento Jasmin convertido em fatura.", created_by="jasmin_service") + return {"quotation": quotation, "invoice": invoice_doc, "invoice_id": invoice_id} + + + +async def refresh_commercial_document_from_jasmin(document_id: str) -> Dict[str, Any]: + """Atualiza número/série/valor de um documento já criado no Jasmin.""" + doc = get_commercial_document(document_id) + if not doc: + raise JasminPayloadError("Documento comercial não encontrado.") + if str(doc.get("system") or "jasmin") != "jasmin": + raise JasminPayloadError("Documento não pertence ao sistema Jasmin.") + external_id = str(doc.get("external_id") or "").strip() + if not external_id: + raise JasminPayloadError("Documento sem external_id Jasmin.") + + client = JasminClient() + kind = str(doc.get("document_kind") or "") + if kind == "quotation": + details = await client.get_quotation(external_id) + elif kind == "invoice": + details = await client.get_invoice(external_id) + else: + raise JasminPayloadError(f"Tipo de documento Jasmin não suportado: {kind}") + + updated = update_commercial_document_details(document_id, _normalize_doc_details(details)) + return updated or doc + + +async def get_commercial_document_pdf(document_id: str) -> tuple[Dict[str, Any], bytes, str]: + """Obtém PDF de orçamento/fatura Jasmin para download via ClientFlow.""" + doc = get_commercial_document(document_id) + if not doc: + raise JasminPayloadError("Documento comercial não encontrado.") + external_id = str(doc.get("external_id") or "").strip() + if not external_id: + raise JasminPayloadError("Documento sem external_id Jasmin.") + + client = JasminClient() + kind = str(doc.get("document_kind") or "") + if kind == "quotation": + data, content_type = await client.print_quotation_pdf(external_id) + elif kind == "invoice": + data, content_type = await client.print_invoice_pdf(external_id) + else: + raise JasminPayloadError(f"Tipo de documento sem PDF Jasmin suportado: {kind}") + return doc, data, content_type + +def enqueue_create_quotation(opportunity_id: str, *, created_by: str = "operator") -> str: + assert_opportunity_ready_for_quotation(opportunity_id) + outbox_id = create_outbox_item( + business_event_id=str(uuid.uuid4()), + target_system="jasmin", + action_type="create_quotation", + payload={"opportunity_id": opportunity_id, "created_by": created_by}, + idempotency_key=f"jasmin:quotation:{opportunity_id}", + ) + if not outbox_id: + raise JasminPayloadError("Já existe pedido de orçamento Jasmin com a mesma chave de idempotência.") + return outbox_id + + +def enqueue_convert_latest_to_invoice(opportunity_id: str, *, created_by: str = "operator") -> str: + quotation = get_latest_active_quotation(opportunity_id) + quotation_external_id = str((quotation or {}).get("external_id") or "") + if not quotation_external_id: + raise JasminPayloadError("Não existe orçamento Jasmin ativo para faturar.") + outbox_id = create_outbox_item( + business_event_id=str(uuid.uuid4()), + target_system="jasmin", + action_type="convert_quotation_to_invoice", + payload={"opportunity_id": opportunity_id, "quotation_external_id": quotation_external_id, "created_by": created_by}, + idempotency_key=f"jasmin:invoice_from_quotation:{quotation_external_id}", + ) + if not outbox_id: + raise JasminPayloadError("Já existe pedido de conversão desta proposta em fatura.") + return outbox_id + + +async def process_create_quotation_outbox(payload: Dict[str, Any]) -> Dict[str, Any]: + opportunity_id = str(payload.get("opportunity_id") or "").strip() + if not opportunity_id: + raise JasminPayloadError("opportunity_id em falta") + return await create_quotation_for_opportunity(opportunity_id) + + +async def process_convert_invoice_outbox(payload: Dict[str, Any]) -> Dict[str, Any]: + opportunity_id = str(payload.get("opportunity_id") or "").strip() + if not opportunity_id: + raise JasminPayloadError("opportunity_id em falta") + return await convert_latest_quotation_to_invoice(opportunity_id) diff --git a/app/logistics/__init__.py b/app/logistics/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/main.py b/app/main.py new file mode 100644 index 0000000..7d4e1cb --- /dev/null +++ b/app/main.py @@ -0,0 +1,42 @@ +from fastapi import FastAPI + +from app.analyzer import analyze +from app.config import settings +from app.schemas import AnalyzeRequest, AnalyzeResponse +from app.webhooks_chatwoot import router as chatwoot_router +from app.db import init_db +from app.admin_ui.router import router as admin_dashboard_router +from app.api.internal import router as internal_api_router + + +app = FastAPI( + title="ClientFlow MVP", + description="Motor de contexto comercial com Qwen3 30B + regras ClientFlow.", + version="0.1.0", +) + + +app.include_router(chatwoot_router) +app.include_router(internal_api_router) + + +@app.on_event("startup") +async def startup_event() -> None: + init_db() + + +@app.get("/health") +async def health() -> dict: + return { + "status": "ok", + "app": settings.app_name, + "env": settings.env, + "model": settings.openrouter_model, + } + + +@app.post("/analyze", response_model=AnalyzeResponse) +async def analyze_endpoint(request: AnalyzeRequest) -> AnalyzeResponse: + return await analyze(request) + +app.include_router(admin_dashboard_router) diff --git a/app/mautic_client.py b/app/mautic_client.py new file mode 100644 index 0000000..6f53e1b --- /dev/null +++ b/app/mautic_client.py @@ -0,0 +1,92 @@ +import os +from typing import Any, Dict + +import httpx + + +def env_bool(name: str, default: bool = False) -> bool: + fallback = "true" if default else "false" + value = os.getenv(name, fallback).strip().lower() + return value in {"true", "1", "yes", "on"} + + +def get_mautic_config() -> Dict[str, str]: + return { + "base_url": os.getenv("MAUTIC_BASE_URL", "").rstrip("/"), + "api_token": os.getenv("MAUTIC_API_TOKEN", ""), + "add_tag_url": os.getenv("MAUTIC_ADD_TAG_URL", ""), + "remove_tag_url": os.getenv("MAUTIC_REMOVE_TAG_URL", ""), + } + + +def build_tag_payload(payload: Dict[str, Any]) -> Dict[str, Any]: + return { + "tag": payload.get("tag"), + "contact_id": payload.get("contact_id"), + "conversation_id": payload.get("conversation_id"), + "event_type": payload.get("event_type"), + "source": "clientflow", + "metadata": payload, + } + + +def add_tag_from_outbox_payload(payload: Dict[str, Any]) -> Dict[str, Any]: + if not env_bool("MAUTIC_WRITE_ENABLED", False): + raise RuntimeError("MAUTIC_WRITE_ENABLED=false") + + config = get_mautic_config() + + if not config["api_token"]: + raise RuntimeError("MAUTIC_API_TOKEN em falta") + + if not config["add_tag_url"]: + raise RuntimeError("MAUTIC_ADD_TAG_URL em falta") + + headers = { + "Content-Type": "application/json", + "Authorization": f"Bearer {config['api_token']}", + } + + body = build_tag_payload(payload) + + with httpx.Client(timeout=30) as client: + response = client.post(config["add_tag_url"], headers=headers, json=body) + + if response.status_code >= 400: + raise RuntimeError(f"Mautic error {response.status_code}: {response.text}") + + try: + return response.json() + except Exception: + return {"status_code": response.status_code, "text": response.text} + + +def remove_tag_from_outbox_payload(payload: Dict[str, Any]) -> Dict[str, Any]: + if not env_bool("MAUTIC_WRITE_ENABLED", False): + raise RuntimeError("MAUTIC_WRITE_ENABLED=false") + + config = get_mautic_config() + + if not config["api_token"]: + raise RuntimeError("MAUTIC_API_TOKEN em falta") + + if not config["remove_tag_url"]: + raise RuntimeError("MAUTIC_REMOVE_TAG_URL em falta") + + headers = { + "Content-Type": "application/json", + "Authorization": f"Bearer {config['api_token']}", + } + + body = build_tag_payload(payload) + + with httpx.Client(timeout=30) as client: + response = client.post(config["remove_tag_url"], headers=headers, json=body) + + if response.status_code >= 400: + raise RuntimeError(f"Mautic error {response.status_code}: {response.text}") + + try: + return response.json() + except Exception: + return {"status_code": response.status_code, "text": response.text} diff --git a/app/message_cleaner.py b/app/message_cleaner.py new file mode 100644 index 0000000..ce662e3 --- /dev/null +++ b/app/message_cleaner.py @@ -0,0 +1,65 @@ +from typing import Any, Dict + + +def get_nested(data: Dict[str, Any], *keys: str) -> Any: + current: Any = data + + for key in keys: + if not isinstance(current, dict): + return None + current = current.get(key) + + return current + + +def clean_email_reply_text(content: str) -> str: + text = str(content or "").replace("\r\n", "\n").replace("\r", "\n").strip() + + cut_markers = [ + "\nDe:", + "\nEnviada:", + "\nEnviado:", + "\nAssunto:", + "\nPara:", + "\nCc:", + "\nÀs ", + "\nOn ", + "\nFrom:", + "\nSent:", + "\nSubject:", + "\nTo:", + "\n-----Original Message-----", + "\n________________________________", + ] + + cut_at = len(text) + + for marker in cut_markers: + idx = text.find(marker) + if idx != -1: + cut_at = min(cut_at, idx) + + cleaned = text[:cut_at].strip() + + lines = [] + for line in cleaned.split("\n"): + if line.strip().startswith(">"): + continue + lines.append(line) + + return "\n".join(lines).strip() + + +def extract_chatwoot_content(payload: Dict[str, Any], message: Dict[str, Any]) -> str: + content = ( + get_nested(message, "content_attributes", "email", "html_content", "reply") + or get_nested(message, "content_attributes", "email", "text_content", "reply") + or get_nested(payload, "content_attributes", "email", "html_content", "reply") + or get_nested(payload, "content_attributes", "email", "text_content", "reply") + or message.get("processed_message_content") + or message.get("content") + or payload.get("content") + or "" + ) + + return clean_email_reply_text(str(content or "")) diff --git a/app/models.py b/app/models.py new file mode 100644 index 0000000..75e9753 --- /dev/null +++ b/app/models.py @@ -0,0 +1,8 @@ +""" +Sem modelos ORM ativos. + +ClientFlow Action Core usa SQL explícito em: +- app/persistence.py +- app/task_service.py +- scripts/migrate_clean_action_core_schema.py +""" diff --git a/app/odoo_client.py b/app/odoo_client.py new file mode 100644 index 0000000..eb3490e --- /dev/null +++ b/app/odoo_client.py @@ -0,0 +1,126 @@ +from __future__ import annotations + +import ssl +import xmlrpc.client +from dataclasses import dataclass +from typing import Any, Dict, List, Optional + +from app.config import settings + + +class OdooIntegrationError(RuntimeError): + pass + + +@dataclass +class OdooConfig: + base_url: str + db: str + username: str + api_key: str + + +def get_odoo_config() -> OdooConfig: + base_url = (settings.odoo_base_url or "").rstrip("/") + db = settings.odoo_db or "" + username = settings.odoo_username or "" + api_key = settings.odoo_api_key or "" + + if not base_url: + raise OdooIntegrationError("ODOO_BASE_URL não está configurado.") + if not db: + raise OdooIntegrationError("ODOO_DB não está configurado.") + if not username: + raise OdooIntegrationError("ODOO_USERNAME não está configurado.") + if not api_key: + raise OdooIntegrationError("ODOO_API_KEY não está configurado.") + + return OdooConfig(base_url=base_url, db=db, username=username, api_key=api_key) + + +class OdooClient: + def __init__(self, config: Optional[OdooConfig] = None): + self.config = config or get_odoo_config() + self._uid: Optional[int] = None + context = ssl._create_unverified_context() + + self.common = xmlrpc.client.ServerProxy( + f"{self.config.base_url}/xmlrpc/2/common", + allow_none=True, + context=context, + ) + self.models = xmlrpc.client.ServerProxy( + f"{self.config.base_url}/xmlrpc/2/object", + allow_none=True, + context=context, + ) + + def version(self) -> Dict[str, Any]: + try: + return dict(self.common.version()) + except Exception as exc: + raise OdooIntegrationError(f"Falha ao consultar versão Odoo: {exc}") from exc + + def authenticate(self) -> int: + if self._uid: + return self._uid + try: + uid = self.common.authenticate( + self.config.db, + self.config.username, + self.config.api_key, + {}, + ) + except Exception as exc: + raise OdooIntegrationError(f"Falha de autenticação Odoo: {exc}") from exc + + if not uid: + raise OdooIntegrationError("Autenticação Odoo falhou. Verifica ODOO_USERNAME e ODOO_API_KEY/password.") + + self._uid = int(uid) + return self._uid + + def execute_kw( + self, + model: str, + method: str, + args: Optional[List[Any]] = None, + kwargs: Optional[Dict[str, Any]] = None, + ) -> Any: + uid = self.authenticate() + try: + return self.models.execute_kw( + self.config.db, + uid, + self.config.api_key, + model, + method, + args or [], + kwargs or {}, + ) + except Exception as exc: + raise OdooIntegrationError(f"Odoo {model}.{method} falhou: {exc}") from exc + + def search_read( + self, + model: str, + domain: Optional[List[Any]] = None, + fields: Optional[List[str]] = None, + *, + limit: int = 500, + offset: int = 0, + order: str = "id asc", + context: Optional[Dict[str, Any]] = None, + ) -> List[Dict[str, Any]]: + kwargs: Dict[str, Any] = { + "fields": fields or [], + "limit": int(limit), + "offset": int(offset), + "order": order, + } + if context is not None: + kwargs["context"] = context + return list(self.execute_kw(model, "search_read", [domain or []], kwargs) or []) + + def count(self, model: str, domain: Optional[List[Any]] = None) -> int: + return int(self.execute_kw(model, "search_count", [domain or []]) or 0) diff --git a/app/odoo_service.py b/app/odoo_service.py new file mode 100644 index 0000000..af4331c --- /dev/null +++ b/app/odoo_service.py @@ -0,0 +1,669 @@ +from __future__ import annotations + +import json +import uuid +from collections import defaultdict +from datetime import datetime, timezone +from decimal import Decimal, InvalidOperation +from typing import Any, Dict, List, Optional, Tuple + +from sqlalchemy import text + +from app.config import settings +from app.db import engine +from app.odoo_client import OdooClient + + +_SCHEMA_READY = False + + +def _json(value: Any) -> str: + return json.dumps(value or {}, ensure_ascii=False, default=str) + + +def _money(value: Any, default: str = "0") -> str: + try: + if value is None or str(value).strip() == "": + return f"{Decimal(default):.2f}" + return f"{Decimal(str(value).replace(',', '.')).quantize(Decimal('0.01'))}" + except (InvalidOperation, ValueError): + return f"{Decimal(default):.2f}" + + +def _m2o_id(value: Any) -> Optional[int]: + if isinstance(value, (list, tuple)) and value: + try: + return int(value[0]) + except Exception: + return None + if isinstance(value, int): + return int(value) + return None + + +def _m2o_name(value: Any) -> str: + if isinstance(value, (list, tuple)) and len(value) > 1: + return str(value[1] or "") + return "" + + +def ensure_odoo_schema() -> None: + global _SCHEMA_READY + if _SCHEMA_READY: + return + + with engine.begin() as conn: + conn.execute(text(""" + CREATE TABLE IF NOT EXISTS odoo_sync_runs ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + sync_type TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'running', + started_at TIMESTAMPTZ NOT NULL DEFAULT now(), + finished_at TIMESTAMPTZ, + total_seen INTEGER NOT NULL DEFAULT 0, + total_changed INTEGER NOT NULL DEFAULT 0, + total_errors INTEGER NOT NULL DEFAULT 0, + message TEXT, + payload JSONB NOT NULL DEFAULT '{}'::jsonb + ) + """)) + conn.execute(text("CREATE INDEX IF NOT EXISTS idx_odoo_sync_runs_type_started ON odoo_sync_runs(sync_type, started_at DESC)")) + conn.execute(text("CREATE INDEX IF NOT EXISTS idx_products_metadata_odoo_id ON products ((metadata->'odoo'->>'product_id'))")) + conn.execute(text("CREATE INDEX IF NOT EXISTS idx_products_metadata_odoo_sync ON products ((metadata->'odoo'->>'last_synced_at'))")) + + _SCHEMA_READY = True + + +def test_odoo_connection() -> Dict[str, Any]: + client = OdooClient() + version = client.version() + uid = client.authenticate() + counts = {} + for model in ["product.product", "stock.quant", "mrp.bom", "mrp.production", "sale.order"]: + try: + counts[model] = client.count(model, []) + except Exception as exc: + counts[model] = f"erro: {exc}" + + return { + "ok": True, + "base_url": settings.odoo_base_url, + "db": settings.odoo_db, + "username": settings.odoo_username, + "uid": uid, + "version": version, + "counts": counts, + } + + +def _start_sync(sync_type: str, payload: Optional[Dict[str, Any]] = None) -> str: + ensure_odoo_schema() + sync_id = str(uuid.uuid4()) + with engine.begin() as conn: + conn.execute(text(""" + INSERT INTO odoo_sync_runs (id, sync_type, status, payload) + VALUES (CAST(:id AS UUID), :sync_type, 'running', CAST(:payload AS JSONB)) + """), {"id": sync_id, "sync_type": sync_type, "payload": _json(payload or {})}) + return sync_id + + +def _finish_sync(sync_id: str, *, status: str, total_seen: int = 0, total_changed: int = 0, total_errors: int = 0, message: str = "", payload: Optional[Dict[str, Any]] = None) -> None: + with engine.begin() as conn: + conn.execute(text(""" + UPDATE odoo_sync_runs + SET status = :status, + finished_at = now(), + total_seen = :total_seen, + total_changed = :total_changed, + total_errors = :total_errors, + message = :message, + payload = payload || CAST(:payload AS JSONB) + WHERE id = CAST(:id AS UUID) + """), { + "id": sync_id, + "status": status, + "total_seen": int(total_seen), + "total_changed": int(total_changed), + "total_errors": int(total_errors), + "message": message, + "payload": _json(payload or {}), + }) + + +def list_odoo_sync_runs(limit: int = 20) -> List[Dict[str, Any]]: + ensure_odoo_schema() + with engine.begin() as conn: + rows = conn.execute(text(""" + SELECT id::text, sync_type, status, started_at, finished_at, + total_seen, total_changed, total_errors, message, payload + FROM odoo_sync_runs + ORDER BY started_at DESC + LIMIT :limit + """), {"limit": int(limit)}).mappings().all() + return [dict(row) for row in rows] + + +def _fetch_stock_by_product(client: OdooClient, *, limit: int = 10000) -> Dict[int, Dict[str, float]]: + fields = ["product_id", "location_id", "quantity", "reserved_quantity"] + try: + quants = client.search_read( + "stock.quant", + [["location_id.usage", "=", "internal"]], + fields, + limit=limit, + context={"active_test": False}, + ) + except Exception: + fields = ["product_id", "location_id", "quantity"] + quants = client.search_read( + "stock.quant", + [["location_id.usage", "=", "internal"]], + fields, + limit=limit, + context={"active_test": False}, + ) + + by_product: Dict[int, Dict[str, float]] = defaultdict(lambda: {"quantity": 0.0, "reserved": 0.0}) + for q in quants: + pid = _m2o_id(q.get("product_id")) + if not pid: + continue + by_product[pid]["quantity"] += float(q.get("quantity") or 0) + by_product[pid]["reserved"] += float(q.get("reserved_quantity") or 0) + + for vals in by_product.values(): + vals["available"] = vals["quantity"] - vals["reserved"] + + return by_product + + +def _fetch_bom_index(client: OdooClient, *, limit: int = 5000) -> Tuple[Dict[int, int], Dict[int, int]]: + boms = client.search_read( + "mrp.bom", + [], + ["id", "product_id", "product_tmpl_id", "type", "active"], + limit=limit, + context={"active_test": False}, + ) + + by_product: Dict[int, int] = defaultdict(int) + by_template: Dict[int, int] = defaultdict(int) + + for bom in boms: + product_id = _m2o_id(bom.get("product_id")) + template_id = _m2o_id(bom.get("product_tmpl_id")) + if product_id: + by_product[product_id] += 1 + if template_id: + by_template[template_id] += 1 + + return dict(by_product), dict(by_template) + + +def _upsert_product(row: Dict[str, Any], metadata: Dict[str, Any]) -> bool: + odoo_product_id = int(row["id"]) + sku = str(row.get("default_code") or "").strip() + if not sku: + sku = f"ODOO-{odoo_product_id}" + + name = str(row.get("display_name") or row.get("name") or sku).strip() + category = _m2o_name(row.get("categ_id")) or "Odoo" + price = _money(row.get("lst_price") if "lst_price" in row else row.get("list_price")) + + with engine.begin() as conn: + result = conn.execute(text(""" + INSERT INTO products ( + id, sku, name, category, description, + default_unit_price, vat_rate, active, metadata, created_at, updated_at + ) + VALUES ( + CAST(:id AS UUID), :sku, :name, :category, :description, + :default_unit_price, 23, :active, CAST(:metadata AS JSONB), now(), now() + ) + ON CONFLICT (sku) + DO UPDATE SET + name = EXCLUDED.name, + category = EXCLUDED.category, + default_unit_price = EXCLUDED.default_unit_price, + active = EXCLUDED.active, + metadata = products.metadata || EXCLUDED.metadata, + updated_at = now() + RETURNING id + """), { + "id": str(uuid.uuid4()), + "sku": sku, + "name": name, + "category": category, + "description": str(row.get("description_sale") or ""), + "default_unit_price": price, + "active": bool(row.get("active", True)), + "metadata": _json({"odoo": metadata}), + }) + return result.rowcount > 0 + + +def sync_odoo_products(*, limit: int = 500, include_inactive: bool = True) -> Dict[str, Any]: + ensure_odoo_schema() + sync_id = _start_sync("products", {"limit": limit, "include_inactive": include_inactive}) + + total_seen = 0 + total_changed = 0 + total_errors = 0 + errors: List[str] = [] + + try: + client = OdooClient() + client.authenticate() + + stock_by_product = _fetch_stock_by_product(client) + bom_by_product, bom_by_template = _fetch_bom_index(client) + + fields = [ + "id", + "display_name", + "default_code", + "active", + "lst_price", + "standard_price", + "categ_id", + "product_tmpl_id", + "sale_ok", + "purchase_ok", + "type", + "description_sale", + ] + + try: + products = client.search_read( + "product.product", + [["sale_ok", "=", True]], + fields, + limit=int(limit), + context={"active_test": not include_inactive}, + ) + except Exception: + fallback_fields = ["id", "display_name", "default_code", "active", "lst_price", "categ_id", "product_tmpl_id", "sale_ok", "purchase_ok"] + products = client.search_read( + "product.product", + [["sale_ok", "=", True]], + fallback_fields, + limit=int(limit), + context={"active_test": not include_inactive}, + ) + + synced_at = datetime.now(timezone.utc).isoformat() + + for product in products: + total_seen += 1 + try: + product_id = int(product["id"]) + template_id = _m2o_id(product.get("product_tmpl_id")) + stock = stock_by_product.get(product_id, {"quantity": 0.0, "reserved": 0.0, "available": 0.0}) + bom_count = int(bom_by_product.get(product_id, 0) + (bom_by_template.get(template_id, 0) if template_id else 0)) + + metadata = { + "source": "odoo", + "product_id": product_id, + "template_id": template_id, + "last_synced_at": synced_at, + "sale_ok": bool(product.get("sale_ok")), + "purchase_ok": bool(product.get("purchase_ok")), + "type": product.get("type"), + "cost": product.get("standard_price"), + "stock": { + "quantity_on_hand": stock["quantity"], + "reserved": stock["reserved"], + "available": stock["available"], + }, + "has_bom": bom_count > 0, + "bom_count": bom_count, + } + + if _upsert_product(product, metadata): + total_changed += 1 + except Exception as exc: + total_errors += 1 + errors.append(f"Produto {product.get('id')}: {exc}") + + status = "success" if total_errors == 0 else "partial" + _finish_sync(sync_id, status=status, total_seen=total_seen, total_changed=total_changed, total_errors=total_errors, message="sync produtos concluído", payload={"errors": errors[:20]}) + + return { + "ok": total_errors == 0, + "sync_id": sync_id, + "status": status, + "total_seen": total_seen, + "total_changed": total_changed, + "total_errors": total_errors, + "errors": errors[:20], + } + + except Exception as exc: + _finish_sync(sync_id, status="failed", total_seen=total_seen, total_changed=total_changed, total_errors=total_errors + 1, message=str(exc), payload={"errors": errors[:20]}) + raise + + +def get_odoo_product_snapshot(limit: int = 20) -> Dict[str, Any]: + ensure_odoo_schema() + with engine.begin() as conn: + stats = conn.execute(text(""" + SELECT + count(*) FILTER (WHERE metadata ? 'odoo') AS synced_products, + count(*) FILTER (WHERE COALESCE((metadata->'odoo'->>'has_bom')::boolean, false)) AS products_with_bom, + count(*) FILTER (WHERE COALESCE((metadata->'odoo'->'stock'->>'available')::numeric, 0) > 0) AS products_with_available_stock, + max(metadata->'odoo'->>'last_synced_at') AS last_synced_at + FROM products + """)).mappings().first() + + rows = conn.execute(text(""" + SELECT + id::text, + sku, + name, + category, + default_unit_price, + active, + metadata->'odoo' AS odoo + FROM products + WHERE metadata ? 'odoo' + ORDER BY name + LIMIT :limit + """), {"limit": int(limit)}).mappings().all() + + return { + "stats": dict(stats or {}), + "products": [dict(row) for row in rows], + "sync_runs": list_odoo_sync_runs(limit=10), + } + +# === ClientFlow Odoo physical status integration === + +def _compact_m2o(value): + if isinstance(value, (list, tuple)) and len(value) >= 2: + return {"id": value[0], "name": value[1]} + if isinstance(value, int): + return {"id": value, "name": ""} + return {"id": None, "name": ""} + + +def _find_odoo_sale_order_link(opportunity_id: str) -> dict: + with engine.begin() as conn: + row = conn.execute(text(""" + SELECT + opportunity_id::text, + external_id, + external_name, + external_url, + status, + payload + FROM operation_links + WHERE opportunity_id = CAST(:opportunity_id AS UUID) + AND system = 'odoo' + AND external_type = 'sale_order' + LIMIT 1 + """), {"opportunity_id": opportunity_id}).mappings().first() + return dict(row or {}) + + +def _find_sale_order(client: OdooClient, external_id: str = "", external_name: str = "") -> dict: + fields = ["id", "name", "state", "partner_id", "amount_total", "date_order"] + + if external_id and str(external_id).isdigit(): + rows = client.search_read("sale.order", [["id", "=", int(external_id)]], fields, limit=1) + if rows: + return dict(rows[0]) + + for ref in [external_name, external_id]: + ref = str(ref or "").strip() + if not ref: + continue + rows = client.search_read("sale.order", [["name", "=", ref]], fields, limit=1) + if rows: + return dict(rows[0]) + + return {} + + +def _search_odoo_pickings(client: OdooClient, sale_name: str) -> list: + if not sale_name: + return [] + fields = ["id", "name", "state", "origin", "picking_type_id", "scheduled_date", "date_done"] + try: + rows = client.search_read("stock.picking", [["origin", "ilike", sale_name]], fields, limit=100, order="id desc") + except Exception: + rows = [] + return [dict(r) for r in rows] + + +def _search_odoo_productions(client: OdooClient, sale_name: str) -> list: + if not sale_name: + return [] + fields = ["id", "name", "state", "origin", "product_id", "product_qty", "date_start", "date_finished"] + try: + rows = client.search_read("mrp.production", [["origin", "ilike", sale_name]], fields, limit=100, order="id desc") + except Exception: + rows = [] + return [dict(r) for r in rows] + + +def _is_outgoing_picking(picking: dict) -> bool: + name = str(picking.get("name") or "").upper() + picking_type = _m2o_name(picking.get("picking_type_id")).lower() + + if "/OUT/" in name or name.startswith("WH/OUT"): + return True + + keywords = ["delivery", "outgoing", "entrega", "expedição", "expedicao", "saída", "saida"] + return any(k in picking_type for k in keywords) + + +def _derive_physical_status(sale_order: dict, pickings: list, productions: list) -> dict: + sale_state = str(sale_order.get("state") or "") + outgoing = [p for p in pickings if _is_outgoing_picking(p)] or pickings + + picking_states = {str(p.get("state") or "") for p in outgoing} + production_states = {str(mo.get("state") or "") for mo in productions} + + if sale_state in {"cancel", "cancelled"}: + return { + "physical_status": "cancelled", + "label": "Cancelada", + "reason": "A venda no Odoo está cancelada.", + "ready_to_ship": False, + "next_action": "Rever oportunidade no ClientFlow.", + "stage": None, + } + + if outgoing and all(str(p.get("state") or "") == "done" for p in outgoing): + return { + "physical_status": "shipped", + "label": "Expedida no Odoo", + "reason": "A entrega/picking no Odoo está concluída.", + "ready_to_ship": False, + "next_action": "Confirmar tracking/entrega no ClientFlow.", + "stage": "SHIPMENT_CREATED", + } + + if any(state == "assigned" for state in picking_states): + return { + "physical_status": "ready_to_ship", + "label": "Pronta para despacho", + "reason": "O picking/entrega está reservado e disponível no Odoo.", + "ready_to_ship": True, + "next_action": "Emitir fatura se necessário e criar envio Packlink.", + "stage": "READY_TO_SHIP", + } + + if any(state in {"progress", "to_close", "confirmed"} for state in production_states): + return { + "physical_status": "in_production", + "label": "Em produção/preparação", + "reason": "Existe ordem de produção ativa no Odoo.", + "ready_to_ship": False, + "next_action": "Aguardar conclusão da produção/preparação no Odoo.", + "stage": "IN_PRODUCTION", + } + + if any(state in {"waiting", "confirmed"} for state in picking_states): + return { + "physical_status": "waiting_stock", + "label": "A aguardar stock/preparação", + "reason": "A entrega ainda não está disponível para despacho.", + "ready_to_ship": False, + "next_action": "Aguardar stock, compra ou produção no Odoo.", + "stage": None, + } + + if sale_state in {"draft", "sent"}: + return { + "physical_status": "quote_only", + "label": "Cotação no Odoo", + "reason": "A venda ainda não está confirmada no Odoo.", + "ready_to_ship": False, + "next_action": "Confirmar venda/pagamento antes de preparar.", + "stage": None, + } + + if sale_state in {"sale", "done"}: + return { + "physical_status": "order_created", + "label": "Venda criada", + "reason": "Venda confirmada, mas sem picking pronto identificado.", + "ready_to_ship": False, + "next_action": "Verificar preparação física no Odoo.", + "stage": "ODOO_ORDER_CREATED", + } + + return { + "physical_status": "unknown", + "label": "Estado desconhecido", + "reason": "Não foi possível interpretar o estado físico a partir do Odoo.", + "ready_to_ship": False, + "next_action": "Rever venda diretamente no Odoo.", + "stage": None, + } + + +def sync_opportunity_odoo_status(opportunity_id: str) -> dict: + """Consulta Odoo e guarda um resumo físico simples na operation_links. + + Não altera o Odoo. Apenas lê sale.order, stock.picking e mrp.production, + e guarda em operation_links external_type='physical_status'. + """ + link = _find_odoo_sale_order_link(opportunity_id) + if not link: + payload = { + "physical_status": "no_order", + "label": "Sem venda Odoo", + "reason": "A oportunidade ainda não tem venda Odoo ligada.", + "ready_to_ship": False, + "next_action": "Criar ou associar venda Odoo.", + } + _upsert_odoo_physical_status_link(opportunity_id, "", "Sem venda Odoo", "", "no_order", payload) + return payload + + client = OdooClient() + sale = _find_sale_order(client, str(link.get("external_id") or ""), str(link.get("external_name") or "")) + + if not sale: + payload = { + "physical_status": "not_found", + "label": "Venda não encontrada", + "reason": "A referência guardada no ClientFlow não foi encontrada no Odoo.", + "ready_to_ship": False, + "next_action": "Confirmar o número/id da venda Odoo.", + "linked_sale_order": link, + } + _upsert_odoo_physical_status_link(opportunity_id, str(link.get("external_id") or ""), str(link.get("external_name") or ""), str(link.get("external_url") or ""), "not_found", payload) + return payload + + sale_name = str(sale.get("name") or "") + pickings = _search_odoo_pickings(client, sale_name) + productions = _search_odoo_productions(client, sale_name) + derived = _derive_physical_status(sale, pickings, productions) + + payload = { + **derived, + "sale_order": { + "id": sale.get("id"), + "name": sale.get("name"), + "state": sale.get("state"), + "partner": _compact_m2o(sale.get("partner_id")), + "amount_total": sale.get("amount_total"), + "date_order": sale.get("date_order"), + }, + "pickings": [ + { + "id": p.get("id"), + "name": p.get("name"), + "state": p.get("state"), + "type": _compact_m2o(p.get("picking_type_id")), + "scheduled_date": p.get("scheduled_date"), + "date_done": p.get("date_done"), + } + for p in pickings + ], + "productions": [ + { + "id": mo.get("id"), + "name": mo.get("name"), + "state": mo.get("state"), + "product": _compact_m2o(mo.get("product_id")), + "qty": mo.get("product_qty"), + "date_start": mo.get("date_start"), + "date_finished": mo.get("date_finished"), + } + for mo in productions + ], + } + + _upsert_odoo_physical_status_link( + opportunity_id, + str(sale.get("id") or link.get("external_id") or ""), + sale_name, + str(link.get("external_url") or ""), + derived["physical_status"], + payload, + ) + + stage = derived.get("stage") + if stage in {"IN_PRODUCTION", "READY_TO_SHIP", "SHIPMENT_CREATED"}: + try: + from app.opportunity_service import set_opportunity_stage + set_opportunity_stage(opportunity_id, stage, note=derived.get("reason") or "", created_by="odoo_sync") + except Exception: + pass + + return payload + + +def _upsert_odoo_physical_status_link(opportunity_id: str, external_id: str, external_name: str, external_url: str, status: str, payload: dict) -> None: + with engine.begin() as conn: + conn.execute(text(""" + INSERT INTO operation_links ( + opportunity_id, system, external_type, + external_id, external_name, external_url, + status, payload, last_synced_at, updated_at + ) + VALUES ( + CAST(:opportunity_id AS UUID), 'odoo', 'physical_status', + :external_id, :external_name, :external_url, + :status, CAST(:payload AS JSONB), now(), now() + ) + ON CONFLICT (opportunity_id, system, external_type) + DO UPDATE SET + external_id = EXCLUDED.external_id, + external_name = EXCLUDED.external_name, + external_url = EXCLUDED.external_url, + status = EXCLUDED.status, + payload = EXCLUDED.payload, + last_synced_at = now(), + updated_at = now() + """), { + "opportunity_id": opportunity_id, + "external_id": external_id, + "external_name": external_name, + "external_url": external_url, + "status": status, + "payload": _json(payload), + }) diff --git a/app/operation_noise.py b/app/operation_noise.py new file mode 100644 index 0000000..304320a --- /dev/null +++ b/app/operation_noise.py @@ -0,0 +1,113 @@ +"""Noise detection helpers for the operator work queue. + +These helpers are intentionally conservative. They identify messages that are +not useful ClientFlow work items because they are system delivery reports, +postmaster notifications, automatic bounces/NDRs, or similar mailbox noise. +Chatwoot/Thunderbird remain the source for reading those emails; ClientFlow +should not turn them into tasks/opportunities. +""" +from __future__ import annotations + +import re +from typing import Any, Mapping + +SYSTEM_SENDER_PATTERNS = [ + r"\bpostmaster\b", + r"\bmailer[-_ ]daemon\b", + r"\bmail delivery subsystem\b", + r"\bmail delivery system\b", + r"\bmicrosoft exchange\b", + r"\boffice 365\b", + r"\bmicrosoft outlook\b", +] + +BOUNCE_NDR_PATTERNS = [ + r"\bundeliverable\b", + r"\breturned mail\b", + r"\bmail delivery failed\b", + r"\bdelivery status notification\b", + r"\bnon[- ]delivery report\b", + r"\bdelivery has failed\b", + r"\bfailure notice\b", + r"\bunknown to address\b", + r"\brecipient (?:wasn['’]?t|was not) found\b", + r"\byour message (?:to .* )?couldn['’]?t be delivered\b", + r"\bremote server returned\b", + r"\b550\s+5\.1\.1\b", + r"\b5\.1\.10\b", + r"\bwasn['’]?t found at\b", +] + +TECHNICAL_NOISE_ACTION_CODES = { + "IGNORE_BOUNCE", + "IGNORE_SPAM", + "NO_ACTION", +} + +LOW_VALUE_NO_OPPORTUNITY_ACTION_CODES = { + "REMOVE_FROM_LIST", +} + + +def normalize_noise_text(value: Any) -> str: + """Normalize text for deterministic noise matching.""" + text = str(value or "").casefold() + text = re.sub(r"<[^>]+>", " ", text) + text = re.sub(r"https?://\S+", " ", text) + text = re.sub(r"\s+", " ", text).strip() + return text + + +def contains_any_pattern(text: str, patterns: list[str]) -> bool: + normalized = normalize_noise_text(text) + return any(re.search(pattern, normalized, flags=re.I) for pattern in patterns) + + +def is_system_sender(value: Any) -> bool: + return contains_any_pattern(str(value or ""), SYSTEM_SENDER_PATTERNS) + + +def is_bounce_or_ndr_text(value: Any) -> bool: + return contains_any_pattern(str(value or ""), SYSTEM_SENDER_PATTERNS + BOUNCE_NDR_PATTERNS) + + +def _join_item_text(item: Mapping[str, Any]) -> str: + return "\n".join( + str(item.get(key) or "") + for key in ( + "customer_name", + "contact_display_name", + "fiscal_customer_name", + "message_subject", + "title", + "detail", + "request_text", + "source_system", + "action_code", + "queue", + ) + ) + + +def is_noise_operation_item(item: Mapping[str, Any]) -> bool: + """Return True for items that should not appear in Operations.""" + action_code = str(item.get("action_code") or "").strip().upper() + status = str(item.get("status") or "").strip().lower() + reason = str(item.get("no_opportunity_reason") or "").strip().lower() + + if status in {"ignored", "skipped", "cancelled"}: + return True + if action_code in TECHNICAL_NOISE_ACTION_CODES: + return True + if reason in {"system_or_bounce_message", "bounce_ignored", "non_commercial_system"}: + return True + return is_bounce_or_ndr_text(_join_item_text(item)) + + +def is_low_value_no_opportunity_item(item: Mapping[str, Any]) -> bool: + """Return True for non-commercial work that can be hidden from the main queue.""" + action_code = str(item.get("action_code") or "").strip().upper() + reason = str(item.get("no_opportunity_reason") or "").strip().lower() + if action_code in LOW_VALUE_NO_OPPORTUNITY_ACTION_CODES: + return True + return reason in {"marketing_without_commercial_intent", "non_commercial"} diff --git a/app/operation_service.py b/app/operation_service.py new file mode 100644 index 0000000..4e801ee --- /dev/null +++ b/app/operation_service.py @@ -0,0 +1,149 @@ +import json +import uuid +from typing import Any, Dict, List, Optional + +from sqlalchemy import text + +from app.db import engine +from app.config import settings +from app.opportunity_service import set_opportunity_stage +from app.workflow_guard import validate_operation_action + +_SCHEMA_READY = False + +OPERATION_ACTIONS: Dict[str, Dict[str, str]] = { + "jasmin_quotation": {"label": "Registar orçamento Jasmin", "system": "jasmin", "external_type": "quotation", "status": "created", "stage": "QUOTE_SENT", "note": "Orçamento criado/registado no Jasmin."}, + "jasmin_proforma": {"label": "Registar pró-forma Jasmin", "system": "jasmin", "external_type": "proforma", "status": "issued", "stage": "PROFORMA_SENT", "note": "Pró-forma registada no fluxo Jasmin."}, + "payment_confirmed": {"label": "Registar pagamento confirmado", "system": "clientflow", "external_type": "payment", "status": "confirmed", "stage": "PAYMENT_CONFIRMED", "note": "Pagamento confirmado pelo operador."}, + "odoo_sale_order": {"label": "Registar venda Odoo", "system": "odoo", "external_type": "sale_order", "status": "created", "stage": "ODOO_ORDER_CREATED", "note": "Venda/encomenda registada no Odoo."}, + "odoo_production": {"label": "Registar produção Odoo", "system": "odoo", "external_type": "production", "status": "in_progress", "stage": "IN_PRODUCTION", "note": "Produção/preparação em curso no Odoo."}, + "odoo_physical_validated": {"label": "Validar encomenda física", "system": "odoo", "external_type": "physical_validation", "status": "validated", "stage": "READY_TO_SHIP", "note": "Encomenda física validada no Odoo."}, + "jasmin_invoice": {"label": "Registar fatura Jasmin", "system": "jasmin", "external_type": "invoice", "status": "issued", "stage": "INVOICED", "note": "Fatura emitida/registada no Jasmin."}, + "packlink_shipment": {"label": "Registar envio Packlink", "system": "packlink", "external_type": "shipment", "status": "created", "stage": "SHIPMENT_CREATED", "note": "Envio criado/registado no Packlink."}, + "tracking_sent": {"label": "Registar tracking enviado", "system": "clientflow", "external_type": "tracking", "status": "sent", "stage": "TRACKING_SENT", "note": "Tracking enviado ao cliente via Chatwoot/email."}, + "delivered": {"label": "Registar entrega concluída", "system": "packlink", "external_type": "delivery", "status": "delivered", "stage": "WON", "note": "Processo entregue/concluído."}, +} + +OPERATION_CARDS: List[Dict[str, str]] = [ + {"key": "payment", "label": "Pagamento", "system": "clientflow", "external_type": "payment", "empty": "Por confirmar"}, + {"key": "odoo_sale_order", "label": "Venda Odoo", "system": "odoo", "external_type": "sale_order", "empty": "Não criada"}, + {"key": "odoo_physical_status", "label": "Estado físico Odoo", "system": "odoo", "external_type": "physical_status", "empty": "Não sincronizado"}, + {"key": "odoo_production", "label": "Produção Odoo", "system": "odoo", "external_type": "production", "empty": "Não iniciada"}, + {"key": "physical_validation", "label": "Encomenda física", "system": "odoo", "external_type": "physical_validation", "empty": "A validar"}, + {"key": "jasmin_quotation", "label": "Orçamento Jasmin", "system": "jasmin", "external_type": "quotation", "empty": "Não criado"}, + {"key": "jasmin_invoice", "label": "Fatura Jasmin", "system": "jasmin", "external_type": "invoice", "empty": "Não emitida"}, + {"key": "packlink_shipment", "label": "Envio Packlink", "system": "packlink", "external_type": "shipment", "empty": "Não criado"}, +] + +STATUS_LABELS = { + "not_created": "Não criado", "pending": "Pendente", "issued": "Emitida", "confirmed": "Confirmado", "created": "Criado", "converted": "Convertido", "superseded": "Substituído", "in_progress": "Em curso", "validated": "Validado", "sent": "Enviado", "delivered": "Entregue", "failed": "Falhou", "ready_to_ship": "Pronta para despacho", "waiting_stock": "A aguardar stock", "in_production": "Em produção", "shipped": "Expedida", "order_created": "Venda criada", "quote_only": "Cotação", "not_found": "Não encontrada", "no_order": "Sem venda", "unknown": "Desconhecido", "cancelled": "Cancelada", +} + +def _json(value: Any) -> str: + return json.dumps(value or {}, ensure_ascii=False, default=str) + +def ensure_operation_schema() -> None: + global _SCHEMA_READY + if _SCHEMA_READY: + return + with engine.begin() as conn: + conn.execute(text(""" + CREATE TABLE IF NOT EXISTS operation_links ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + opportunity_id UUID NOT NULL REFERENCES opportunities(id) ON DELETE CASCADE, + system TEXT NOT NULL, + external_type TEXT NOT NULL, + external_id TEXT, + external_name TEXT, + external_url TEXT, + status TEXT NOT NULL DEFAULT 'pending', + payload JSONB NOT NULL DEFAULT '{}'::jsonb, + last_synced_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + """)) + conn.execute(text("""CREATE UNIQUE INDEX IF NOT EXISTS ux_operation_links_key ON operation_links(opportunity_id, system, external_type)""")) + conn.execute(text("CREATE INDEX IF NOT EXISTS idx_operation_links_opportunity ON operation_links(opportunity_id)")) + conn.execute(text("CREATE INDEX IF NOT EXISTS idx_operation_links_system ON operation_links(system, external_type, status)")) + _SCHEMA_READY = True + +def integration_settings_summary() -> Dict[str, Dict[str, Any]]: + return { + "odoo": {"enabled": bool(settings.odoo_enabled), "label": "Odoo", "public_url": settings.odoo_public_url or settings.odoo_base_url, "role": "Vendas, stock, produção e validação física"}, + "jasmin": {"enabled": bool(settings.jasmin_enabled), "label": "Jasmin", "public_url": settings.jasmin_public_url or settings.jasmin_base_url, "role": "Pró-formas, faturas e documentos fiscais"}, + "packlink": {"enabled": bool(settings.packlink_enabled), "label": "Packlink PRO", "public_url": settings.packlink_public_url or settings.packlink_base_url, "role": "Recolhas, envios, etiquetas e tracking"}, + "chatwoot": {"enabled": bool(settings.chatwoot_base_url), "label": "Chatwoot", "public_url": settings.chatwoot_public_url or settings.chatwoot_base_url, "role": "Comunicação com cliente"}, + } + +def get_operation_links(opportunity_id: str) -> List[Dict[str, Any]]: + ensure_operation_schema() + with engine.begin() as conn: + rows = conn.execute(text(""" + SELECT id::text, opportunity_id::text, system, external_type, external_id, external_name, external_url, status, payload, last_synced_at, created_at, updated_at + FROM operation_links + WHERE opportunity_id = CAST(:opportunity_id AS UUID) + ORDER BY system, external_type + """), {"opportunity_id": opportunity_id}).mappings().all() + return [dict(r) for r in rows] + +def get_operation_snapshot(opportunity_id: str) -> Dict[str, Any]: + links = get_operation_links(opportunity_id) + by_key = {(x["system"], x["external_type"]): x for x in links} + cards = [] + for card in OPERATION_CARDS: + link = by_key.get((card["system"], card["external_type"])) + if link: + status = link.get("status") or "pending" + cards.append({**card, **link, "status_label": STATUS_LABELS.get(status, status)}) + else: + cards.append({**card, "status": "not_created", "status_label": card["empty"], "external_name": "", "external_url": ""}) + return {"cards": cards, "links": links, "settings": integration_settings_summary(), "actions": OPERATION_ACTIONS} + +def register_operation_action(opportunity_id: str, action_key: str, *, external_id: str = "", external_name: str = "", external_url: str = "", note: str = "", payload: Optional[Dict[str, Any]] = None, created_by: str = "operator") -> Dict[str, Any]: + ensure_operation_schema() + action = OPERATION_ACTIONS.get(str(action_key or "")) + if not action: + raise ValueError(f"Unsupported operation action: {action_key}") + validate_operation_action(opportunity_id, action_key) + system = action["system"] + external_type = action["external_type"] + status = action["status"] + stage = action["stage"] + note = note or action.get("note") or action.get("label") or action_key + external_name = external_name or action.get("label") or external_type + with engine.begin() as conn: + row = conn.execute(text(""" + INSERT INTO operation_links (opportunity_id, system, external_type, external_id, external_name, external_url, status, payload, last_synced_at, updated_at) + VALUES (CAST(:opportunity_id AS UUID), CAST(:system AS TEXT), CAST(:external_type AS TEXT), NULLIF(CAST(:external_id AS TEXT), ''), NULLIF(CAST(:external_name AS TEXT), ''), NULLIF(CAST(:external_url AS TEXT), ''), CAST(:status AS TEXT), CAST(:payload AS JSONB), now(), now()) + ON CONFLICT (opportunity_id, system, external_type) + DO UPDATE SET external_id = COALESCE(EXCLUDED.external_id, operation_links.external_id), external_name = COALESCE(EXCLUDED.external_name, operation_links.external_name), external_url = COALESCE(EXCLUDED.external_url, operation_links.external_url), status = EXCLUDED.status, payload = operation_links.payload || EXCLUDED.payload, last_synced_at = now(), updated_at = now() + RETURNING id::text, system, external_type, status + """), {"opportunity_id": opportunity_id, "system": system, "external_type": external_type, "external_id": external_id or "", "external_name": external_name or "", "external_url": external_url or "", "status": status, "payload": _json({"action_key": action_key, "note": note, **(payload or {})})}).mappings().first() + conn.execute(text(""" + INSERT INTO opportunity_events (id, opportunity_id, event_type, from_stage, to_stage, note, payload, created_by) + SELECT CAST(:id AS UUID), id, 'operation_action_registered', stage, CAST(:stage AS TEXT), CAST(:note AS TEXT), CAST(:payload AS JSONB), CAST(:created_by AS TEXT) + FROM opportunities WHERE id = CAST(:opportunity_id AS UUID) + """), {"id": str(uuid.uuid4()), "opportunity_id": opportunity_id, "stage": stage, "note": note, "payload": _json({"action_key": action_key, "system": system, "external_type": external_type, "status": status}), "created_by": created_by}) + set_opportunity_stage(opportunity_id, stage, note=note, created_by=created_by) + return dict(row or {}) + +def operation_next_steps(snapshot: Dict[str, Any], stage: str) -> List[str]: + stage = str(stage or "") + status_by = {c["key"]: c.get("status") for c in snapshot.get("cards", [])} + steps: List[str] = [] + if stage in {"QUOTE_SENT", "PROFORMA_SENT", "WAITING_PAYMENT"} and status_by.get("payment") != "confirmed": + steps.append("Confirmar pagamento antes de criar venda operacional no Odoo.") + if stage in {"PAYMENT_CONFIRMED"} and status_by.get("odoo_sale_order") == "not_created": + steps.append("Criar venda/encomenda no Odoo com os dados validados no ClientFlow.") + if stage in {"ODOO_ORDER_CREATED", "IN_PRODUCTION"} and status_by.get("physical_validation") != "validated": + steps.append("Aguardar produção/preparação e validar a encomenda física no Odoo.") + if stage in {"READY_TO_SHIP", "INVOICED"} and status_by.get("jasmin_invoice") != "issued": + steps.append("Emitir fatura no Jasmin antes do envio, salvo exceção operacional.") + if stage in {"READY_TO_SHIP", "INVOICED"} and status_by.get("packlink_shipment") != "created": + steps.append("Criar envio/recolha no Packlink quando a encomenda estiver pronta.") + if stage == "SHIPMENT_CREATED" and status_by.get("tracking") != "sent": + steps.append("Enviar tracking ao cliente pelo Chatwoot/email.") + if not steps: + steps.append("Fluxo sem bloqueios aparentes. Rever tarefas pendentes e próximos contactos.") + return steps diff --git a/app/operations_service.py b/app/operations_service.py new file mode 100644 index 0000000..cd44f26 --- /dev/null +++ b/app/operations_service.py @@ -0,0 +1,481 @@ +"""Operational summaries for ClientFlow. + +This module is deliberately query-oriented: it builds the daily operation view, +health checks and unified opportunity timeline without changing business state. +""" +from __future__ import annotations + +import os +import subprocess +from typing import Any, Dict, List, Optional + +from sqlalchemy import text + +from app.config import settings +from app.db import engine +from app.operation_noise import is_low_value_no_opportunity_item, is_noise_operation_item + + +def _int(value: Any) -> int: + try: + return int(value or 0) + except Exception: + return 0 + + +def _is_manually_resolved_error(value: Any) -> bool: + text_value = str(value or "").strip().casefold() + return "limpo manualmente" in text_value or "resolvido manualmente" in text_value + + +def _humanize_operation_detail(value: Any) -> str: + detail = str(value or "").strip() + lower = detail.casefold() + if "resposta llm inválida" in lower or "invalid" in lower and "action_code" in lower: + return "Classificação da mensagem falhou. Rever no Chatwoot e escolher a ação correta." + if _is_manually_resolved_error(detail): + return "Item já limpo manualmente. Deve ficar no histórico/outbox, não na fila diária." + return detail + + +def _systemd_state(unit: str) -> Dict[str, Any]: + """Return a best-effort systemd unit status. + + Works on the production server and degrades gracefully elsewhere. + """ + try: + result = subprocess.run( + ["systemctl", "is-active", unit], + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=3, + check=False, + ) + return { + "unit": unit, + "active_state": result.stdout.strip() or result.stderr.strip() or "unknown", + "ok": result.returncode == 0, + } + except Exception as exc: + return {"unit": unit, "active_state": "unknown", "ok": False, "error": str(exc)} + + +def build_chatwoot_conversation_url(conversation_id: Optional[str]) -> str: + """Build a public Chatwoot URL when configuration is available.""" + conversation_id = str(conversation_id or "").strip() + if not conversation_id: + return "" + base_url = ( + getattr(settings, "chatwoot_public_url", "") + or getattr(settings, "chatwoot_base_url", "") + or "" + ).rstrip("/") + account_id = str(getattr(settings, "chatwoot_account_id", "") or "").strip() + if not base_url or not account_id: + return "" + return f"{base_url}/app/accounts/{account_id}/conversations/{conversation_id}" + + +def _attach_operation_urls(items: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + cleaned: List[Dict[str, Any]] = [] + for item in items: + original_detail = item.get("detail") + if str(item.get("source") or "") == "outbox" and _is_manually_resolved_error(original_detail): + continue + item["detail"] = _humanize_operation_detail(original_detail) + # v4.9.0: Operations is not a mailbox. Delivery bounces, postmaster + # notifications, and technical no-action items must stay in + # Chatwoot/Thunderbird/logs, not in the daily work queue. + if is_noise_operation_item(item) or is_low_value_no_opportunity_item(item): + continue + if str(item.get("source_system") or "") == "chatwoot" or item.get("conversation_id"): + item["chatwoot_url"] = build_chatwoot_conversation_url(item.get("conversation_id")) + else: + item["chatwoot_url"] = "" + cleaned.append(item) + return cleaned + + +def get_operations_summary(limit: int = 24) -> Dict[str, Any]: + """Build the /operations work queue summary. + + /operations is intentionally not a mini-dashboard. It returns a compact + set of counters and a single prioritized queue of human work items. + Technical lists remain available in their own pages and should only appear + here when they block an operator action. + """ + with engine.begin() as conn: + counts = conn.execute(text(""" + SELECT + (SELECT COUNT(*) FROM opportunities WHERE status = 'open')::int AS open_opportunities, + (SELECT COUNT(*) FROM tasks WHERE status = 'pending')::int AS pending_tasks, + (SELECT COUNT(*) FROM integration_outbox WHERE status = 'pending')::int AS outbox_pending, + (SELECT COUNT(*) FROM integration_outbox WHERE status = 'failed' AND COALESCE(last_error,'') NOT ILIKE '%limpo manualmente%' AND COALESCE(last_error,'') NOT ILIKE '%resolvido manualmente%')::int AS outbox_failed, + (SELECT COUNT(*) FROM commercial_documents WHERE document_kind = 'quotation' AND status NOT IN ('failed','cancelled','converted'))::int AS open_quotations, + (SELECT COUNT(*) FROM commercial_documents WHERE document_kind = 'invoice' AND status IN ('issued','created'))::int AS active_invoices, + (SELECT COUNT(*) FROM commercial_documents WHERE document_kind = 'invoice' AND created_at::date = CURRENT_DATE)::int AS invoices_today, + (SELECT COUNT(*) FROM shipments WHERE status NOT IN ('cancelled','failed','delivered','shipped'))::int AS shipments_pending, + (SELECT COUNT(*) FROM customers WHERE COALESCE(tax_id,'') = '' OR COALESCE(street_name,'') = '' OR COALESCE(postal_zone,'') = '' OR COALESCE(city_name,'') = '')::int AS customers_incomplete, + (SELECT COUNT(*) FROM products WHERE active = TRUE AND COALESCE(jasmin_sales_item,'') = '')::int AS products_missing_jasmin, + (SELECT COUNT(*) FROM communications WHERE status IN ('new','classified','needs_review'))::int AS communications_open, + (SELECT COUNT(*) FROM communications WHERE status = 'needs_review')::int AS communications_needs_review, + (SELECT COUNT(*) FROM communications WHERE customer_id IS NULL AND direction = 'inbound')::int AS communications_without_customer, + (SELECT COUNT(*) FROM tasks WHERE status = 'pending' AND due_at IS NOT NULL AND due_at < now())::int AS overdue_tasks, + (SELECT COUNT(*) FROM tasks WHERE status = 'pending' AND (route = 'rever' OR action_code = 'REVIEW_MANUALLY'))::int AS review_tasks, + (SELECT COUNT(*) FROM integration_outbox WHERE status IN ('failed','blocked') AND COALESCE(last_error,'') NOT ILIKE '%limpo manualmente%' AND COALESCE(last_error,'') NOT ILIKE '%resolvido manualmente%')::int AS blocked_outbox, + ( + (SELECT COUNT(*) FROM tasks WHERE status = 'pending') + + (SELECT COUNT(*) FROM integration_outbox WHERE status IN ('failed','blocked') AND COALESCE(last_error,'') NOT ILIKE '%limpo manualmente%' AND COALESCE(last_error,'') NOT ILIKE '%resolvido manualmente%') + + (SELECT COUNT(*) FROM communications WHERE status = 'needs_review') + )::int AS work_queue_total + """)).mappings().first() or {} + + recent_outbox = conn.execute(text(""" + SELECT id::text, target_system, action_type, status, last_error, created_at, updated_at + FROM integration_outbox + WHERE status IN ('pending','failed','blocked') + AND NOT (status = 'failed' AND (COALESCE(last_error,'') ILIKE '%limpo manualmente%' OR COALESCE(last_error,'') ILIKE '%resolvido manualmente%')) + ORDER BY created_at DESC + LIMIT :limit + """), {"limit": int(limit)}).mappings().all() + + recent_documents = conn.execute(text(""" + SELECT cd.id::text, cd.opportunity_id::text, cd.document_kind, cd.status, + cd.document_number, cd.external_id, cd.total_amount, cd.currency, + c.name AS customer_name, cd.created_at + FROM commercial_documents cd + LEFT JOIN customers c ON c.id = cd.customer_id + WHERE (cd.document_kind = 'quotation' AND cd.status NOT IN ('failed','cancelled','converted')) + OR (cd.document_kind = 'invoice' AND cd.status IN ('issued','created')) + OR (cd.status IN ('failed','blocked')) + ORDER BY cd.created_at DESC + LIMIT :limit + """), {"limit": int(limit)}).mappings().all() + + problem_products = conn.execute(text(""" + SELECT sku, name, default_unit_price + FROM products + WHERE active = TRUE AND COALESCE(jasmin_sales_item,'') = '' + ORDER BY name + LIMIT :limit + """), {"limit": int(limit)}).mappings().all() + + incomplete_customers = conn.execute(text(""" + SELECT id::text, name, tax_id, street_name, postal_zone, city_name, updated_at + FROM customers + WHERE COALESCE(tax_id,'') = '' + OR COALESCE(street_name,'') = '' + OR COALESCE(postal_zone,'') = '' + OR COALESCE(city_name,'') = '' + ORDER BY updated_at DESC + LIMIT :limit + """), {"limit": int(limit)}).mappings().all() + + recent_communications = conn.execute(text(""" + SELECT c.id::text, c.sender_name, c.sender_email, c.subject, c.classification, + c.confidence, c.status, c.created_at, c.customer_id::text, + c.opportunity_id::text, cu.name AS customer_name, o.title AS opportunity_title + FROM communications c + LEFT JOIN customers cu ON cu.id = c.customer_id + LEFT JOIN opportunities o ON o.id = c.opportunity_id + WHERE c.status IN ('new','classified','needs_review') + ORDER BY c.created_at DESC + LIMIT :limit + """), {"limit": int(limit)}).mappings().all() + + work_items = conn.execute(text(""" + SELECT * FROM ( + SELECT 'task' AS source, t.id::text AS id, t.created_at, + CASE + WHEN COALESCE(t.metadata->>'opportunity_linking_status','') = 'ambiguous' THEN 'alta' + ELSE COALESCE(t.priority, CASE WHEN t.due_at < now() THEN 'alta' ELSE 'normal' END) + END AS priority, + CASE + WHEN COALESCE(t.metadata->>'opportunity_linking_status','') = 'ambiguous' THEN 'rever' + ELSE COALESCE(t.route, 'rever') + END AS queue, + CASE + WHEN COALESCE(t.metadata->>'opportunity_linking_status','') = 'ambiguous' THEN 'ASSOCIATE_OPPORTUNITY' + ELSE t.action_code + END AS action_code, + CASE + WHEN COALESCE(t.metadata->>'opportunity_linking_status','') = 'ambiguous' THEN 'Confirmar associação da oportunidade' + ELSE COALESCE(t.action, t.action_code, 'Tarefa') + END AS title, + CASE + WHEN COALESCE(t.metadata->>'opportunity_linking_status','') = 'ambiguous' THEN 'O sistema encontrou mais de uma oportunidade aberta para este contacto. Confirme a oportunidade e o cliente fiscal antes de emitir documentos.' + ELSE COALESCE(t.note, '') + END AS detail, + t.status, + t.source_system, + t.conversation_id, + t.contact_id, + COALESCE(m.clean_body, m.raw_body, re.payload->>'content', '') AS request_text, + t.opportunity_id::text, + COALESCE(o.title, '') AS opportunity_title, + COALESCE(NULLIF(re.payload->'sender'->>'name',''), NULLIF(re.payload->'sender'->>'email',''), NULLIF(t.contact_id,''), '') AS customer_name, + COALESCE(cu_opp.name, cu_task.name, '') AS fiscal_customer_name, + COALESCE(cu_opp.email, cu_task.email, '') AS fiscal_customer_email, + COALESCE(cu_opp.tax_id, cu_task.tax_id, '') AS fiscal_customer_tax_id, + COALESCE(cu_opp.street_name, cu_task.street_name, '') AS fiscal_customer_street_name, + COALESCE(cu_opp.postal_zone, cu_task.postal_zone, '') AS fiscal_customer_postal_zone, + COALESCE(cu_opp.city_name, cu_task.city_name, '') AS fiscal_customer_city_name, + COALESCE(NULLIF(re.payload->'sender'->>'name',''), NULLIF(re.payload->'sender'->>'email',''), NULLIF(t.contact_id,''), '') AS contact_display_name, + COALESCE(t.metadata->>'no_opportunity_reason', '') AS no_opportunity_reason, + '/tasks/' || t.id::text AS href, + 'Abrir' AS action_label + FROM tasks t + LEFT JOIN opportunities o ON o.id = t.opportunity_id + LEFT JOIN customers cu_opp ON cu_opp.id = o.local_customer_id + LEFT JOIN customers cu_task ON cu_task.id::text = t.customer_id + LEFT JOIN messages m ON m.id = t.message_id + LEFT JOIN raw_events re ON re.id = t.raw_event_id + WHERE t.status = 'pending' + + UNION ALL + + SELECT 'outbox' AS source, io.id::text AS id, io.created_at, + CASE WHEN io.status = 'failed' THEN 'alta' ELSE 'normal' END AS priority, + 'sistema' AS queue, + upper(io.target_system || '_' || io.action_type) AS action_code, + io.target_system || '.' || io.action_type AS title, + COALESCE(io.last_error, io.idempotency_key, '') AS detail, + io.status, + io.target_system AS source_system, + NULL::text AS conversation_id, + NULL::text AS contact_id, + ''::text AS request_text, + NULLIF(io.payload->>'opportunity_id','') AS opportunity_id, + COALESCE(o.title, '') AS opportunity_title, + COALESCE(cu.name, '') AS customer_name, + COALESCE(cu.name, '') AS fiscal_customer_name, + COALESCE(cu.email, '') AS fiscal_customer_email, + COALESCE(cu.tax_id, '') AS fiscal_customer_tax_id, + COALESCE(cu.street_name, '') AS fiscal_customer_street_name, + COALESCE(cu.postal_zone, '') AS fiscal_customer_postal_zone, + COALESCE(cu.city_name, '') AS fiscal_customer_city_name, + ''::text AS contact_display_name, + ''::text AS no_opportunity_reason, + '/outbox/' || io.id::text AS href, + CASE WHEN io.status = 'failed' THEN 'Reprocessar' ELSE 'Abrir' END AS action_label + FROM integration_outbox io + LEFT JOIN opportunities o ON o.id::text = NULLIF(io.payload->>'opportunity_id','') + LEFT JOIN customers cu ON cu.id = o.local_customer_id + WHERE io.status IN ('pending','failed','blocked') + AND NOT (io.status = 'failed' AND (COALESCE(io.last_error,'') ILIKE '%limpo manualmente%' OR COALESCE(io.last_error,'') ILIKE '%resolvido manualmente%')) + + UNION ALL + + SELECT 'communication' AS source, c.id::text AS id, c.created_at, + CASE WHEN c.status = 'needs_review' THEN 'normal' ELSE 'normal' END AS priority, + CASE + WHEN c.classification IN ('comprovativo_pagamento','pedido_fatura','aceitacao_orcamento','dados_fiscais') THEN 'financeiro' + WHEN c.classification IN ('pedido_tracking') THEN 'operacoes' + WHEN c.classification IN ('reclamacao') THEN 'suporte' + WHEN c.classification IN ('pedido_remocao_lista') THEN 'marketing' + ELSE 'vendas' + END AS queue, + upper(COALESCE(c.classification, 'REVIEW_MANUALLY')) AS action_code, + COALESCE(c.classification, 'Mensagem por classificar') AS title, + COALESCE(c.subject, c.sender_email, '') AS detail, + c.status, + c.source_system, + c.conversation_id, + c.contact_id, + COALESCE(c.body, '') AS request_text, + c.opportunity_id::text, + COALESCE(o.title, '') AS opportunity_title, + COALESCE(cu.name, c.sender_name, c.sender_email, '') AS customer_name, + COALESCE(cu.name, '') AS fiscal_customer_name, + COALESCE(cu.email, '') AS fiscal_customer_email, + COALESCE(cu.tax_id, '') AS fiscal_customer_tax_id, + COALESCE(cu.street_name, '') AS fiscal_customer_street_name, + COALESCE(cu.postal_zone, '') AS fiscal_customer_postal_zone, + COALESCE(cu.city_name, '') AS fiscal_customer_city_name, + COALESCE(c.sender_name, c.sender_email, c.contact_id, '') AS contact_display_name, + ''::text AS no_opportunity_reason, + '/communications/' || c.id::text AS href, + CASE WHEN c.customer_id IS NULL THEN 'Associar cliente' ELSE 'Abrir' END AS action_label + FROM communications c + LEFT JOIN customers cu ON cu.id = c.customer_id + LEFT JOIN opportunities o ON o.id = c.opportunity_id + WHERE c.status IN ('new','classified','needs_review') + ) items + ORDER BY + CASE lower(priority) WHEN 'alta' THEN 1 WHEN 'high' THEN 1 WHEN 'urgente' THEN 0 WHEN 'normal' THEN 2 ELSE 3 END, + created_at DESC + LIMIT :limit + """), {"limit": int(limit)}).mappings().all() + + cleaned_work_items = _attach_operation_urls([dict(r) for r in work_items]) + cleaned_counts = {k: _int(v) for k, v in dict(counts).items()} + # v4.9.0: the visible Operations total should match the queue the + # operator can actually act on, not raw pending tasks that include mailbox + # noise awaiting cleanup. The cleanup script still fixes the data source. + cleaned_counts["work_queue_total"] = len(cleaned_work_items) + + return { + "counts": cleaned_counts, + "recent_outbox": [dict(r) for r in recent_outbox], + "recent_documents": [dict(r) for r in recent_documents], + "problem_products": [dict(r) for r in problem_products], + "incomplete_customers": [dict(r) for r in incomplete_customers], + "recent_communications": [dict(r) for r in recent_communications], + "work_items": cleaned_work_items, + } + + +def get_system_health_summary() -> Dict[str, Any]: + """Return human and API friendly health details.""" + db_ok = True + db_error: Optional[str] = None + try: + with engine.begin() as conn: + conn.execute(text("SELECT 1")) + except Exception as exc: + db_ok = False + db_error = str(exc) + + outbox_counts: Dict[str, Dict[str, int]] = {} + document_counts: Dict[str, Dict[str, int]] = {} + operational_metrics: Dict[str, int] = {} + if db_ok: + with engine.begin() as conn: + rows = conn.execute(text(""" + SELECT target_system, status, COUNT(*)::int AS total + FROM integration_outbox + GROUP BY target_system, status + ORDER BY target_system, status + """)).mappings().all() + for row in rows: + outbox_counts.setdefault(row["target_system"] or "unknown", {})[row["status"] or "unknown"] = row["total"] + + rows = conn.execute(text(""" + SELECT document_kind, status, COUNT(*)::int AS total + FROM commercial_documents + GROUP BY document_kind, status + ORDER BY document_kind, status + """)).mappings().all() + for row in rows: + document_counts.setdefault(row["document_kind"] or "unknown", {})[row["status"] or "unknown"] = row["total"] + + row = conn.execute(text(""" + SELECT + (SELECT COUNT(*) FROM tasks WHERE status = 'pending')::int AS tasks_pending, + (SELECT COUNT(*) FROM tasks WHERE status = 'done' AND done_at >= now() - interval '24 hours')::int AS tasks_done_24h, + (SELECT COUNT(*) FROM task_events WHERE event_type = 'task_auto_completed' AND created_at >= now() - interval '24 hours')::int AS tasks_auto_completed_24h, + (SELECT COUNT(*) FROM tasks WHERE status = 'pending' AND COALESCE(metadata->>'opportunity_linking_status','') = 'ambiguous')::int AS ambiguous_opportunity_tasks, + (SELECT COUNT(*) FROM integration_outbox WHERE status = 'processing')::int AS outbox_processing, + (SELECT COUNT(*) FROM integration_outbox WHERE status = 'processing' AND locked_at < now() - interval '30 minutes')::int AS outbox_processing_stale, + (SELECT COUNT(*) FROM integration_outbox WHERE status = 'stale')::int AS outbox_stale, + (SELECT COUNT(*) FROM integration_outbox WHERE status IN ('failed','blocked','stale'))::int AS outbox_blocked_or_failed, + (SELECT COUNT(*) FROM business_events WHERE event_type = 'operator_action' AND created_at >= now() - interval '24 hours')::int AS operator_actions_24h, + (SELECT COUNT(*) FROM opportunities WHERE status = 'open' AND local_customer_id IS NULL)::int AS open_opportunities_without_fiscal_customer, + (SELECT COUNT(*) FROM opportunities o JOIN customers c ON c.id = o.local_customer_id WHERE o.status = 'open' AND (COALESCE(c.tax_id,'') = '' OR COALESCE(c.email,'') = '' OR COALESCE(c.street_name,'') = '' OR COALESCE(c.postal_zone,'') = '' OR COALESCE(c.city_name,'') = ''))::int AS active_incomplete_fiscal_customers, + (SELECT COUNT(*) FROM products WHERE active = TRUE AND COALESCE(jasmin_sales_item,'') = '')::int AS products_missing_external_code, + (SELECT COUNT(*) FROM raw_events WHERE source_system = 'chatwoot')::int AS chatwoot_events_total, + (SELECT EXTRACT(EPOCH FROM (now() - max(created_at)))::int FROM raw_events WHERE source_system = 'chatwoot')::int AS seconds_since_last_chatwoot_webhook, + (SELECT COUNT(*) FROM tasks WHERE status = 'pending' AND route = 'vendas')::int AS tasks_pending_vendas, + (SELECT COUNT(*) FROM tasks WHERE status = 'pending' AND route = 'financeiro')::int AS tasks_pending_financeiro, + (SELECT COUNT(*) FROM tasks WHERE status = 'pending' AND route IN ('operacoes','logistica'))::int AS tasks_pending_operacoes, + (SELECT COUNT(*) FROM tasks WHERE status = 'pending' AND route IN ('rever','revisao'))::int AS tasks_pending_rever + """)).mappings().first() or {} + operational_metrics = {key: _int(value) for key, value in dict(row).items()} + + configured_auto_codes = os.getenv("CHATWOOT_AUTO_COMPLETE_ACTION_CODES", "").strip() + + return { + "status": "ok" if db_ok else "degraded", + "database": {"ok": db_ok, "error": db_error}, + "settings": { + "app_name": settings.app_name, + "env": settings.env, + "jasmin_enabled": bool(settings.jasmin_enabled), + "packlink_enabled": bool(settings.packlink_enabled), + "jasmin_company_key": settings.jasmin_company_key, + "jasmin_quotation_serie": settings.jasmin_quotation_serie, + "packlink_default_service_id": settings.packlink_default_service_id, + "chatwoot_auto_complete_on_outgoing": os.getenv("CHATWOOT_AUTO_COMPLETE_ON_OUTGOING", "true"), + "chatwoot_auto_complete_action_codes": configured_auto_codes or "default_safe_codes", + "outbox_stale_processing_minutes": os.getenv("OUTBOX_STALE_PROCESSING_MINUTES", "30"), + "outbox_stale_recovery_mode": os.getenv("OUTBOX_STALE_RECOVERY_MODE", "manual_only"), + }, + "timers": { + "jasmin": _systemd_state("clientflow-outbox-jasmin.timer"), + "packlink": _systemd_state("clientflow-outbox-packlink.timer"), + }, + "outbox": outbox_counts, + "documents": document_counts, + "operational_metrics": operational_metrics, + } + + +def list_unified_opportunity_timeline(opportunity_id: str, limit: int = 60) -> List[Dict[str, Any]]: + """Return a combined opportunity timeline from events, documents, outbox, shipments and items.""" + params = {"opportunity_id": opportunity_id, "limit": int(limit)} + with engine.begin() as conn: + rows = conn.execute(text(""" + SELECT * FROM ( + SELECT created_at, COALESCE(source, 'timeline') AS source, title, + COALESCE(description, '') AS detail, payload, NULL::text AS status, related_id AS external_id + FROM timeline_events + WHERE opportunity_id = CAST(:opportunity_id AS UUID) + + UNION ALL + + SELECT created_at, 'event' AS source, event_type AS title, + COALESCE(note, '') AS detail, payload, NULL::text AS status, NULL::text AS external_id + FROM opportunity_events + WHERE opportunity_id = CAST(:opportunity_id AS UUID) + + UNION ALL + + SELECT created_at, 'document' AS source, + CASE WHEN document_kind = 'quotation' THEN 'Orçamento Jasmin' ELSE 'Fatura Jasmin' END AS title, + COALESCE(document_number, external_id, '') AS detail, + payload, status, external_id + FROM commercial_documents + WHERE opportunity_id = CAST(:opportunity_id AS UUID) + + UNION ALL + + SELECT created_at, 'outbox' AS source, + target_system || '.' || action_type AS title, + COALESCE(last_error, idempotency_key, '') AS detail, + payload, status, id::text AS external_id + FROM integration_outbox + WHERE payload->>'opportunity_id' = :opportunity_id + + UNION ALL + + SELECT created_at, 'shipment' AS source, + 'Envio Packlink' AS title, + COALESCE(carrier || ' · ' || service_name, external_reference, '') AS detail, + payload, status, external_reference AS external_id + FROM shipments + WHERE opportunity_id = CAST(:opportunity_id AS UUID) + + UNION ALL + + SELECT created_at, 'communication' AS source, + COALESCE(classification, 'Comunicação recebida') AS title, + COALESCE(subject, sender_email, '') AS detail, + metadata AS payload, status, id::text AS external_id + FROM communications + WHERE opportunity_id = CAST(:opportunity_id AS UUID) + + UNION ALL + + SELECT created_at, 'product' AS source, + 'Produto adicionado' AS title, + COALESCE(product_name, sku, '') AS detail, + jsonb_build_object('sku', sku, 'jasmin_sales_item', jasmin_sales_item, 'quantity', quantity, 'unit_price', unit_price) AS payload, + NULL::text AS status, id::text AS external_id + FROM opportunity_items + WHERE opportunity_id = CAST(:opportunity_id AS UUID) + ) x + ORDER BY created_at DESC + LIMIT :limit + """), params).mappings().all() + return [dict(r) for r in rows] diff --git a/app/operator_audit_service.py b/app/operator_audit_service.py new file mode 100644 index 0000000..85fdab3 --- /dev/null +++ b/app/operator_audit_service.py @@ -0,0 +1,108 @@ +"""Lightweight operator audit helpers. + +v4.8 records sensitive/manual operator actions without adding a new table. +The durable audit stream is stored in business_events with event_type +``operator_action`` and, when a task_id is present, mirrored to task_events. +""" +from __future__ import annotations + +import json +from typing import Any, Dict, Optional + +from sqlalchemy import text + +from app.db import engine + + +def _json(value: Any) -> str: + return json.dumps(value or {}, ensure_ascii=False) + + +def record_operator_action( + *, + action: str, + entity_type: str, + entity_id: Optional[str] = None, + actor: str = "operator", + task_id: Optional[str] = None, + opportunity_id: Optional[str] = None, + conversation_id: Optional[str] = None, + contact_id: Optional[str] = None, + before: Optional[Dict[str, Any]] = None, + after: Optional[Dict[str, Any]] = None, + payload: Optional[Dict[str, Any]] = None, +) -> None: + """Record a manual/sensitive operator action. + + Audit SQL invariant: event_type = 'operator_action'. + + This intentionally avoids a schema migration. ``business_events.task_id`` is + nullable, so it can hold system/outbox/admin events as a generic audit log. + """ + action = str(action or "operator_action").strip() or "operator_action" + entity_type = str(entity_type or "unknown").strip() or "unknown" + audit_payload = { + "action": action, + "entity_type": entity_type, + "entity_id": str(entity_id or task_id or opportunity_id or ""), + "actor": str(actor or "operator"), + "opportunity_id": opportunity_id, + "before": before or {}, + "after": after or {}, + **(payload or {}), + } + + task_uuid = task_id if task_id else None + with engine.begin() as conn: + conn.execute(text(""" + INSERT INTO business_events ( + event_type, + task_id, + conversation_id, + contact_id, + payload, + created_by + ) + VALUES ( + 'operator_action', + CAST(:task_id AS UUID), + :conversation_id, + :contact_id, + CAST(:payload AS JSONB), + :created_by + ) + """), { + "task_id": task_uuid, + "conversation_id": conversation_id, + "contact_id": contact_id, + "payload": _json(audit_payload), + "created_by": str(actor or "operator"), + }) + + if task_id: + conn.execute(text(""" + INSERT INTO task_events ( + task_id, + event_type, + payload, + created_by + ) + VALUES ( + CAST(:task_id AS UUID), + 'operator_action', + CAST(:payload AS JSONB), + :created_by + ) + """), { + "task_id": task_id, + "payload": _json(audit_payload), + "created_by": str(actor or "operator"), + }) + + +def record_operator_action_best_effort(**kwargs: Any) -> None: + """Audit helper that must never break the operational action itself.""" + try: + record_operator_action(**kwargs) + except Exception as exc: # pragma: no cover - defensive logging only + print(f"ClientFlow operator audit failed: {exc}", flush=True) diff --git a/app/opportunities/__init__.py b/app/opportunities/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/opportunity_next_action_service.py b/app/opportunity_next_action_service.py new file mode 100644 index 0000000..cbec3e9 --- /dev/null +++ b/app/opportunity_next_action_service.py @@ -0,0 +1,194 @@ +"""Central decision service for opportunity next actions. + +v4.9.27 keeps this deliberately small and read-only: it does not replace +existing task workflow yet, but gives UI/reconciliation a single vocabulary for +what the operator should do next. +""" +from __future__ import annotations + +from dataclasses import dataclass, asdict +from typing import Any, Dict, Optional + +from sqlalchemy import text + +from app.db import engine + + +@dataclass +class OpportunityNextAction: + action_code: str + label: str + description: str + priority: str = "normal" + target_url: Optional[str] = None + can_execute: bool = True + reason_if_blocked: Optional[str] = None + document_id: Optional[str] = None + document_number: Optional[str] = None + + def to_dict(self) -> Dict[str, Any]: + return asdict(self) + + +def _first_row(conn: Any, sql: str, params: Dict[str, Any]) -> Optional[Dict[str, Any]]: + row = conn.execute(text(sql), params).mappings().first() + return dict(row) if row else None + + +def get_opportunity_next_action(opportunity_id: str) -> Dict[str, Any]: + """Return the recommended operator action for one opportunity. + + Priority order is intentionally conservative: + 1. unresolved pending task; + 2. missing fiscal customer; + 3. pending reconciliation evidence; + 4. Jasmin commercial document state; + 5. generic follow-up. + """ + params = {"opportunity_id": opportunity_id} + with engine.begin() as conn: + opp = _first_row(conn, """ + SELECT id::text, stage, status, title, fiscal_customer_id::text, customer_id::text, metadata + FROM opportunities + WHERE id = CAST(:opportunity_id AS UUID) + """, params) + if not opp: + return OpportunityNextAction( + action_code="NOT_FOUND", + label="Oportunidade não encontrada", + description="Não foi possível encontrar esta oportunidade.", + priority="baixa", + can_execute=False, + reason_if_blocked="opportunity_not_found", + ).to_dict() + + task = _first_row(conn, """ + SELECT id::text, action_code, action, note, priority, route, due_at, created_at + FROM tasks + WHERE opportunity_id = CAST(:opportunity_id AS UUID) + AND status = 'pending' + ORDER BY + CASE COALESCE(priority, 'normal') WHEN 'alta' THEN 1 WHEN 'normal' THEN 2 WHEN 'baixa' THEN 3 ELSE 4 END, + due_at NULLS LAST, + created_at DESC + LIMIT 1 + """, params) + if task: + return OpportunityNextAction( + action_code=str(task.get("action_code") or "OPEN_TASK"), + label=str(task.get("action") or task.get("action_code") or "Abrir tarefa pendente"), + description=str(task.get("note") or "Abrir a tarefa pendente para continuar o processo."), + priority=str(task.get("priority") or "normal"), + target_url=f"/tasks/{task.get('id')}", + ).to_dict() + + if not opp.get("fiscal_customer_id"): + return OpportunityNextAction( + action_code="VALIDATE_FISCAL_CUSTOMER", + label="Validar cliente fiscal", + description="Antes de emitir documentos oficiais, confirma ou associa o cliente fiscal correto.", + priority="alta", + target_url=f"/opportunities/{opportunity_id}/fiscal-enrich", + ).to_dict() + + candidate = _first_row(conn, """ + SELECT id::text, source_system, external_type, document_number, title, confidence + FROM reconciliation_items + WHERE opportunity_id = CAST(:opportunity_id AS UUID) + AND status = 'open' + ORDER BY confidence DESC NULLS LAST, created_at DESC + LIMIT 1 + """, params) + if candidate: + doc_ref = candidate.get("document_number") or candidate.get("title") or "documento encontrado" + return OpportunityNextAction( + action_code="RECONCILE_DOCUMENTS", + label="Confirmar documento encontrado", + description=f"Existe evidência por decidir: {doc_ref}.", + priority="alta", + target_url="/reconciliation", + document_number=str(candidate.get("document_number") or "") or None, + ).to_dict() + + current_doc = _first_row(conn, """ + SELECT id::text, document_kind, document_number, status, total_amount, document_date, role + FROM commercial_documents + WHERE opportunity_id = CAST(:opportunity_id AS UUID) + AND system = 'jasmin' + AND COALESCE(is_active, TRUE) = TRUE + AND COALESCE(role, 'current') IN ('current', 'accepted') + ORDER BY + CASE document_kind WHEN 'invoice' THEN 1 WHEN 'proforma' THEN 2 WHEN 'quotation' THEN 3 ELSE 4 END, + COALESCE(document_date, created_at::date) DESC, + created_at DESC + LIMIT 1 + """, params) + if not current_doc: + return OpportunityNextAction( + action_code="CREATE_JASMIN_QUOTE", + label="Criar orçamento no Jasmin", + description="A oportunidade tem cliente fiscal, mas ainda não tem orçamento Jasmin ligado.", + priority="normal", + target_url=f"/opportunities/{opportunity_id}#documentos", + ).to_dict() + + kind = str(current_doc.get("document_kind") or "") + number = current_doc.get("document_number") or "documento" + if kind == "quotation": + return OpportunityNextAction( + action_code="SEND_PROFORMA", + label="Avançar a partir do orçamento", + description=f"Orçamento atual {number}. Se o cliente aceitou, emitir ou enviar pró-forma.", + priority="normal", + target_url=f"/opportunities/{opportunity_id}#documentos", + document_id=current_doc.get("id"), + document_number=number, + ).to_dict() + payment_confirmed = bool(_first_row(conn, """ + SELECT id::text + FROM operation_links + WHERE opportunity_id = CAST(:opportunity_id AS UUID) + AND system = 'clientflow' + AND external_type = 'payment' + AND status = 'confirmed' + LIMIT 1 + """, params)) + if kind == "proforma": + return OpportunityNextAction( + action_code="CONFIRM_PAYMENT", + label="Confirmar pagamento", + description=f"Pró-forma {number} ligada. Confirmar pagamento ou acompanhar o cliente.", + priority="alta", + target_url=f"/opportunities/{opportunity_id}#tasks", + document_id=current_doc.get("id"), + document_number=number, + ).to_dict() + if kind == "invoice": + if not payment_confirmed: + legacy_note = " Se for um registo antigo/reconstruído, rever pagamento e marcar como concluído quando validado." + return OpportunityNextAction( + action_code="CONFIRM_PAYMENT", + label="Aguardar / confirmar pagamento", + description=f"Fatura {number} ligada. Confirmar pagamento ou acompanhar o cliente." + legacy_note, + priority="alta", + target_url=f"/opportunities/{opportunity_id}#tasks", + document_id=current_doc.get("id"), + document_number=number, + ).to_dict() + return OpportunityNextAction( + action_code="PREPARE_ORDER", + label="Preparar encomenda / envio", + description=f"Fatura {number} ligada e pagamento confirmado. Validar preparação, Odoo e envio.", + priority="alta", + target_url=f"/opportunities/{opportunity_id}#tasks", + document_id=current_doc.get("id"), + document_number=number, + ).to_dict() + + return OpportunityNextAction( + action_code="FOLLOW_UP", + label="Acompanhar oportunidade", + description="Processo sem bloqueios óbvios. Rever estado e próximo contacto.", + priority="baixa", + target_url=f"/opportunities/{opportunity_id}", + ).to_dict() diff --git a/app/opportunity_service.py b/app/opportunity_service.py new file mode 100644 index 0000000..e68dd1e --- /dev/null +++ b/app/opportunity_service.py @@ -0,0 +1,1137 @@ +import json +import re +import uuid +from typing import Any, Dict, List, Optional, Tuple + +from sqlalchemy import text + +from app.db import engine + + +OPPORTUNITY_STAGE_LABELS: Dict[str, str] = { + "NEW_LEAD": "Novo pedido", + "INFO_REQUESTED": "Informação solicitada", + "INFO_SENT": "Informação enviada", + "QUOTE_REQUESTED": "Orçamento solicitado", + "QUOTE_SENT": "Orçamento enviado", + "PROFORMA_REQUESTED": "Pró-forma solicitada", + "PROFORMA_SENT": "Pró-forma enviada", + "INVOICE_REQUESTED": "Fatura solicitada", + "INVOICE_SENT": "Fatura enviada", + "WAITING_PAYMENT": "A aguardar pagamento", + "PAYMENT_CONFIRMED": "Pagamento confirmado", + "ODOO_ORDER_CREATED": "Venda Odoo criada", + "IN_PRODUCTION": "Em produção", + "READY_TO_SHIP": "Pronto para envio", + "INVOICED": "Faturado", + "SHIPMENT_CREATED": "Envio criado", + "TRACKING_SENT": "Tracking enviado", + "DELIVERED": "Entregue", + "ORDER_PREPARATION": "Em preparação", + "SHIPPED": "Enviado", + "WON": "Concluído", + "LOST": "Perdido", + "NO_INTEREST": "Sem interesse", + "REVIEW": "Rever", +} + +OPPORTUNITY_STAGE_RANK: Dict[str, int] = { + "NEW_LEAD": 10, + "INFO_REQUESTED": 20, + "INFO_SENT": 30, + "QUOTE_REQUESTED": 40, + "QUOTE_SENT": 50, + "PROFORMA_REQUESTED": 55, + "PROFORMA_SENT": 60, + "INVOICE_REQUESTED": 62, + "INVOICE_SENT": 65, + "WAITING_PAYMENT": 70, + "PAYMENT_CONFIRMED": 80, + "ODOO_ORDER_CREATED": 85, + "IN_PRODUCTION": 88, + "ORDER_PREPARATION": 90, + "READY_TO_SHIP": 92, + "INVOICED": 94, + "SHIPMENT_CREATED": 98, + "SHIPPED": 100, + "TRACKING_SENT": 102, + "DELIVERED": 108, + "WON": 110, + "LOST": 120, + "NO_INTEREST": 118, + "REVIEW": 5, +} + +OPPORTUNITY_BOARD_COLUMNS: List[Tuple[str, str, List[str]]] = [ + ("requests", "Pedidos", ["NEW_LEAD", "INFO_REQUESTED", "QUOTE_REQUESTED", "PROFORMA_REQUESTED", "INVOICE_REQUESTED", "REVIEW"]), + ("sent", "Info / orçamento", ["INFO_SENT", "QUOTE_SENT", "PROFORMA_SENT", "INVOICE_SENT"]), + ("payment", "Pagamento", ["WAITING_PAYMENT", "PAYMENT_CONFIRMED"]), + ("operations", "Operação / envio", ["PAYMENT_CONFIRMED", "ODOO_ORDER_CREATED", "IN_PRODUCTION", "ORDER_PREPARATION", "READY_TO_SHIP", "INVOICED", "SHIPMENT_CREATED", "SHIPPED", "TRACKING_SENT", "DELIVERED"]), + ("closed", "Fechadas", ["WON", "LOST", "NO_INTEREST"]), +] + +# Actions that may update an existing commercial opportunity. Creation is more +# restrictive and is controlled by can_create_new_opportunity_for_task(). +OPPORTUNITY_RELEVANT_ACTION_CODES = { + "SEND_INFO", + "SEND_QUOTE", + "SEND_PROFORMA", + "SEND_INVOICE", + "CONFIRM_PAYMENT", + "PREPARE_ORDER", + "CREATE_SHIPMENT", + "MARK_NO_INTEREST", +} + +# Actions that are allowed to create a brand-new opportunity when there is no +# strong/unique existing match. Keep this list conservative: Operations can still +# hold a task without creating a commercial process. +OPPORTUNITY_CREATE_ACTION_CODES = { + "SEND_QUOTE", + "SEND_PROFORMA", + "SEND_INVOICE", + "CONFIRM_PAYMENT", + "PREPARE_ORDER", + "CREATE_SHIPMENT", +} + +NEVER_CREATE_OPPORTUNITY_ACTION_CODES = { + "REVIEW_MANUALLY", + "REMOVE_FROM_LIST", + "IGNORE_SPAM", + "NO_ACTION", + "IGNORE_BOUNCE", + "SUPPORT", + "MARK_NO_INTEREST", +} + +SYSTEM_SENDER_PATTERNS = ( + "mail delivery subsystem", + "mailer-daemon", + "postmaster", + "delivery status notification", + "microsoft exchange", + "office 365", +) + +SYSTEM_SUBJECT_PATTERNS = ( + "returned mail", + "undelivered mail", + "delivery status notification", + "failure notice", + "mail delivery failed", + "undeliverable", + "delivery has failed", + "non-delivery report", + "non delivery report", + "your message couldn't be delivered", + "your message couldn’t be delivered", + "recipient wasn't found", + "recipient was not found", + "unknown to address", + "remote server returned", + "550 5.1.1", + "5.1.10", + "wasn't found at", + "was not found at", + "office 365", + "microsoft exchange", + "não entregue", +) + +COMMERCIAL_INTENT_TERMS = ( + "orçamento", "orcamento", "cotação", "cotacao", "proposta", "preço", "preco", + "comprar", "encomendar", "encomenda", "fatura", "factura", "pró-forma", + "proforma", "pagamento", "comprovativo", "disponibilidade", "quero avançar", + "pretendo avançar", "adjudicar", "pedido de cotação", "pedido de orçamento", +) + +STAGE_ON_TASK_CREATED = { + "SEND_INFO": "INFO_REQUESTED", + "SEND_QUOTE": "QUOTE_REQUESTED", + "SEND_PROFORMA": "PROFORMA_REQUESTED", + "SEND_INVOICE": "INVOICE_REQUESTED", + "CONFIRM_PAYMENT": "WAITING_PAYMENT", + "PREPARE_ORDER": "ORDER_PREPARATION", + "CREATE_SHIPMENT": "READY_TO_SHIP", + "MARK_NO_INTEREST": "REVIEW", +} + +STAGE_ON_TASK_DONE = { + "SEND_INFO": "INFO_SENT", + "SEND_QUOTE": "QUOTE_SENT", + "SEND_PROFORMA": "WAITING_PAYMENT", + "SEND_INVOICE": "WAITING_PAYMENT", + "CONFIRM_PAYMENT": "PAYMENT_CONFIRMED", + "PREPARE_ORDER": "ORDER_PREPARATION", + "CREATE_SHIPMENT": "SHIPMENT_CREATED", + "MARK_NO_INTEREST": "NO_INTEREST", +} + + +_SCHEMA_READY = False + + +def _json(value: Any) -> str: + return json.dumps(value or {}, ensure_ascii=False, default=str) + + +def _uuid(value: Optional[str]) -> Optional[str]: + value = str(value or "").strip() + return value or None + + +def ensure_opportunity_schema() -> None: + """Cria a primeira versão local do pipeline ClientFlow. + + É intencionalmente aditiva: não altera nem remove dados antigos. + """ + global _SCHEMA_READY + if _SCHEMA_READY: + return + + with engine.begin() as conn: + conn.execute(text(""" + CREATE TABLE IF NOT EXISTS opportunities ( + id UUID PRIMARY KEY, + title TEXT NOT NULL, + stage TEXT NOT NULL DEFAULT 'NEW_LEAD', + status TEXT NOT NULL DEFAULT 'open', + contact_id TEXT, + customer_id TEXT, + conversation_id TEXT, + customer_name TEXT, + customer_email TEXT, + customer_phone TEXT, + product_interest TEXT, + value_amount NUMERIC(12,2), + currency TEXT NOT NULL DEFAULT 'EUR', + source_system TEXT NOT NULL DEFAULT 'clientflow', + source_event_id TEXT, + last_action_code TEXT, + last_task_id UUID, + last_message_at TIMESTAMPTZ, + metadata JSONB NOT NULL DEFAULT '{}'::jsonb, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + closed_at TIMESTAMPTZ + ) + """)) + conn.execute(text(""" + CREATE TABLE IF NOT EXISTS opportunity_events ( + id UUID PRIMARY KEY, + opportunity_id UUID NOT NULL REFERENCES opportunities(id) ON DELETE CASCADE, + event_type TEXT NOT NULL, + task_id UUID, + action_code TEXT, + from_stage TEXT, + to_stage TEXT, + note TEXT, + payload JSONB NOT NULL DEFAULT '{}'::jsonb, + created_by TEXT NOT NULL DEFAULT 'system', + created_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + """)) + conn.execute(text("ALTER TABLE tasks ADD COLUMN IF NOT EXISTS opportunity_id UUID")) + conn.execute(text("CREATE INDEX IF NOT EXISTS idx_opportunities_stage ON opportunities(stage)")) + conn.execute(text("CREATE INDEX IF NOT EXISTS idx_opportunities_contact ON opportunities(contact_id)")) + conn.execute(text("CREATE INDEX IF NOT EXISTS idx_opportunities_conversation ON opportunities(conversation_id)")) + conn.execute(text("CREATE INDEX IF NOT EXISTS idx_opportunity_events_opportunity ON opportunity_events(opportunity_id, created_at DESC)")) + conn.execute(text("CREATE INDEX IF NOT EXISTS idx_tasks_opportunity_id ON tasks(opportunity_id)")) + + _SCHEMA_READY = True + + +def stage_label(stage: Optional[str]) -> str: + return OPPORTUNITY_STAGE_LABELS.get(str(stage or ""), str(stage or "—")) + + +def _stage_rank(stage: Optional[str]) -> int: + return OPPORTUNITY_STAGE_RANK.get(str(stage or ""), 0) + + +def _advance_stage(current: Optional[str], suggested: Optional[str]) -> str: + current = str(current or "NEW_LEAD") + suggested = str(suggested or current) + if current in {"WON", "LOST", "NO_INTEREST"}: + return current + if _stage_rank(suggested) >= _stage_rank(current): + return suggested + return current + + +def _detect_product_interest(text_value: str) -> str: + text_value = str(text_value or "").lower() + patterns = [ + ("Carregador EV", ["carregador", "wallbox", "wall box", "ev charger", "carregamento"]), + ("Cabo", ["cabo", "type 2", "tipo 2"]), + ("Instalação", ["instalação", "instalacao", "instalar"]), + ] + for label, terms in patterns: + if any(term in text_value for term in terms): + return label + return "" + + + + +def _normalized_text(*values: Any) -> str: + return " ".join(str(value or "") for value in values).casefold() + + +def is_system_or_bounce_task(task: Dict[str, Any]) -> bool: + """Detect automatic e-mail system messages that should not become sales opportunities.""" + sender = _normalized_text(task.get("customer_name"), task.get("customer_email")) + subject = _normalized_text(task.get("message_subject"), task.get("request_text"), task.get("note")) + if any(pattern in sender for pattern in SYSTEM_SENDER_PATTERNS): + return True + return any(pattern in subject for pattern in SYSTEM_SUBJECT_PATTERNS) + + +def has_commercial_intent(task: Dict[str, Any]) -> bool: + text_value = _normalized_text(task.get("message_subject"), task.get("request_text"), task.get("note"), task.get("action")) + return any(term in text_value for term in COMMERCIAL_INTENT_TERMS) + + +def _mark_task_no_opportunity(task_id: str, *, reason: str, created_by: str = "system") -> None: + sql = text(""" + UPDATE tasks + SET metadata = COALESCE(metadata, '{}'::jsonb) || CAST(:metadata AS JSONB), + updated_at = now() + WHERE id = CAST(:task_id AS UUID) + """) + with engine.begin() as conn: + conn.execute(sql, { + "task_id": task_id, + "metadata": _json({ + "no_opportunity_reason": reason, + "opportunity_creation_status": "skipped", + "opportunity_creation_checked_by": created_by, + }), + }) + + +def can_create_new_opportunity_for_task(task: Dict[str, Any], action_code: str) -> tuple[bool, str]: + """Return whether this task is allowed to create a new opportunity. + + v4.8.3 guardrail: not every inbound Chatwoot message is a commercial + opportunity. Bounces, spam/system messages, unsubscribe requests and failed + classifications should stay as tasks/review items without polluting the + opportunity board. + """ + action_code = str(action_code or "").upper() + if is_system_or_bounce_task(task): + return False, "system_or_bounce_message" + if action_code in NEVER_CREATE_OPPORTUNITY_ACTION_CODES: + return False, "action_code_never_creates_opportunity" + if action_code == "SEND_INFO" and not has_commercial_intent(task): + return False, "send_info_without_clear_commercial_intent" + if action_code == "SEND_INFO": + return True, "send_info_with_commercial_intent" + if action_code in OPPORTUNITY_CREATE_ACTION_CODES: + return True, "commercial_action_code" + return False, "action_code_not_commercial" + +def _compact(value: Any, limit: int = 180) -> str: + value = re.sub(r"\s+", " ", str(value or "")).strip() + if len(value) > limit: + return value[: limit - 1].rstrip() + "…" + return value + + +def get_task_context(task_id: str) -> Optional[Dict[str, Any]]: + ensure_opportunity_schema() + sql = text(""" + SELECT + t.id::text, + t.action_run_id::text, + t.message_id::text, + t.raw_event_id::text, + t.opportunity_id::text, + t.conversation_id, + t.contact_id, + t.customer_id, + t.action_code, + t.route, + t.action, + t.note, + t.status, + t.source_system, + t.source_event_id, + t.created_at, + t.updated_at, + COALESCE( + NULLIF(re.payload->'sender'->>'name', ''), + NULLIF(re.payload->'conversation'->'meta'->'sender'->>'name', ''), + NULLIF(t.customer_id, ''), + NULLIF(t.contact_id, '') + ) AS customer_name, + COALESCE( + NULLIF(re.payload->'sender'->>'email', ''), + NULLIF(re.payload->'conversation'->'meta'->'sender'->>'email', ''), + NULLIF(re.payload->'conversation'->'contact_inbox'->>'source_id', '') + ) AS customer_email, + COALESCE( + NULLIF(re.payload->'sender'->>'phone_number', ''), + NULLIF(re.payload->'conversation'->'meta'->'sender'->>'phone_number', '') + ) AS customer_phone, + COALESCE( + NULLIF(re.payload->'conversation'->'additional_attributes'->>'mail_subject', ''), + NULLIF(re.payload->'content_attributes'->'email'->>'subject', ''), + NULLIF(re.payload->'conversation'->'messages'->0->'content_attributes'->'email'->>'subject', '') + ) AS message_subject, + COALESCE( + NULLIF(m.clean_body, ''), + NULLIF(m.raw_body, ''), + NULLIF(re.payload->>'content', '') + ) AS request_text + FROM tasks t + LEFT JOIN messages m ON m.id = t.message_id + LEFT JOIN raw_events re ON re.id = t.raw_event_id + WHERE t.id = CAST(:task_id AS UUID) + LIMIT 1 + """) + with engine.begin() as conn: + row = conn.execute(sql, {"task_id": task_id}).mappings().first() + return dict(row) if row else None + + +CONTACT_MATCH_RECENT_DAYS = 45 + + +def _find_open_opportunity_by_conversation(conversation_id: object) -> Optional[Dict[str, Any]]: + conversation_id = str(conversation_id or "").strip() + if not conversation_id: + return None + + sql = text(""" + SELECT * + FROM opportunities + WHERE status = 'open' + AND conversation_id = :conversation_id + ORDER BY updated_at DESC + LIMIT 1 + """) + with engine.begin() as conn: + row = conn.execute(sql, {"conversation_id": conversation_id}).mappings().first() + if not row: + return None + result = dict(row) + result["_link_match_reason"] = "conversation_id" + return result + + +def _open_opportunities_for_contact(contact_id: object, *, limit: int = 3) -> List[Dict[str, Any]]: + contact_id = str(contact_id or "").strip() + if not contact_id: + return [] + + sql = text(""" + SELECT * + FROM opportunities + WHERE status = 'open' + AND contact_id = :contact_id + AND updated_at >= now() - make_interval(days => :recent_days) + ORDER BY updated_at DESC + LIMIT :limit + """) + with engine.begin() as conn: + rows = conn.execute(sql, { + "contact_id": contact_id, + "recent_days": CONTACT_MATCH_RECENT_DAYS, + "limit": int(limit), + }).mappings().all() + return [dict(row) for row in rows] + + +def has_ambiguous_opportunity_match(task: Dict[str, Any]) -> bool: + """Return True when a Chatwoot contact maps to more than one recent open opportunity. + + conversation_id is the only strong automatic match. contact_id is weak because + it represents a Chatwoot person/contact, while fiscal customer can be a company. + """ + if _find_open_opportunity_by_conversation(task.get("conversation_id")): + return False + return len(_open_opportunities_for_contact(task.get("contact_id"), limit=2)) > 1 + + +def mark_task_opportunity_link_ambiguous(task_id: str, *, reason: str, created_by: str = "system") -> None: + sql = text(""" + UPDATE tasks + SET + route = CASE WHEN status = 'pending' THEN 'rever' ELSE route END, + metadata = COALESCE(metadata, '{}'::jsonb) || CAST(:metadata AS JSONB), + updated_at = now() + WHERE id = CAST(:task_id AS UUID) + """) + with engine.begin() as conn: + conn.execute(sql, { + "task_id": task_id, + "metadata": _json({ + "opportunity_linking_status": "ambiguous", + "opportunity_linking_reason": reason, + "opportunity_linking_checked_by": created_by, + }), + }) + + +def find_open_opportunity_for_task(task: Dict[str, Any]) -> Optional[Dict[str, Any]]: + ensure_opportunity_schema() + + # Strong match: the exact Chatwoot conversation already belongs to an open opportunity. + by_conversation = _find_open_opportunity_by_conversation(task.get("conversation_id")) + if by_conversation: + return by_conversation + + # Weak match: contact_id may be a person whose fiscal customer is a company. + # Only auto-link when there is exactly one recent open opportunity for that contact. + contact_matches = _open_opportunities_for_contact(task.get("contact_id"), limit=2) + if len(contact_matches) == 1: + result = dict(contact_matches[0]) + result["_link_match_reason"] = "contact_id_unique_recent" + return result + + return None + + +def _build_title(task: Dict[str, Any]) -> str: + subject = _compact(task.get("message_subject"), 90) + customer = _compact(task.get("customer_name") or task.get("customer_email") or task.get("contact_id"), 70) + product = _detect_product_interest(" ".join([ + str(task.get("message_subject") or ""), + str(task.get("request_text") or ""), + str(task.get("note") or ""), + ])) + if subject: + return subject + if customer and product: + return f"{customer} · {product}" + if customer: + return f"Oportunidade · {customer}" + return "Nova oportunidade" + + +def upsert_opportunity_for_task( + task_id: str, + *, + trigger: str = "task_created", + created_by: str = "system", +) -> Optional[str]: + ensure_opportunity_schema() + task = get_task_context(task_id) + if not task: + return None + + action_code = str(task.get("action_code") or "").upper() + if action_code not in OPPORTUNITY_RELEVANT_ACTION_CODES: + _mark_task_no_opportunity(task_id, reason="action_code_not_opportunity_relevant", created_by=created_by) + return None + + if is_system_or_bounce_task(task): + _mark_task_no_opportunity(task_id, reason="system_or_bounce_message", created_by=created_by) + return None + + suggested_stage = STAGE_ON_TASK_CREATED.get(action_code, "NEW_LEAD") + existing = find_open_opportunity_for_task(task) + if not existing and has_ambiguous_opportunity_match(task): + mark_task_opportunity_link_ambiguous( + task_id, + reason="multiple_recent_open_opportunities_for_chatwoot_contact", + created_by=created_by, + ) + return None + if action_code == "MARK_NO_INTEREST" and not existing: + _mark_task_no_opportunity(task_id, reason="mark_no_interest_without_existing_opportunity", created_by=created_by) + return None + if not existing: + can_create, skip_reason = can_create_new_opportunity_for_task(task, action_code) + if not can_create: + _mark_task_no_opportunity(task_id, reason=skip_reason, created_by=created_by) + return None + product_interest = _detect_product_interest(" ".join([ + str(task.get("message_subject") or ""), + str(task.get("request_text") or ""), + str(task.get("note") or ""), + ])) + + with engine.begin() as conn: + if existing: + opportunity_id = str(existing["id"]) + old_stage = str(existing.get("stage") or "NEW_LEAD") + new_stage = _advance_stage(old_stage, suggested_stage) + conn.execute(text(""" + UPDATE opportunities + SET + title = COALESCE(NULLIF(title, ''), :title), + stage = :stage, + contact_id = COALESCE(NULLIF(contact_id, ''), :contact_id), + customer_id = COALESCE(NULLIF(customer_id, ''), :customer_id), + conversation_id = COALESCE(NULLIF(conversation_id, ''), :conversation_id), + customer_name = COALESCE(NULLIF(:customer_name, ''), customer_name), + customer_email = COALESCE(NULLIF(:customer_email, ''), customer_email), + customer_phone = COALESCE(NULLIF(:customer_phone, ''), customer_phone), + product_interest = COALESCE(NULLIF(:product_interest, ''), product_interest), + last_action_code = :action_code, + last_task_id = CAST(:task_id AS UUID), + last_message_at = COALESCE(:last_message_at, now()), + updated_at = now(), + metadata = COALESCE(metadata, '{}'::jsonb) || CAST(:metadata AS JSONB) + WHERE id = CAST(:opportunity_id AS UUID) + """), { + "opportunity_id": opportunity_id, + "title": _build_title(task), + "stage": new_stage, + "contact_id": task.get("contact_id"), + "customer_id": task.get("customer_id"), + "conversation_id": task.get("conversation_id"), + "customer_name": task.get("customer_name") or "", + "customer_email": task.get("customer_email") or "", + "customer_phone": task.get("customer_phone") or "", + "product_interest": product_interest or "", + "action_code": action_code, + "task_id": task_id, + "last_message_at": task.get("created_at"), + "metadata": _json({"last_trigger": trigger}), + }) + event_id = str(uuid.uuid4()) + conn.execute(text(""" + INSERT INTO opportunity_events ( + id, opportunity_id, event_type, task_id, action_code, + from_stage, to_stage, note, payload, created_by + ) VALUES ( + CAST(:id AS UUID), CAST(:opportunity_id AS UUID), :event_type, + CAST(:task_id AS UUID), :action_code, + :from_stage, :to_stage, :note, CAST(:payload AS JSONB), :created_by + ) + """), { + "id": event_id, + "opportunity_id": opportunity_id, + "event_type": trigger, + "task_id": task_id, + "action_code": action_code, + "from_stage": old_stage, + "to_stage": new_stage, + "note": task.get("note") or task.get("action") or "", + "payload": _json({"task_status": task.get("status")}), + "created_by": created_by, + }) + else: + opportunity_id = str(uuid.uuid4()) + new_stage = suggested_stage + conn.execute(text(""" + INSERT INTO opportunities ( + id, title, stage, status, contact_id, customer_id, conversation_id, + customer_name, customer_email, customer_phone, product_interest, + source_system, source_event_id, last_action_code, last_task_id, + last_message_at, metadata + ) VALUES ( + CAST(:id AS UUID), :title, :stage, 'open', :contact_id, :customer_id, :conversation_id, + :customer_name, :customer_email, :customer_phone, :product_interest, + :source_system, :source_event_id, :action_code, CAST(:task_id AS UUID), + COALESCE(:last_message_at, now()), CAST(:metadata AS JSONB) + ) + """), { + "id": opportunity_id, + "title": _build_title(task), + "stage": new_stage, + "contact_id": task.get("contact_id"), + "customer_id": task.get("customer_id"), + "conversation_id": task.get("conversation_id"), + "customer_name": task.get("customer_name"), + "customer_email": task.get("customer_email"), + "customer_phone": task.get("customer_phone"), + "product_interest": product_interest, + "source_system": task.get("source_system") or "clientflow", + "source_event_id": task.get("source_event_id"), + "action_code": action_code, + "task_id": task_id, + "last_message_at": task.get("created_at"), + "metadata": _json({"created_from_task_id": task_id, "last_trigger": trigger}), + }) + conn.execute(text(""" + INSERT INTO opportunity_events ( + id, opportunity_id, event_type, task_id, action_code, + from_stage, to_stage, note, payload, created_by + ) VALUES ( + CAST(:id AS UUID), CAST(:opportunity_id AS UUID), :event_type, + CAST(:task_id AS UUID), :action_code, + NULL, :to_stage, :note, CAST(:payload AS JSONB), :created_by + ) + """), { + "id": str(uuid.uuid4()), + "opportunity_id": opportunity_id, + "event_type": trigger, + "task_id": task_id, + "action_code": action_code, + "to_stage": new_stage, + "note": task.get("note") or task.get("action") or "", + "payload": _json({"task_status": task.get("status")}), + "created_by": created_by, + }) + + conn.execute(text(""" + UPDATE tasks + SET opportunity_id = CAST(:opportunity_id AS UUID), + metadata = COALESCE(metadata, '{}'::jsonb) || jsonb_build_object('opportunity_id', :opportunity_id), + updated_at = now() + WHERE id = CAST(:task_id AS UUID) + """), {"opportunity_id": opportunity_id, "task_id": task_id}) + + # v4.9.25: newly created/updated opportunities should immediately try to + # get a fiscal customer suggestion. This is best-effort and never blocks + # task processing or opportunity creation. + try: + from app.fiscal_enrichment_service import enrich_opportunity + enrich_opportunity(opportunity_id, apply_safe=True) + except Exception: + pass + + return opportunity_id + + +def advance_opportunity_after_task_done( + task_id: str, + *, + event_type: Optional[str] = None, + payload: Optional[Dict[str, Any]] = None, + created_by: str = "operator", +) -> Optional[str]: + ensure_opportunity_schema() + task = get_task_context(task_id) + if not task: + return None + + opportunity_id = task.get("opportunity_id") or upsert_opportunity_for_task( + task_id, + trigger="task_completed_link_created", + created_by=created_by, + ) + if not opportunity_id: + return None + + action_code = str(task.get("action_code") or "").upper() + suggested_stage = STAGE_ON_TASK_DONE.get(action_code) + if not suggested_stage: + return str(opportunity_id) + + with engine.begin() as conn: + current = conn.execute(text(""" + SELECT stage, status FROM opportunities WHERE id = CAST(:opportunity_id AS UUID) + """), {"opportunity_id": opportunity_id}).mappings().first() + if not current: + return str(opportunity_id) + old_stage = str(current.get("stage") or "NEW_LEAD") + new_stage = _advance_stage(old_stage, suggested_stage) + status = "closed" if new_stage in {"WON", "LOST", "NO_INTEREST"} else "open" + conn.execute(text(""" + UPDATE opportunities + SET + stage = :stage, + status = :status, + closed_at = CASE WHEN CAST(:status AS TEXT) = 'closed' THEN COALESCE(closed_at, now()) ELSE closed_at END, + last_action_code = :action_code, + last_task_id = CAST(:task_id AS UUID), + updated_at = now(), + metadata = COALESCE(metadata, '{}'::jsonb) || CAST(:metadata AS JSONB) + WHERE id = CAST(:opportunity_id AS UUID) + """), { + "opportunity_id": opportunity_id, + "stage": new_stage, + "status": status, + "action_code": action_code, + "task_id": task_id, + "metadata": _json({"last_done_event_type": event_type or "task_done"}), + }) + conn.execute(text(""" + INSERT INTO opportunity_events ( + id, opportunity_id, event_type, task_id, action_code, + from_stage, to_stage, note, payload, created_by + ) VALUES ( + CAST(:id AS UUID), CAST(:opportunity_id AS UUID), CAST(:event_type AS TEXT), + CAST(:task_id AS UUID), CAST(:action_code AS TEXT), + CAST(:from_stage AS TEXT), CAST(:to_stage AS TEXT), CAST(:note AS TEXT), CAST(:payload AS JSONB), CAST(:created_by AS TEXT) + ) + """), { + "id": str(uuid.uuid4()), + "opportunity_id": opportunity_id, + "event_type": event_type or "task_done", + "task_id": task_id, + "action_code": action_code, + "from_stage": old_stage, + "to_stage": new_stage, + "note": f"Tarefa concluída: {task.get('action') or action_code}", + "payload": _json(payload or {}), + "created_by": created_by, + }) + + return str(opportunity_id) + + +MANUAL_REQUEST_TYPES = { + "quote": ("SEND_QUOTE", "QUOTE_REQUESTED", "Orçamento"), + "info": ("SEND_INFO", "INFO_REQUESTED", "Informação"), + "proforma": ("SEND_PROFORMA", "PROFORMA_REQUESTED", "Pró-forma"), + "invoice": ("SEND_INVOICE", "INVOICE_REQUESTED", "Fatura"), + "order": ("SEND_QUOTE", "QUOTE_REQUESTED", "Encomenda"), + "support": ("SUPPORT", "REVIEW", "Assistência"), + "manual": ("SEND_QUOTE", "NEW_LEAD", "Pedido manual"), +} + +MANUAL_ORIGIN_LABELS = { + "phone": "telefone", + "manual": "manual", + "email": "email", + "whatsapp": "WhatsApp", + "presential": "presencial", + "reconciliation": "reconciliação", +} + + +def create_manual_opportunity_from_customer( + customer_id: str, + *, + origin: str = "phone", + request_type: str = "quote", + contact_name: str = "", + contact_email: str = "", + contact_phone: str = "", + product_interest: str = "", + notes: str = "", + create_task: bool = True, + created_by: str = "operator", +) -> Dict[str, Any]: + """Create a fiscal-customer-first opportunity from the customer page. + + This covers manual/phone/WhatsApp/presential requests where no Chatwoot + conversation exists yet. The opportunity is created already linked to the + fiscal customer, and can optionally create the initial human task. + """ + from app.action_catalog import get_action_config + from app.commercial_service import get_customer, ensure_commercial_schema + from app.db import ensure_core_schema + + ensure_core_schema() + ensure_opportunity_schema() + ensure_commercial_schema() + + customer = get_customer(customer_id) + if not customer: + raise ValueError("Cliente fiscal não encontrado") + + origin_key = str(origin or "phone").strip().lower() or "phone" + request_key = str(request_type or "quote").strip().lower() or "quote" + action_code, stage, request_label = MANUAL_REQUEST_TYPES.get(request_key, MANUAL_REQUEST_TYPES["quote"]) + origin_label = MANUAL_ORIGIN_LABELS.get(origin_key, origin_key) + + contact_name = str(contact_name or "").strip() + contact_email = str(contact_email or "").strip() + contact_phone = str(contact_phone or "").strip() + product_interest = str(product_interest or "").strip() + notes = str(notes or "").strip() + + fiscal_name = str(customer.get("name") or "Cliente").strip() + title_bits = [request_label, fiscal_name] + if product_interest: + title_bits.append(product_interest[:80]) + title = " — ".join(bit for bit in title_bits if bit) + + config = get_action_config(action_code) + metadata = { + "created_manually": True, + "created_from_customer_page": True, + "origin": origin_key, + "origin_label": origin_label, + "request_type": request_key, + "notes": notes, + "customer_snapshot": { + "id": str(customer.get("id") or customer_id), + "name": customer.get("name"), + "tax_id": customer.get("tax_id"), + "email": customer.get("email"), + "phone": customer.get("phone"), + "street_name": customer.get("street_name"), + "postal_zone": customer.get("postal_zone"), + "city_name": customer.get("city_name"), + "country": customer.get("country"), + }, + } + + opportunity_id = str(uuid.uuid4()) + task_id: Optional[str] = None + with engine.begin() as conn: + conn.execute(text(""" + INSERT INTO opportunities ( + id, title, stage, status, local_customer_id, + customer_name, customer_email, customer_phone, product_interest, + source_system, source_event_id, last_action_code, metadata + ) VALUES ( + CAST(:id AS UUID), :title, :stage, 'open', CAST(:customer_id AS UUID), + :customer_name, :customer_email, :customer_phone, :product_interest, + 'clientflow_manual', :source_event_id, :action_code, CAST(:metadata AS JSONB) + ) + """), { + "id": opportunity_id, + "title": title, + "stage": stage, + "customer_id": customer_id, + "customer_name": contact_name or fiscal_name, + "customer_email": contact_email or customer.get("email"), + "customer_phone": contact_phone or customer.get("phone"), + "product_interest": product_interest, + "source_event_id": f"manual:{opportunity_id}", + "action_code": action_code, + "metadata": _json(metadata), + }) + + conn.execute(text(""" + INSERT INTO opportunity_events ( + id, opportunity_id, event_type, action_code, from_stage, to_stage, + note, payload, created_by + ) VALUES ( + gen_random_uuid(), CAST(:opportunity_id AS UUID), 'manual_opportunity_created', + :action_code, NULL, :stage, :note, CAST(:payload AS JSONB), :created_by + ) + """), { + "opportunity_id": opportunity_id, + "action_code": action_code, + "stage": stage, + "note": notes or f"Oportunidade criada manualmente a partir do cliente fiscal por {origin_label}.", + "payload": _json(metadata), + "created_by": created_by, + }) + + if create_task and action_code: + task_row = conn.execute(text(""" + INSERT INTO tasks ( + opportunity_id, customer_id, action_code, route, action, note, + action_required, safe_to_post, status, source_system, source_event_id, + idempotency_key, metadata + ) VALUES ( + CAST(:opportunity_id AS UUID), :customer_id, :action_code, :route, :action, :note, + :action_required, :safe_to_post, 'pending', 'clientflow_manual', :source_event_id, + :idempotency_key, CAST(:metadata AS JSONB) + ) + RETURNING id::text + """), { + "opportunity_id": opportunity_id, + "customer_id": str(customer_id), + "action_code": action_code, + "route": config.get("route") or "vendas", + "action": config.get("action") or action_code, + "note": notes or f"Pedido criado manualmente por {origin_label}. Preparar próxima ação.", + "action_required": bool(config.get("action_required", True)), + "safe_to_post": bool(config.get("safe_to_post", False)), + "source_event_id": f"manual:{opportunity_id}", + "idempotency_key": f"manual-opportunity:{opportunity_id}:initial-task", + "metadata": _json({ + "created_from_manual_opportunity": True, + "opportunity_id": opportunity_id, + "origin": origin_key, + "request_type": request_key, + "customer_snapshot": metadata["customer_snapshot"], + }), + }).mappings().first() + task_id = str(task_row["id"]) if task_row else None + if task_id: + conn.execute(text(""" + UPDATE opportunities + SET last_task_id = CAST(:task_id AS UUID), updated_at = now() + WHERE id = CAST(:opportunity_id AS UUID) + """), {"opportunity_id": opportunity_id, "task_id": task_id}) + conn.execute(text(""" + INSERT INTO task_events (task_id, event_type, payload, created_by) + VALUES (CAST(:task_id AS UUID), 'task_created', CAST(:payload AS JSONB), :created_by) + """), { + "task_id": task_id, + "payload": _json({"source": "manual_opportunity", "opportunity_id": opportunity_id}), + "created_by": created_by, + }) + + return { + "ok": True, + "opportunity_id": opportunity_id, + "task_id": task_id, + "customer_id": str(customer_id), + "stage": stage, + "action_code": action_code, + "next_url": f"/opportunities/{opportunity_id}", + } + +def list_opportunities( + *, + stage: Optional[str] = None, + q: Optional[str] = None, + status: Optional[str] = None, + limit: int = 300, +) -> List[Dict[str, Any]]: + ensure_opportunity_schema() + # A oportunidade mantém dois conceitos diferentes: + # - o.customer_*: contacto/origem captado da conversa/tarefa; + # - o.local_customer_id -> customers: ficha fiscal usada em Jasmin/documentos. + # A board deve mostrar ambos quando divergem, para evitar abrir uma oportunidade + # que parece ser de um contacto mas emite documentos para outro cliente fiscal. + filters = [] + params: Dict[str, Any] = {"limit": int(limit)} + if stage and stage != "all": + filters.append("o.stage = :stage") + params["stage"] = stage + if status and status != "all": + filters.append("o.status = :status") + params["status"] = status + if q: + filters.append(""" + ( + o.title ILIKE :q OR o.customer_name ILIKE :q OR o.customer_email ILIKE :q + OR o.contact_id ILIKE :q OR o.conversation_id ILIKE :q OR o.product_interest ILIKE :q + OR c.name ILIKE :q OR c.email ILIKE :q OR c.tax_id ILIKE :q + ) + """) + params["q"] = f"%{str(q).strip()}%" + where_sql = "WHERE " + " AND ".join(filters) if filters else "" + sql = text(f""" + SELECT + o.*, + c.id::text AS linked_customer_id, + c.name AS linked_customer_name, + c.email AS linked_customer_email, + c.tax_id AS linked_customer_tax_id, + c.street_name AS linked_customer_street_name, + c.postal_zone AS linked_customer_postal_zone, + c.city_name AS linked_customer_city_name, + c.phone AS linked_customer_phone, + (SELECT count(*) FROM tasks t WHERE t.opportunity_id = o.id) AS task_count, + (SELECT count(*) FROM tasks t WHERE t.opportunity_id = o.id AND t.status = 'pending') AS pending_task_count + FROM opportunities o + LEFT JOIN customers c ON c.id = o.local_customer_id + {where_sql} + ORDER BY + CASE o.status WHEN 'open' THEN 0 ELSE 1 END, + o.updated_at DESC + LIMIT :limit + """) + with engine.begin() as conn: + rows = conn.execute(sql, params).mappings().all() + return [dict(row) for row in rows] + + +def get_opportunity(opportunity_id: str) -> Optional[Dict[str, Any]]: + ensure_opportunity_schema() + sql = text(""" + SELECT + o.*, + c.id::text AS linked_customer_id, + c.name AS linked_customer_name, + c.email AS linked_customer_email, + c.tax_id AS linked_customer_tax_id, + c.street_name AS linked_customer_street_name, + c.postal_zone AS linked_customer_postal_zone, + c.city_name AS linked_customer_city_name, + c.phone AS linked_customer_phone, + (SELECT count(*) FROM tasks t WHERE t.opportunity_id = o.id) AS task_count, + (SELECT count(*) FROM tasks t WHERE t.opportunity_id = o.id AND t.status = 'pending') AS pending_task_count + FROM opportunities o + LEFT JOIN customers c ON c.id = o.local_customer_id + WHERE o.id = CAST(:opportunity_id AS UUID) + LIMIT 1 + """) + with engine.begin() as conn: + row = conn.execute(sql, {"opportunity_id": opportunity_id}).mappings().first() + return dict(row) if row else None + + +def list_opportunity_tasks(opportunity_id: str, *, limit: int = 100) -> List[Dict[str, Any]]: + ensure_opportunity_schema() + sql = text(""" + SELECT + id::text, + created_at, + updated_at, + action_code, + route, + action, + note, + status, + conversation_id, + contact_id, + done_at, + done_by + FROM tasks + WHERE opportunity_id = CAST(:opportunity_id AS UUID) + ORDER BY created_at DESC + LIMIT :limit + """) + with engine.begin() as conn: + rows = conn.execute(sql, {"opportunity_id": opportunity_id, "limit": int(limit)}).mappings().all() + return [dict(row) for row in rows] + + +def list_opportunity_events(opportunity_id: str, *, limit: int = 100) -> List[Dict[str, Any]]: + ensure_opportunity_schema() + sql = text(""" + SELECT + id::text, + opportunity_id::text, + event_type, + task_id::text, + action_code, + from_stage, + to_stage, + note, + payload, + created_by, + created_at + FROM opportunity_events + WHERE opportunity_id = CAST(:opportunity_id AS UUID) + ORDER BY created_at DESC + LIMIT :limit + """) + with engine.begin() as conn: + rows = conn.execute(sql, {"opportunity_id": opportunity_id, "limit": int(limit)}).mappings().all() + return [dict(row) for row in rows] + + +def set_opportunity_stage( + opportunity_id: str, + stage: str, + *, + note: str = "", + created_by: str = "operator", +) -> bool: + ensure_opportunity_schema() + stage = str(stage or "").strip().upper() + if stage not in OPPORTUNITY_STAGE_LABELS: + raise ValueError(f"Unsupported opportunity stage: {stage}") + + with engine.begin() as conn: + current = conn.execute(text(""" + SELECT stage FROM opportunities WHERE id = CAST(:opportunity_id AS UUID) + """), {"opportunity_id": opportunity_id}).mappings().first() + if not current: + return False + old_stage = current.get("stage") + status = "closed" if stage in {"WON", "LOST", "NO_INTEREST"} else "open" + conn.execute(text(""" + UPDATE opportunities + SET stage = CAST(:stage AS TEXT), + status = CAST(:status AS TEXT), + closed_at = CASE WHEN CAST(:status AS TEXT) = 'closed' THEN COALESCE(closed_at, now()) ELSE NULL END, + updated_at = now(), + metadata = COALESCE(metadata, '{}'::jsonb) || jsonb_build_object('manual_stage_changed_at', now(), 'manual_stage_changed_by', CAST(:created_by AS TEXT)) + WHERE id = CAST(:opportunity_id AS UUID) + """), {"opportunity_id": opportunity_id, "stage": stage, "status": status, "created_by": created_by}) + conn.execute(text(""" + INSERT INTO opportunity_events ( + id, opportunity_id, event_type, from_stage, to_stage, note, payload, created_by + ) VALUES ( + CAST(:id AS UUID), CAST(:opportunity_id AS UUID), 'manual_stage_changed', + CAST(:from_stage AS TEXT), CAST(:to_stage AS TEXT), CAST(:note AS TEXT), '{}'::jsonb, CAST(:created_by AS TEXT) + ) + """), { + "id": str(uuid.uuid4()), + "opportunity_id": opportunity_id, + "from_stage": old_stage, + "to_stage": stage, + "note": note or f"Estado alterado manualmente para {stage_label(stage)}.", + "created_by": created_by, + }) + return True diff --git a/app/outbox/__init__.py b/app/outbox/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/packlink_client.py b/app/packlink_client.py new file mode 100644 index 0000000..47a55c5 --- /dev/null +++ b/app/packlink_client.py @@ -0,0 +1,145 @@ +"""Cliente mínimo para Packlink PRO. + +A Packlink PRO usa API key no header Authorization, sem Bearer. +Base testada: https://api.packlink.com/v1 +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Dict, List, Optional + +import httpx + +from app.config import settings + + +class PacklinkError(RuntimeError): + pass + + +@dataclass(frozen=True) +class PacklinkConfig: + base_url: str + api_key: str + + +def get_packlink_config() -> PacklinkConfig: + base_url = (settings.packlink_base_url or "https://api.packlink.com/v1").rstrip("/") + api_key = (settings.packlink_api_key or "").strip() + if not api_key: + raise PacklinkError("PACKLINK_API_KEY em falta") + return PacklinkConfig(base_url=base_url, api_key=api_key) + + +class PacklinkClient: + def __init__(self, config: Optional[PacklinkConfig] = None, timeout: float = 30.0): + self.config = config or get_packlink_config() + self.timeout = timeout + + @property + def headers(self) -> Dict[str, str]: + return { + "Authorization": self.config.api_key, + "Accept": "application/json", + "Content-Type": "application/json", + } + + async def _request( + self, + method: str, + path: str, + *, + params: Optional[Dict[str, Any]] = None, + json_body: Optional[Dict[str, Any]] = None, + ) -> Any: + url = f"{self.config.base_url}/{path.lstrip('/')}" + async with httpx.AsyncClient(timeout=self.timeout) as client: + response = await client.request( + method.upper(), + url, + headers=self.headers, + params=params, + json=json_body, + ) + + if response.status_code >= 400: + detail = response.text + try: + detail = response.json() + except Exception: + pass + raise PacklinkError(f"Packlink error {response.status_code} em {method.upper()} {path}: {detail}") + + if not response.content: + return None + try: + return response.json() + except Exception as exc: + raise PacklinkError(f"Resposta Packlink não é JSON em {method.upper()} {path}: {response.text[:500]}") from exc + + async def get_client(self) -> Dict[str, Any]: + return await self._request("GET", "/clients") + + async def get_warehouses(self) -> List[Dict[str, Any]]: + data = await self._request("GET", "/clients/warehouses") + return data or [] + + async def get_parcels(self) -> List[Dict[str, Any]]: + data = await self._request("GET", "/users/parcels") + return data or [] + + async def validate_postal_code(self, country: str, zip_code: str) -> List[Dict[str, Any]]: + return await self._request("GET", f"/locations/postalcodes/{country}/{zip_code}") + + async def quote_services( + self, + *, + from_country: str, + from_zip: str, + to_country: str, + to_zip: str, + packages: List[Dict[str, Any]], + source: str = "PRO", + ) -> List[Dict[str, Any]]: + params: Dict[str, Any] = { + "from[country]": from_country, + "from[zip]": from_zip, + "to[country]": to_country, + "to[zip]": to_zip, + "source": source, + } + for idx, package in enumerate(packages): + params[f"packages[{idx}][height]"] = package.get("height") + params[f"packages[{idx}][width]"] = package.get("width") + params[f"packages[{idx}][length]"] = package.get("length") + params[f"packages[{idx}][weight]"] = package.get("weight") + data = await self._request("GET", "/services", params=params) + return data or [] + + async def get_service_details(self, service_id: str | int) -> Dict[str, Any]: + return await self._request("GET", f"/services/available/{service_id}/details") + + async def create_shipment(self, payload: Dict[str, Any]) -> Dict[str, Any]: + return await self._request("POST", "/shipments", json_body=payload) + + async def list_shipments(self, *, limit: int = 100, offset: int = 0) -> Any: + """List shipments when Packlink exposes the collection endpoint. + + Some Packlink tenants/API versions may restrict this endpoint. The + reconciliation sync catches those API errors and reports them instead + of creating operational tasks. + """ + params = {"limit": min(max(int(limit), 1), 100), "offset": max(int(offset), 0)} + return await self._request("GET", "/shipments", params=params) + + async def get_shipment(self, reference: str) -> Dict[str, Any]: + return await self._request("GET", f"/shipments/{reference}") + + async def get_labels(self, reference: str) -> List[Any]: + data = await self._request("GET", f"/shipments/{reference}/labels") + return data or [] + + async def get_tracking(self, reference: str) -> List[Dict[str, Any]]: + data = await self._request("GET", f"/shipments/{reference}/track") + return data or [] diff --git a/app/packlink_service.py b/app/packlink_service.py new file mode 100644 index 0000000..e120ca6 --- /dev/null +++ b/app/packlink_service.py @@ -0,0 +1,383 @@ +"""Serviço Packlink para ClientFlow. + +Implementa a primeira integração segura: +- normalização de códigos postais PT para cotação; +- criação de payload de envio a partir da oportunidade/preparação; +- criação do envio via Packlink e registo em operation_links. +""" + +from __future__ import annotations + +import json +import os +import re +import uuid +from datetime import date, timedelta +from decimal import Decimal +from typing import Any, Dict, List, Optional, Tuple + +from sqlalchemy import text + +from app.config import settings +from app.db import engine +from app.integration_outbox_service import create_outbox_item +from app.commercial_service import get_customer_for_opportunity +from app.operation_service import register_operation_action +from app.opportunity_service import get_opportunity +from app.packlink_client import PacklinkClient, PacklinkError +from app.product_service import list_opportunity_items +from app.workflow_guard import validate_operation_action + + +class PacklinkPayloadError(ValueError): + pass + + +def _env(name: str, default: str = "") -> str: + return os.getenv(name, default).strip() + + +def _env_float(name: str, default: float) -> float: + value = _env(name) + if not value: + return default + try: + return float(value) + except ValueError: + return default + + +def _as_float(value: Any, default: float = 0.0) -> float: + if value is None: + return default + if isinstance(value, Decimal): + return float(value) + try: + return float(value) + except Exception: + return default + + +def _compact(value: Any, limit: int = 120) -> str: + value = re.sub(r"\s+", " ", str(value or "")).strip() + return value[:limit].strip() + + +def normalize_packlink_zip(country: str, zip_code: str, *, for_quote: bool = True) -> str: + """Normaliza código postal para Packlink. + + Nos testes reais de PT, /services aceitou 4000 mas rejeitou 4000-001. + Para moradas de envio pode ser útil manter o código completo; para cotação + usamos os 4 primeiros dígitos. + """ + country = str(country or "").upper().strip() + zip_code = str(zip_code or "").strip() + if country == "PT" and for_quote: + match = re.search(r"\d{4}", zip_code) + if match: + return match.group(0) + return zip_code + + +def default_package() -> Dict[str, Any]: + return { + "height": int(_env_float("PACKLINK_DEFAULT_PACKAGE_HEIGHT", 10)), + "width": int(_env_float("PACKLINK_DEFAULT_PACKAGE_WIDTH", 20)), + "length": int(_env_float("PACKLINK_DEFAULT_PACKAGE_LENGTH", 30)), + "weight": round(_env_float("PACKLINK_DEFAULT_PACKAGE_WEIGHT", 2), 2), + } + + +def get_latest_shipment_preparation_for_opportunity(opportunity_id: str) -> Optional[Dict[str, Any]]: + sql = text(""" + SELECT tp.* + FROM task_preparations tp + JOIN tasks t ON t.id = tp.task_id + WHERE t.opportunity_id = CAST(:opportunity_id AS UUID) + AND tp.prep_type IN ('shipment', 'pickup') + ORDER BY tp.created_at DESC + LIMIT 1 + """) + with engine.begin() as conn: + row = conn.execute(sql, {"opportunity_id": opportunity_id}).mappings().first() + return dict(row) if row else None + + +def split_pt_address(address: str) -> Tuple[str, str, str]: + """Extrai rua, código postal e cidade de uma morada PT em texto livre.""" + address = str(address or "").strip() + if not address: + return "", "", "" + clean = re.sub(r"\s+", " ", address.replace("\n", " ")).strip() + match = re.search(r"\b(\d{4}-\d{3}|\d{4})\b\s*(.*)$", clean) + if not match: + return clean, "", "" + zip_code = match.group(1) + city = _compact(match.group(2), 80) + street = clean[: match.start()].strip(" ,-") + return street, zip_code, city + + +def _metadata_dict(opp: Dict[str, Any]) -> Dict[str, Any]: + value = opp.get("metadata") or {} + if isinstance(value, dict): + return value + try: + return json.loads(value) + except Exception: + return {} + + +def _shipment_data_from_sources(opportunity: Dict[str, Any]) -> Dict[str, Any]: + metadata = _metadata_dict(opportunity) + shipment = {} + for key in ("packlink", "shipment", "shipping"): + if isinstance(metadata.get(key), dict): + shipment.update(metadata[key]) + + prep = get_latest_shipment_preparation_for_opportunity(str(opportunity["id"])) + if prep: + extracted = prep.get("extracted_data") or {} + if isinstance(extracted, str): + try: + extracted = json.loads(extracted) + except Exception: + extracted = {} + if isinstance(extracted, dict): + prep_shipment = extracted.get("shipment") or {} + if isinstance(prep_shipment, dict): + # A preparação é mais recente, mas não deve apagar campos já explícitos. + shipment = {**prep_shipment, **shipment} + + return shipment + + +def build_to_address(opportunity: Dict[str, Any]) -> Dict[str, Any]: + shipment = _shipment_data_from_sources(opportunity) + linked_customer = None + try: + linked_customer = get_customer_for_opportunity(str(opportunity.get("id"))) + except Exception: + linked_customer = None + linked_customer = linked_customer or {} + + delivery_address = ( + shipment.get("delivery_address") + or shipment.get("address") + or shipment.get("street1") + or linked_customer.get("street_name") + or "" + ) + street, parsed_zip, parsed_city = split_pt_address(delivery_address) + + country = _compact(shipment.get("country") or shipment.get("recipient_country") or linked_customer.get("country") or "PT", 2).upper() or "PT" + zip_code = _compact(shipment.get("zip_code") or shipment.get("postal_code") or linked_customer.get("postal_zone") or parsed_zip, 20) + city = _compact(shipment.get("city") or linked_customer.get("city_name") or parsed_city, 80) + street1 = _compact(shipment.get("street1") or street or linked_customer.get("street_name") or delivery_address, 120) + + name = _compact( + shipment.get("recipient_name") + or shipment.get("name") + or linked_customer.get("name") + or opportunity.get("customer_name") + or opportunity.get("customer_email") + or "Cliente", + 60, + ) + phone = _compact(shipment.get("recipient_phone") or linked_customer.get("phone") or opportunity.get("customer_phone") or _env("PACKLINK_FALLBACK_PHONE"), 30) + email = _compact(shipment.get("recipient_email") or linked_customer.get("email") or opportunity.get("customer_email") or _env("PACKLINK_FALLBACK_EMAIL"), 120) + + missing = [] + if not street1: + missing.append("morada de entrega") + if not zip_code: + missing.append("código postal de entrega") + if not city: + missing.append("cidade de entrega") + if not phone: + missing.append("telefone do destinatário") + if not email: + missing.append("email do destinatário") + if missing: + raise PacklinkPayloadError("Dados em falta para Packlink: " + ", ".join(missing)) + + return { + "name": name, + "surname": _compact(shipment.get("recipient_surname") or ".", 60), + "company": _compact(shipment.get("company") or "", 80), + "street1": street1, + "street2": _compact(shipment.get("street2") or "", 120), + "zip_code": zip_code, + "city": city, + "country": country, + "phone": phone, + "email": email, + } + + +def build_from_address() -> Dict[str, Any]: + required = { + "PACKLINK_SENDER_NAME": _env("PACKLINK_SENDER_NAME"), + "PACKLINK_SENDER_STREET1": _env("PACKLINK_SENDER_STREET1"), + "PACKLINK_SENDER_ZIP": _env("PACKLINK_SENDER_ZIP"), + "PACKLINK_SENDER_CITY": _env("PACKLINK_SENDER_CITY"), + "PACKLINK_SENDER_PHONE": _env("PACKLINK_SENDER_PHONE"), + "PACKLINK_SENDER_EMAIL": _env("PACKLINK_SENDER_EMAIL"), + } + missing = [k for k, v in required.items() if not v] + if missing: + raise PacklinkPayloadError("Configuração Packlink remetente em falta: " + ", ".join(missing)) + + return { + "name": required["PACKLINK_SENDER_NAME"], + "surname": _env("PACKLINK_SENDER_SURNAME", "."), + "company": _env("PACKLINK_SENDER_COMPANY"), + "street1": required["PACKLINK_SENDER_STREET1"], + "street2": _env("PACKLINK_SENDER_STREET2"), + "zip_code": required["PACKLINK_SENDER_ZIP"], + "city": required["PACKLINK_SENDER_CITY"], + "country": _env("PACKLINK_SENDER_COUNTRY", "PT"), + "phone": required["PACKLINK_SENDER_PHONE"], + "email": required["PACKLINK_SENDER_EMAIL"], + } + + +def default_collection_date() -> str: + # Dia seguinte; a Packlink devolve datas reais em /services, mas isto é um fallback. + return (date.today() + timedelta(days=int(_env_float("PACKLINK_COLLECTION_DAYS_AHEAD", 1)))).strftime("%Y/%m/%d") + + +def build_packlink_shipment_payload(opportunity_id: str) -> Dict[str, Any]: + opportunity = get_opportunity(opportunity_id) + if not opportunity: + raise PacklinkPayloadError(f"Oportunidade não encontrada: {opportunity_id}") + + to_address = build_to_address(opportunity) + from_address = build_from_address() + items = list_opportunity_items(opportunity_id) + content_names = [str(i.get("product_name") or i.get("sku") or "Produto") for i in items] or [opportunity.get("product_interest") or "Produto"] + content_value = sum(_as_float(i.get("total_price")) for i in items) or _as_float(opportunity.get("value_amount"), 1.0) + + service_id = _env("PACKLINK_DEFAULT_SERVICE_ID", "20571") + service_name = _env("PACKLINK_DEFAULT_SERVICE", "Paq 24") + carrier = _env("PACKLINK_DEFAULT_CARRIER", "Correos Express") + + return { + "platform": _env("PACKLINK_PLATFORM", "PRO"), + "platform_country": _env("PACKLINK_PLATFORM_COUNTRY", "UN"), + "source": _env("PACKLINK_SOURCE", "PRO"), + "service": service_name, + "carrier": carrier, + "service_id": service_id, + "collection_date": _env("PACKLINK_COLLECTION_DATE") or default_collection_date(), + "collection_time": _env("PACKLINK_COLLECTION_TIME", "09:00-14:00"), + "from": from_address, + "to": to_address, + "packages": [default_package()], + "content": content_names[:5], + "contentvalue": round(float(content_value or 1.0), 2), + "content_second_hand": False, + "shipment_custom_reference": _compact(f"CF-{opportunity_id}", 50), + "priority": False, + "contentValue_currency": str(opportunity.get("currency") or "EUR"), + "has_customs": False, + } + + +async def quote_opportunity_shipping(opportunity_id: str) -> List[Dict[str, Any]]: + opportunity = get_opportunity(opportunity_id) + if not opportunity: + raise PacklinkPayloadError(f"Oportunidade não encontrada: {opportunity_id}") + to_address = build_to_address(opportunity) + from_address = build_from_address() + client = PacklinkClient() + return await client.quote_services( + from_country=from_address["country"], + from_zip=normalize_packlink_zip(from_address["country"], from_address["zip_code"], for_quote=True), + to_country=to_address["country"], + to_zip=normalize_packlink_zip(to_address["country"], to_address["zip_code"], for_quote=True), + packages=[default_package()], + source=_env("PACKLINK_SOURCE", "PRO"), + ) + + +def enqueue_packlink_shipment(opportunity_id: str, *, created_by: str = "operator") -> str: + validate_operation_action(opportunity_id, "packlink_shipment") + payload = build_packlink_shipment_payload(opportunity_id) + outbox_id = create_outbox_item( + business_event_id=str(uuid.uuid4()), + target_system="packlink", + action_type="create_shipment", + payload={"opportunity_id": opportunity_id, "shipment": payload, "created_by": created_by}, + idempotency_key=f"packlink:create_shipment:{opportunity_id}", + ) + if not outbox_id: + raise PacklinkPayloadError("Já existe pedido Packlink pendente/enviado para esta oportunidade") + return outbox_id + + +async def create_shipment_from_outbox_payload(payload: Dict[str, Any]) -> Dict[str, Any]: + opportunity_id = str(payload.get("opportunity_id") or "").strip() + if not opportunity_id: + raise PacklinkPayloadError("opportunity_id em falta no payload Packlink") + shipment_payload = payload.get("shipment") or build_packlink_shipment_payload(opportunity_id) + + client = PacklinkClient() + result = await client.create_shipment(shipment_payload) + reference = result.get("reference") or result.get("packlink_reference") or result.get("id") + if not reference: + raise PacklinkError(f"Packlink não devolveu referência: {result}") + + shipment = None + labels: List[Any] = [] + tracking: List[Dict[str, Any]] = [] + try: + shipment = await client.get_shipment(str(reference)) + except Exception: + shipment = None + try: + labels = await client.get_labels(str(reference)) + except Exception: + labels = [] + try: + tracking = await client.get_tracking(str(reference)) + except Exception: + tracking = [] + + payload_to_store = { + "packlink_result": result, + "shipment": shipment, + "labels": labels, + "tracking": tracking, + "request": shipment_payload, + } + external_url = (settings.packlink_public_url or "https://pro.packlink.pt").rstrip("/") + try: + from app.commercial_service import upsert_shipment_record, get_customer_for_opportunity + linked_customer = get_customer_for_opportunity(opportunity_id) or {} + upsert_shipment_record({ + "customer_id": linked_customer.get("id"), + "opportunity_id": opportunity_id, + "external_reference": str(reference), + "carrier": shipment_payload.get("carrier"), + "service_id": shipment_payload.get("service_id"), + "service_name": shipment_payload.get("service"), + "status": "created", + "price": result.get("price") or result.get("total_price"), + "currency": result.get("currency") or shipment_payload.get("contentValue_currency") or "EUR", + "payload": payload_to_store, + }) + except Exception: + pass + + register_operation_action( + opportunity_id, + "packlink_shipment", + external_id=str(reference), + external_name=f"Packlink {reference}", + external_url=external_url, + note="Envio Packlink criado automaticamente pelo outbox.", + payload=payload_to_store, + created_by=str(payload.get("created_by") or "system"), + ) + return {"reference": reference, "shipment": shipment, "labels": labels, "tracking": tracking} diff --git a/app/persistence.py b/app/persistence.py new file mode 100644 index 0000000..3f160f9 --- /dev/null +++ b/app/persistence.py @@ -0,0 +1,290 @@ +from typing import Any, Dict, Optional, Tuple +import json + +from sqlalchemy import text + +from app.config import settings +from app.db import engine +from app.schemas import ( + ActionDecision, + ActionResult, + AnalyzeRequest, + CurrentState, + UsageInfo, +) + + +def _json(value) -> str: + if hasattr(value, "model_dump"): + value = value.model_dump() + return json.dumps(value or {}, ensure_ascii=False) + + +def _uuid_or_none(value): + value = str(value or "").strip() + return value or None + + +def save_action_run( + *, + request: AnalyzeRequest, + action_decision: ActionDecision, + action_result: ActionResult, + usage: UsageInfo, + needs_review: bool, + model: str, + decision_source: str, + raw_body: Optional[str] = None, + clean_body: Optional[str] = None, + raw_event_id: Optional[str] = None, +) -> Tuple[str, str]: + if not settings.clientflow_persist: + return "", "" + + conversation_id = request.conversation_id or "manual" + + with engine.begin() as conn: + message_row = conn.execute(text(""" + INSERT INTO messages ( + raw_event_id, + source_system, + source_event_id, + conversation_id, + contact_id, + direction, + raw_body, + clean_body, + previous_context, + metadata + ) + VALUES ( + CAST(:raw_event_id AS UUID), + :source_system, + NULL, + :conversation_id, + :contact_id, + 'inbound', + :raw_body, + :clean_body, + :previous_context, + CAST(:metadata AS JSONB) + ) + RETURNING id::text + """), { + "raw_event_id": raw_event_id, + "source_system": request.source or "manual", + "conversation_id": conversation_id, + "contact_id": request.contact_id, + "raw_body": raw_body or request.last_customer_message, + "clean_body": clean_body or request.last_customer_message, + "previous_context": request.previous_context, + "metadata": _json({ + "current_state": request.current_state.model_dump(), + }), + }).fetchone() + + message_id = message_row[0] + + run_row = conn.execute(text(""" + INSERT INTO action_runs ( + message_id, + raw_event_id, + conversation_id, + contact_id, + source_system, + + model, + provider, + openrouter_generation_id, + decision_source, + + action_decision, + action_result, + + prompt_tokens, + completion_tokens, + total_tokens, + cost, + usage, + + needs_review + ) + VALUES ( + CAST(:message_id AS UUID), + CAST(:raw_event_id AS UUID), + :conversation_id, + :contact_id, + :source_system, + + :model, + :provider, + :openrouter_generation_id, + :decision_source, + + CAST(:action_decision AS JSONB), + CAST(:action_result AS JSONB), + + :prompt_tokens, + :completion_tokens, + :total_tokens, + :cost, + CAST(:usage AS JSONB), + + :needs_review + ) + RETURNING id::text + """), { + "message_id": message_id, + "raw_event_id": raw_event_id, + "conversation_id": conversation_id, + "contact_id": request.contact_id, + "source_system": request.source or "manual", + + "model": model, + "provider": usage.provider, + "openrouter_generation_id": usage.id, + "decision_source": decision_source, + + "action_decision": _json(action_decision), + "action_result": _json(action_result), + + "prompt_tokens": usage.prompt_tokens, + "completion_tokens": usage.completion_tokens, + "total_tokens": usage.total_tokens, + "cost": usage.cost, + "usage": _json(usage), + + "needs_review": needs_review, + }).fetchone() + + action_run_id = run_row[0] + + if raw_event_id: + conn.execute(text(""" + UPDATE raw_events + SET + message_id = CAST(:message_id AS UUID), + action_run_id = CAST(:action_run_id AS UUID) + WHERE id = CAST(:raw_event_id AS UUID) + """), { + "message_id": message_id, + "action_run_id": action_run_id, + "raw_event_id": raw_event_id, + }) + + return action_run_id, message_id + + +def save_raw_event( + source_system: str, + event_type: str | None, + source_event_id: str | None, + conversation_id: str | None, + contact_id: str | None, + payload: dict, +) -> Dict[str, Any]: + with engine.begin() as conn: + row = conn.execute(text(""" + INSERT INTO raw_events ( + source_system, + event_type, + source_event_id, + conversation_id, + contact_id, + payload + ) + VALUES ( + :source_system, + :event_type, + :source_event_id, + :conversation_id, + :contact_id, + CAST(:payload AS JSONB) + ) + ON CONFLICT (source_system, source_event_id) + WHERE source_event_id IS NOT NULL + DO UPDATE SET + payload = EXCLUDED.payload, + event_type = EXCLUDED.event_type, + conversation_id = COALESCE(EXCLUDED.conversation_id, raw_events.conversation_id), + contact_id = COALESCE(EXCLUDED.contact_id, raw_events.contact_id) + RETURNING + id::text, + processed, + ignored, + message_id::text, + action_run_id::text, + (xmax = 0) AS inserted + """), { + "source_system": source_system, + "event_type": event_type, + "source_event_id": source_event_id, + "conversation_id": conversation_id, + "contact_id": contact_id, + "payload": _json(payload), + }).mappings().first() + + return dict(row or {}) + + +def mark_raw_event_processed( + raw_event_id: str, + action_run_id: str | None = None, + message_id: str | None = None, + ignored: bool = False, + error: str | None = None, +) -> None: + with engine.begin() as conn: + conn.execute(text(""" + UPDATE raw_events + SET + processed = TRUE, + ignored = :ignored, + processing_error = :error, + action_run_id = COALESCE(CAST(:action_run_id AS UUID), action_run_id), + message_id = COALESCE(CAST(:message_id AS UUID), message_id), + processed_at = now() + WHERE id = CAST(:raw_event_id AS UUID) + """), { + "raw_event_id": raw_event_id, + "ignored": ignored, + "error": error, + "action_run_id": _uuid_or_none(action_run_id), + "message_id": _uuid_or_none(message_id), + }) + + +def get_state_for_conversation(conversation_id: str | None) -> CurrentState: + if not conversation_id: + return CurrentState() + + with engine.begin() as conn: + row = conn.execute(text(""" + SELECT + t.action_code, + t.route, + t.status, + t.action, + t.note, + t.created_at + FROM tasks t + WHERE t.conversation_id = :conversation_id + ORDER BY t.created_at DESC + LIMIT 1 + """), { + "conversation_id": conversation_id, + }).mappings().first() + + if not row: + return CurrentState() + + return CurrentState( + last_action_code=row.get("action_code") or "desconhecido", + last_route=row.get("route") or "desconhecido", + last_task_status=row.get("status") or "desconhecido", + metadata={ + "last_action": row.get("action"), + "last_note": row.get("note"), + "last_created_at": str(row.get("created_at")), + }, + ) diff --git a/app/posting_policy.py b/app/posting_policy.py new file mode 100644 index 0000000..c4f4d82 --- /dev/null +++ b/app/posting_policy.py @@ -0,0 +1,23 @@ +from app.schemas import ActionResult + + +def can_post_private_note(action_result: ActionResult) -> bool: + """ + Política de segurança para escrever notas privadas no Chatwoot. + + Só permite publicar notas para ações realmente operacionais. + Bloqueia spam, revisão manual e ações sem necessidade operacional. + """ + if not action_result.safe_to_post: + return False + + if not action_result.action_required: + return False + + if action_result.route in {"spam", "rever"}: + return False + + if not str(action_result.action or "").strip(): + return False + + return True diff --git a/app/preparation_service.py b/app/preparation_service.py new file mode 100644 index 0000000..bdcf9fc --- /dev/null +++ b/app/preparation_service.py @@ -0,0 +1,29 @@ +from __future__ import annotations + +from typing import Any, Dict, Optional + +from app.config import settings + + +def _psycopg_url() -> str: + url = settings.database_url + if url.startswith("postgresql+psycopg://"): + return "postgresql://" + url.split("postgresql+psycopg://", 1)[1] + return url + + +def prepare_task( + *, + task_id: Optional[str] = None, + conversation_id: Optional[str] = None, + prep_type: str = "generic", +) -> Dict[str, Any]: + """Executa a preparação operacional sem subprocess nem paths absolutos.""" + from scripts.prepare_task import run_preparation + + return run_preparation( + task_id=task_id, + conversation_id=conversation_id, + prep_type=prep_type, + database_url=_psycopg_url(), + ) diff --git a/app/preparation_view_model.py b/app/preparation_view_model.py new file mode 100644 index 0000000..83f3f0e --- /dev/null +++ b/app/preparation_view_model.py @@ -0,0 +1,90 @@ +from __future__ import annotations + +from typing import Any, Dict, List, Optional + + +def _as_dict(value: Any) -> Dict[str, Any]: + return value if isinstance(value, dict) else {} + + +def _as_list(value: Any) -> List[Any]: + return value if isinstance(value, list) else [] + + +def _value(value: Any) -> str: + return str(value or "").strip() + + +def _missing_label(field: Any) -> str: + raw = str(field or "").strip() + labels = { + "shipment.recipient_name": "Contacto no local", + "recipient_name": "Contacto no local", + "shipment.recipient_phone": "Telefone", + "recipient_phone": "Telefone", + "produto/equipamento a recolher": "Equipamento a recolher", + "equipment": "Equipamento a recolher", + "motivo/instruções da recolha": "Motivo da recolha", + "reason": "Motivo da recolha", + "data preferida para recolha": "Data preferida", + "preferred_date": "Data preferida", + "billing.tax_id": "NIF", + "billing.billing_address": "Morada fiscal", + "billing.billing_email": "Email faturação", + "shipment.delivery_address": "Morada de entrega", + "shipment.pickup_address": "Morada de recolha", + } + return labels.get(raw, raw.replace("_", " ").replace(".", " · ").strip().capitalize() or "Dado em falta") + + +def build_preparation_view_model(task: Dict[str, Any], preparation: Optional[Dict[str, Any]]) -> Dict[str, Any]: + prep = preparation or {} + extracted = _as_dict(prep.get("extracted_data")) + shipment = _as_dict(extracted.get("shipment")) + customer = _as_dict(extracted.get("customer")) + billing = _as_dict(extracted.get("billing")) + sale = _as_dict(extracted.get("sale")) + + raw_missing = prep.get("missing_fields") or extracted.get("missing_fields") or [] + missing = [{"label": _missing_label(item), "raw": str(item or "")} for item in _as_list(raw_missing)] + + confirmed: List[Dict[str, str]] = [] + for label, value in [ + ("Local de recolha", shipment.get("pickup_address")), + ("Morada de entrega", shipment.get("delivery_address")), + ("Contacto", shipment.get("recipient_name")), + ("Telefone", shipment.get("recipient_phone")), + ("Empresa", customer.get("company") or task.get("customer_name")), + ("Email", customer.get("email") or task.get("customer_email")), + ("NIF", customer.get("tax_id") or billing.get("tax_id")), + ]: + v = _value(value) + if v: + confirmed.append({"label": label, "value": v}) + + products = _as_list(sale.get("products")) + product_names = [str(p.get("name") or p.get("description") or "").strip() for p in products if isinstance(p, dict)] + product_names = [x for x in product_names if x] + if product_names: + confirmed.append({"label": "Produto", "value": ", ".join(product_names[:3])}) + + suggested = _value(prep.get("suggested_reply") or extracted.get("suggested_reply")) + + action = _value(task.get("action")) or "Rever tarefa" + primary_action = "Pedir dados em falta ao cliente" if missing else action + + return { + "status": prep.get("status") or ("missing_data" if missing else "ready"), + "prep_type": prep.get("prep_type") or "generic", + "primary_action": primary_action, + "missing_fields": missing, + "confirmed_fields": confirmed, + "suggested_reply": suggested, + "confidence": prep.get("confidence"), + "technical": { + "customer": customer, + "billing": billing, + "sale": sale, + "shipment": shipment, + }, + } diff --git a/app/product_service.py b/app/product_service.py new file mode 100644 index 0000000..0b653bb --- /dev/null +++ b/app/product_service.py @@ -0,0 +1,533 @@ +import json +import uuid +from decimal import Decimal, InvalidOperation +from typing import Any, Dict, List, Optional + +from sqlalchemy import text + +from app.db import engine + + +_PRODUCT_SCHEMA_READY = False + + +DEFAULT_PRODUCTS = [ + { + "sku": "EV-AC-7KW", + "jasmin_sales_item": "CARREGADOR_MONO_7KW", + "name": "Carregador EV 7.4kW monofásico", + "category": "Carregadores", + "description": "Carregador AC monofásico para moradias e pequenas instalações.", + "default_unit_price": "590.00", + "vat_rate": "23.00", + }, + { + "sku": "EV-AC-22KW", + "jasmin_sales_item": "CARREGADOR_TRIF_22KW", + "name": "Carregador EV 22kW trifásico", + "category": "Carregadores", + "description": "Carregador AC trifásico para empresas, condomínios e instalações com maior potência.", + "default_unit_price": "790.00", + "vat_rate": "23.00", + }, + { + "sku": "CAB-T2-5M", + "jasmin_sales_item": "CABO_VE", + "name": "Cabo Type 2 5m", + "category": "Acessórios", + "description": "Cabo Type 2 para carregamento de veículos elétricos.", + "default_unit_price": "120.00", + "vat_rate": "23.00", + }, + { + "sku": "CAB-T2-7M", + "jasmin_sales_item": "CABO_VE", + "name": "Cabo Type 2 7m", + "category": "Acessórios", + "description": "Cabo Type 2 de 7 metros para carregamento de veículos elétricos.", + "default_unit_price": "150.00", + "vat_rate": "23.00", + }, + { + "sku": "INST-BASIC", + "jasmin_sales_item": "INSTALACAO", + "name": "Instalação básica", + "category": "Serviços", + "description": "Serviço de instalação básica sujeito a validação técnica.", + "default_unit_price": "350.00", + "vat_rate": "23.00", + }, + { + "sku": "SHIP-STD", + "jasmin_sales_item": "TRANSPORTE", + "name": "Transporte nacional", + "category": "Serviços", + "description": "Envio nacional standard.", + "default_unit_price": "15.00", + "vat_rate": "23.00", + }, +] + + +def _json(value: Any) -> str: + return json.dumps(value or {}, ensure_ascii=False, default=str) + + +def _uuid_or_none(value: Optional[str]) -> Optional[str]: + value = str(value or "").strip() + return value or None + + +def _decimal(value: Any, default: str = "0") -> Decimal: + try: + if value is None or str(value).strip() == "": + return Decimal(default) + return Decimal(str(value).replace(",", ".").strip()) + except (InvalidOperation, ValueError): + return Decimal(default) + + +def _money(value: Any) -> str: + return f"{_decimal(value):.2f}" + + +def _bool(value: Any) -> bool: + return str(value or "").lower() in {"1", "true", "yes", "on", "sim", "ativo"} + + +def ensure_product_schema(*, seed: bool = True) -> None: + """Cria catálogo simples de produtos e tabelas de linhas. + + Sem variantes: cada produto/preço configurável fica numa linha em `products`. + As linhas de oportunidade/encomenda copiam nome, SKU e preço no momento para + preservar histórico quando o catálogo for alterado. + """ + global _PRODUCT_SCHEMA_READY + if _PRODUCT_SCHEMA_READY: + return + + with engine.begin() as conn: + conn.execute(text(""" + CREATE TABLE IF NOT EXISTS products ( + id UUID PRIMARY KEY, + sku TEXT UNIQUE NOT NULL, + jasmin_sales_item TEXT, + name TEXT NOT NULL, + category TEXT NOT NULL DEFAULT 'Geral', + description TEXT, + default_unit_price NUMERIC(12,2) NOT NULL DEFAULT 0, + vat_rate NUMERIC(5,2) NOT NULL DEFAULT 23, + active BOOLEAN NOT NULL DEFAULT TRUE, + metadata JSONB NOT NULL DEFAULT '{}'::jsonb, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + """)) + conn.execute(text(""" + CREATE TABLE IF NOT EXISTS opportunity_items ( + id UUID PRIMARY KEY, + opportunity_id UUID NOT NULL REFERENCES opportunities(id) ON DELETE CASCADE, + product_id UUID REFERENCES products(id) ON DELETE SET NULL, + sku TEXT, + jasmin_sales_item TEXT, + product_name TEXT NOT NULL, + description TEXT, + quantity NUMERIC(12,2) NOT NULL DEFAULT 1, + unit_price NUMERIC(12,2) NOT NULL DEFAULT 0, + discount_amount NUMERIC(12,2) NOT NULL DEFAULT 0, + total_price NUMERIC(12,2) NOT NULL DEFAULT 0, + status TEXT NOT NULL DEFAULT 'INTERESTED', + metadata JSONB NOT NULL DEFAULT '{}'::jsonb, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + """)) + conn.execute(text(""" + CREATE TABLE IF NOT EXISTS order_items ( + id UUID PRIMARY KEY, + order_id UUID, + opportunity_id UUID REFERENCES opportunities(id) ON DELETE SET NULL, + opportunity_item_id UUID REFERENCES opportunity_items(id) ON DELETE SET NULL, + product_id UUID REFERENCES products(id) ON DELETE SET NULL, + sku TEXT, + jasmin_sales_item TEXT, + product_name TEXT NOT NULL, + description TEXT, + quantity NUMERIC(12,2) NOT NULL DEFAULT 1, + unit_price NUMERIC(12,2) NOT NULL DEFAULT 0, + total_price NUMERIC(12,2) NOT NULL DEFAULT 0, + fulfillment_status TEXT NOT NULL DEFAULT 'PENDING', + metadata JSONB NOT NULL DEFAULT '{}'::jsonb, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + """)) + for stmt in [ + "ALTER TABLE products ADD COLUMN IF NOT EXISTS jasmin_sales_item TEXT", + "ALTER TABLE opportunity_items ADD COLUMN IF NOT EXISTS jasmin_sales_item TEXT", + "ALTER TABLE order_items ADD COLUMN IF NOT EXISTS jasmin_sales_item TEXT", + ]: + conn.execute(text(stmt)) + + conn.execute(text("CREATE INDEX IF NOT EXISTS idx_products_category ON products(category)")) + conn.execute(text("CREATE INDEX IF NOT EXISTS idx_products_active ON products(active)")) + conn.execute(text("CREATE INDEX IF NOT EXISTS idx_opportunity_items_opp ON opportunity_items(opportunity_id)")) + conn.execute(text("CREATE INDEX IF NOT EXISTS idx_order_items_opp ON order_items(opportunity_id)")) + + if seed: + count = conn.execute(text("SELECT count(*) FROM products")).scalar() or 0 + if int(count) == 0: + for product in DEFAULT_PRODUCTS: + conn.execute(text(""" + INSERT INTO products ( + id, sku, jasmin_sales_item, name, category, description, + default_unit_price, vat_rate, active, metadata + ) VALUES ( + CAST(:id AS UUID), :sku, :jasmin_sales_item, :name, :category, :description, + :default_unit_price, :vat_rate, TRUE, CAST(:metadata AS JSONB) + ) + ON CONFLICT (sku) DO NOTHING + """), { + "id": str(uuid.uuid4()), + "sku": product["sku"], + "jasmin_sales_item": product.get("jasmin_sales_item"), + "name": product["name"], + "category": product["category"], + "description": product["description"], + "default_unit_price": product["default_unit_price"], + "vat_rate": product["vat_rate"], + "metadata": _json({"seed": True}), + }) + + _PRODUCT_SCHEMA_READY = True + + +def list_product_categories() -> List[str]: + ensure_product_schema() + with engine.begin() as conn: + rows = conn.execute(text(""" + SELECT DISTINCT category + FROM products + WHERE COALESCE(category, '') <> '' + ORDER BY category + """)).all() + return [str(row[0]) for row in rows] + + +def list_products( + *, + q: Optional[str] = None, + category: Optional[str] = None, + active: Optional[str] = "true", + limit: int = 300, +) -> List[Dict[str, Any]]: + ensure_product_schema() + filters = [] + params: Dict[str, Any] = {"limit": int(limit)} + if q: + filters.append("(sku ILIKE :q OR COALESCE(jasmin_sales_item, '') ILIKE :q OR name ILIKE :q OR category ILIKE :q OR COALESCE(description, '') ILIKE :q)") + params["q"] = f"%{str(q).strip()}%" + if category and category != "all": + filters.append("category = :category") + params["category"] = category + if active in {"true", "false"}: + filters.append("active = :active") + params["active"] = active == "true" + where_sql = "WHERE " + " AND ".join(filters) if filters else "" + sql = text(f""" + SELECT + id::text, + sku, + jasmin_sales_item, + name, + category, + description, + default_unit_price, + vat_rate, + active, + created_at, + updated_at + FROM products + {where_sql} + ORDER BY active DESC, category ASC, name ASC + LIMIT :limit + """) + with engine.begin() as conn: + rows = conn.execute(sql, params).mappings().all() + return [dict(row) for row in rows] + + +def get_product(product_id: str) -> Optional[Dict[str, Any]]: + ensure_product_schema() + sql = text(""" + SELECT + id::text, + sku, + jasmin_sales_item, + name, + category, + description, + default_unit_price, + vat_rate, + active, + metadata, + created_at, + updated_at + FROM products + WHERE id = CAST(:product_id AS UUID) + LIMIT 1 + """) + with engine.begin() as conn: + row = conn.execute(sql, {"product_id": product_id}).mappings().first() + return dict(row) if row else None + + +def create_product(data: Dict[str, Any]) -> str: + ensure_product_schema() + product_id = str(uuid.uuid4()) + sku = str(data.get("sku") or "").strip().upper() + name = str(data.get("name") or "").strip() + if not sku: + raise ValueError("SKU é obrigatório.") + if not name: + raise ValueError("Nome do produto é obrigatório.") + with engine.begin() as conn: + conn.execute(text(""" + INSERT INTO products ( + id, sku, jasmin_sales_item, name, category, description, + default_unit_price, vat_rate, active, metadata + ) VALUES ( + CAST(:id AS UUID), :sku, :jasmin_sales_item, :name, :category, :description, + :default_unit_price, :vat_rate, :active, CAST(:metadata AS JSONB) + ) + """), { + "id": product_id, + "sku": sku, + "jasmin_sales_item": str(data.get("jasmin_sales_item") or "").strip().upper() or None, + "name": name, + "category": str(data.get("category") or "Geral").strip() or "Geral", + "description": str(data.get("description") or "").strip(), + "default_unit_price": _money(data.get("default_unit_price")), + "vat_rate": _money(data.get("vat_rate", "23")), + "active": _bool(data.get("active", "true")), + "metadata": _json({}), + }) + return product_id + + +def update_product(product_id: str, data: Dict[str, Any]) -> bool: + ensure_product_schema() + sku = str(data.get("sku") or "").strip().upper() + name = str(data.get("name") or "").strip() + if not sku: + raise ValueError("SKU é obrigatório.") + if not name: + raise ValueError("Nome do produto é obrigatório.") + with engine.begin() as conn: + result = conn.execute(text(""" + UPDATE products + SET sku = :sku, + jasmin_sales_item = :jasmin_sales_item, + name = :name, + category = :category, + description = :description, + default_unit_price = :default_unit_price, + vat_rate = :vat_rate, + active = :active, + updated_at = now() + WHERE id = CAST(:product_id AS UUID) + """), { + "product_id": product_id, + "sku": sku, + "jasmin_sales_item": str(data.get("jasmin_sales_item") or "").strip().upper() or None, + "name": name, + "category": str(data.get("category") or "Geral").strip() or "Geral", + "description": str(data.get("description") or "").strip(), + "default_unit_price": _money(data.get("default_unit_price")), + "vat_rate": _money(data.get("vat_rate", "23")), + "active": _bool(data.get("active")), + }) + return result.rowcount > 0 + + +def set_product_active(product_id: str, active: bool) -> bool: + ensure_product_schema() + with engine.begin() as conn: + result = conn.execute(text(""" + UPDATE products + SET active = :active, + updated_at = now() + WHERE id = CAST(:product_id AS UUID) + """), {"product_id": product_id, "active": bool(active)}) + return result.rowcount > 0 + + +def list_opportunity_items(opportunity_id: str) -> List[Dict[str, Any]]: + ensure_product_schema() + sql = text(""" + SELECT + oi.id::text, + oi.opportunity_id::text, + oi.product_id::text, + oi.sku, + COALESCE(NULLIF(oi.jasmin_sales_item, ''), p.jasmin_sales_item) AS jasmin_sales_item, + oi.product_name, + oi.description, + oi.quantity, + oi.unit_price, + oi.discount_amount, + oi.total_price, + oi.status, + oi.created_at, + oi.updated_at, + p.active AS product_active + FROM opportunity_items oi + LEFT JOIN products p ON p.id = oi.product_id + WHERE oi.opportunity_id = CAST(:opportunity_id AS UUID) + ORDER BY oi.created_at ASC + """) + with engine.begin() as conn: + rows = conn.execute(sql, {"opportunity_id": opportunity_id}).mappings().all() + return [dict(row) for row in rows] + + +def _recalculate_opportunity_value(conn, opportunity_id: str) -> None: + conn.execute(text(""" + UPDATE opportunities + SET value_amount = COALESCE(( + SELECT SUM(total_price) + FROM opportunity_items + WHERE opportunity_id = CAST(:opportunity_id AS UUID) + AND status NOT IN ('REJECTED', 'CANCELLED') + ), 0), + updated_at = now() + WHERE id = CAST(:opportunity_id AS UUID) + """), {"opportunity_id": opportunity_id}) + + +def add_opportunity_item( + opportunity_id: str, + *, + product_id: Optional[str] = None, + product_name: Optional[str] = None, + sku: Optional[str] = None, + jasmin_sales_item: Optional[str] = None, + description: Optional[str] = None, + quantity: Any = "1", + unit_price: Any = None, + discount_amount: Any = "0", + status: str = "INTERESTED", +) -> str: + ensure_product_schema() + item_id = str(uuid.uuid4()) + product: Optional[Dict[str, Any]] = get_product(product_id) if product_id else None + name = str(product_name or (product or {}).get("name") or "").strip() + if not name: + raise ValueError("Produto é obrigatório.") + q = _decimal(quantity, "1") + price = _decimal(unit_price if unit_price not in {None, ""} else (product or {}).get("default_unit_price"), "0") + discount = _decimal(discount_amount, "0") + total = max(Decimal("0"), (q * price) - discount) + normalized_status = str(status or "INTERESTED").strip().upper() + + with engine.begin() as conn: + conn.execute(text(""" + INSERT INTO opportunity_items ( + id, opportunity_id, product_id, sku, jasmin_sales_item, product_name, description, + quantity, unit_price, discount_amount, total_price, status, metadata + ) VALUES ( + CAST(:id AS UUID), CAST(:opportunity_id AS UUID), CAST(:product_id AS UUID), + :sku, :jasmin_sales_item, :product_name, :description, + :quantity, :unit_price, :discount_amount, :total_price, :status, CAST(:metadata AS JSONB) + ) + """), { + "id": item_id, + "opportunity_id": opportunity_id, + "product_id": _uuid_or_none(product_id), + "sku": sku or (product or {}).get("sku") or "", + "jasmin_sales_item": (jasmin_sales_item or (product or {}).get("jasmin_sales_item") or "").strip().upper() or None, + "product_name": name, + "description": description if description is not None else (product or {}).get("description") or "", + "quantity": _money(q), + "unit_price": _money(price), + "discount_amount": _money(discount), + "total_price": _money(total), + "status": normalized_status, + "metadata": _json({"source": "admin", "jasmin_sales_item": (jasmin_sales_item or (product or {}).get("jasmin_sales_item") or "").strip().upper()}), + }) + _recalculate_opportunity_value(conn, opportunity_id) + conn.execute(text(""" + INSERT INTO opportunity_events ( + id, opportunity_id, event_type, note, payload, created_by + ) VALUES ( + CAST(:id AS UUID), CAST(:opportunity_id AS UUID), 'opportunity_item_added', + :note, CAST(:payload AS JSONB), 'operator' + ) + """), { + "id": str(uuid.uuid4()), + "opportunity_id": opportunity_id, + "note": f"Produto adicionado: {name} x {_money(q)}", + "payload": _json({"item_id": item_id, "sku": sku or (product or {}).get("sku") or "", "jasmin_sales_item": (jasmin_sales_item or (product or {}).get("jasmin_sales_item") or "").strip().upper()}), + }) + return item_id + + +def update_opportunity_item(item_id: str, data: Dict[str, Any]) -> Optional[str]: + ensure_product_schema() + q = _decimal(data.get("quantity"), "1") + price = _decimal(data.get("unit_price"), "0") + discount = _decimal(data.get("discount_amount"), "0") + total = max(Decimal("0"), (q * price) - discount) + status = str(data.get("status") or "INTERESTED").strip().upper() + with engine.begin() as conn: + row = conn.execute(text(""" + UPDATE opportunity_items + SET jasmin_sales_item = COALESCE(NULLIF(:jasmin_sales_item, ''), jasmin_sales_item), + quantity = :quantity, + unit_price = :unit_price, + discount_amount = :discount_amount, + total_price = :total_price, + status = :status, + updated_at = now() + WHERE id = CAST(:item_id AS UUID) + RETURNING opportunity_id::text + """), { + "item_id": item_id, + "jasmin_sales_item": str(data.get("jasmin_sales_item") or "").strip().upper(), + "quantity": _money(q), + "unit_price": _money(price), + "discount_amount": _money(discount), + "total_price": _money(total), + "status": status, + }).first() + if row: + _recalculate_opportunity_value(conn, str(row[0])) + return str(row[0]) + return None + + +def delete_opportunity_item(item_id: str) -> Optional[str]: + ensure_product_schema() + with engine.begin() as conn: + row = conn.execute(text(""" + DELETE FROM opportunity_items + WHERE id = CAST(:item_id AS UUID) + RETURNING opportunity_id::text, product_name + """), {"item_id": item_id}).first() + if row: + opportunity_id = str(row[0]) + _recalculate_opportunity_value(conn, opportunity_id) + conn.execute(text(""" + INSERT INTO opportunity_events ( + id, opportunity_id, event_type, note, payload, created_by + ) VALUES ( + CAST(:id AS UUID), CAST(:opportunity_id AS UUID), 'opportunity_item_deleted', + :note, '{}'::jsonb, 'operator' + ) + """), { + "id": str(uuid.uuid4()), + "opportunity_id": opportunity_id, + "note": f"Produto removido: {row[1] or 'produto'}", + }) + return opportunity_id + return None diff --git a/app/prompt.py b/app/prompt.py new file mode 100644 index 0000000..fe5b5cd --- /dev/null +++ b/app/prompt.py @@ -0,0 +1,82 @@ +import json + +from app.schemas import AnalyzeRequest + + +def build_prompt(request: AnalyzeRequest) -> str: + state_json = request.current_state.model_dump_json(indent=2) + + source_line = f"\nORIGEM DO EVENTO:\n{request.source}\n" if request.source else "" + + return f""" +Analisa esta conversa real de cliente B2B. A empresa vende carregadores EV AC / wallbox. + +IMPORTANTE: +- A ÚLTIMA MENSAGEM DO CLIENTE é a mensagem principal a classificar. +- Usa o HISTÓRICO ANTERIOR apenas como contexto. +- Não trates mensagens anteriores da empresa como se fossem a ação final. +- Se a última mensagem é do cliente e contém pedido, pergunta, solicitação, dúvida ou confirmação, who_has_the_ball = empresa e conversation_state = a_aguardar_empresa. + +{source_line} +ÚLTIMA MENSAGEM DO CLIENTE: +{request.last_customer_message} + +HISTÓRICO ANTERIOR / CONTEXTO: +{request.previous_context} + +ESTADO ATUAL CONHECIDO ANTES DA ÚLTIMA MENSAGEM: +{state_json} + +Devolve JSON com estes campos: +{{ + "case_type": "", + "intent": "", + "product_interest": "", + "conversation_state": "", + "commercial_state": "", + "relationship_state": "", + "next_action": "", + "who_has_the_ball": "", + "urgency": "", + "risk": "", + "summary": "", + "evidence": [], + "confidence": 0.0 +}} + +Valores permitidos, usar exatamente estes valores: + +case_type: +comercial, marketing, suporte, financeiro, outro + +intent: +pedido_informacao, pedido_preco, pedido_contacto, pedido_proposta, pedido_orcamento, pedido_ajuste_proposta, followup_proposta, pedido_encomenda, pedido_proforma, pedido_fatura, confirmacao_adjudicacao, pedido_confirmacao_tecnica, comprovativo_pagamento, negociacao_preco, sem_interesse, remover_lista, suporte, resposta_ambigua, outro + +conversation_state: +novo, a_aguardar_empresa, a_aguardar_cliente, resolvida, precisa_revisao + +commercial_state: +novo_lead, interessado, qualificacao, proposta_pedida, proposta_enviada, negociacao, encomenda_pendente, proforma_pedida, pagamento_recebido, preparar_envio, cliente_ativo, sem_interesse, perdido, ganho, desconhecido + +relationship_state: +novo_contacto, lead_existente, cliente_ativo, ex_cliente, desconhecido + +who_has_the_ball: +empresa, cliente, ambos, desconhecido + +urgency: +baixa, media, alta + +risk: +baixo, medio, alto + +Regras: +- Usa exatamente os valores permitidos. Não uses acentos, sinónimos ou variações. +- Para urgency usa "media", nunca "média". +- Para risk usa "medio", nunca "médio". +- next_action deve ser uma frase curta em português, nunca enum nem código. +- evidence deve conter frases curtas retiradas da conversa. +- summary deve ser curto e útil para o comercial. +- Não inventes informação. +- Responde apenas JSON válido, sem markdown. +""".strip() diff --git a/app/reconciliation_decision_service.py b/app/reconciliation_decision_service.py new file mode 100644 index 0000000..d4213be --- /dev/null +++ b/app/reconciliation_decision_service.py @@ -0,0 +1,181 @@ +"""Read-only reconciliation decision helpers for the operator UI. + +v4.9.28 keeps reconciliation decisions explicit without adding hard database +constraints. The service classifies loose evidence into operational buckets so +UI code can show the safest next decision: link, create, review, historical or +ignore. +""" +from __future__ import annotations + +from decimal import Decimal, InvalidOperation +from typing import Any, Dict, Iterable, List + + +OPEN_STATUSES = {"open", "needs_review", "conflict"} +RESOLVED_STATUSES = {"linked", "resolved"} + + +def _to_decimal(value: Any) -> Decimal | None: + if value is None or value == "": + return None + try: + return Decimal(str(value)) + except (InvalidOperation, ValueError): + return None + + +def _confidence_value(value: Any) -> Decimal | None: + confidence = _to_decimal(value) + if confidence is None: + return None + if confidence > 1: + confidence = confidence / Decimal("100") + return confidence + + +def classify_reconciliation_item(item: Dict[str, Any]) -> Dict[str, Any]: + """Classify one reconciliation item into a human decision bucket.""" + status = str(item.get("status") or "open").strip().lower() + external_type = str(item.get("external_type") or "external_record") + suggestions = item.get("operation_suggestions") or [] + priority = str(item.get("priority") or "normal").lower() + confidence = _confidence_value(item.get("confidence")) + has_suggestion = bool(suggestions) + has_opportunity = bool(item.get("opportunity_id")) + + if status in RESOLVED_STATUSES: + return { + "bucket": "resolved", + "label": "Resolvido", + "description": "Já foi ligado ou fechado.", + "primary_decision": "Sem ação", + "chip_class": "cf-chip-green", + } + if status == "historical": + return { + "bucket": "historical", + "label": "Histórico", + "description": "Documento mantido para auditoria, sem ação operacional.", + "primary_decision": "Sem ação operacional", + "chip_class": "cf-chip-gray", + } + if status == "ignored": + return { + "bucket": "ignored", + "label": "Ignorado", + "description": "Retirado da fila operacional.", + "primary_decision": "Sem ação", + "chip_class": "cf-chip-gray", + } + if status == "conflict": + return { + "bucket": "review", + "label": "Conflito", + "description": "Há sinais contraditórios; requer revisão manual antes de ligar/criar.", + "primary_decision": "Rever manualmente", + "chip_class": "cf-chip-red", + } + if status == "needs_review": + return { + "bucket": "review", + "label": "Revisão", + "description": "Ainda não há confiança suficiente para aplicar uma decisão direta.", + "primary_decision": "Rever manualmente", + "chip_class": "cf-chip-orange", + } + + high_confidence = confidence is not None and confidence >= Decimal("0.75") + important_evidence = external_type in {"jasmin_invoice", "jasmin_proforma", "payment_proof", "odoo_sale_order"} + + if has_suggestion: + return { + "bucket": "actionable", + "label": "Ação recomendada", + "description": "Existe oportunidade sugerida para ligação.", + "primary_decision": "Ligar à oportunidade sugerida", + "chip_class": "cf-chip-blue", + } + if has_opportunity: + return { + "bucket": "actionable", + "label": "Confirmar ligação", + "description": "O item já tem oportunidade associada, mas ainda está em aberto.", + "primary_decision": "Confirmar documento", + "chip_class": "cf-chip-blue", + } + if priority in {"alta", "high"} or high_confidence or important_evidence: + return { + "bucket": "actionable", + "label": "Decidir agora", + "description": "Evidência recente/importante; criar oportunidade ou ligar manualmente.", + "primary_decision": "Criar ou ligar oportunidade", + "chip_class": "cf-chip-blue", + } + return { + "bucket": "review", + "label": "Revisão", + "description": "Faltam sinais fortes para ação direta.", + "primary_decision": "Rever manualmente", + "chip_class": "cf-chip-orange", + } + + +def classify_reconciliation_process(candidate: Dict[str, Any]) -> Dict[str, Any]: + """Classify a reconstructed process candidate.""" + review_status = str(candidate.get("review_status") or "needs_review").lower() + suggestions = candidate.get("suggestions") or [] + confidence = str(candidate.get("confidence") or "média").lower() + risks = candidate.get("risks") or [] + + if review_status == "conflict" or risks: + return { + "bucket": "review", + "label": "Requer revisão", + "description": "Processo com risco de associação errada.", + "primary_decision": "Rever antes de aplicar", + "chip_class": "cf-chip-orange" if review_status != "conflict" else "cf-chip-red", + } + if review_status == "ready" or confidence == "alta" or suggestions: + return { + "bucket": "actionable", + "label": "Ação recomendada", + "description": "Processo reconstruído com evidência suficiente para decisão explícita.", + "primary_decision": "Ligar ou criar oportunidade", + "chip_class": "cf-chip-blue", + } + return { + "bucket": "review", + "label": "Requer revisão", + "description": "Processo reconstruído, mas ainda sem confiança operacional alta.", + "primary_decision": "Rever manualmente", + "chip_class": "cf-chip-orange", + } + + +def reconciliation_decision_summary(items: Iterable[Dict[str, Any]], candidates: Iterable[Dict[str, Any]]) -> Dict[str, int]: + summary = {"actionable": 0, "review": 0, "historical": 0, "ignored": 0, "resolved": 0} + for item in items: + decision = classify_reconciliation_item(item) + bucket = decision.get("bucket") or "review" + summary[bucket] = summary.get(bucket, 0) + 1 + for candidate in candidates: + decision = classify_reconciliation_process(candidate) + bucket = decision.get("bucket") or "review" + summary[bucket] = summary.get(bucket, 0) + 1 + return summary + + +def sort_items_for_operator(items: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """Show actionable evidence before review/noise while preserving status filters.""" + order = {"actionable": 0, "review": 1, "historical": 2, "ignored": 3, "resolved": 4} + priority_order = {"alta": 0, "high": 0, "normal": 1, "baixa": 2, "low": 2} + + def key(item: Dict[str, Any]): + decision = classify_reconciliation_item(item) + return ( + order.get(str(decision.get("bucket") or "review"), 9), + priority_order.get(str(item.get("priority") or "normal").lower(), 5), + str(item.get("updated_at") or item.get("created_at") or ""), + ) + + return sorted(items, key=key) diff --git a/app/reconciliation_service.py b/app/reconciliation_service.py new file mode 100644 index 0000000..0731723 --- /dev/null +++ b/app/reconciliation_service.py @@ -0,0 +1,3254 @@ +"""Operational reconciliation and external intake service. + +v4.9.1 introduces a controlled staging area for information that exists +outside ClientFlow: Jasmin documents, Odoo sales, manual WhatsApp/phone/email +requests and payment proofs. The service deliberately creates reconciliation +candidates first; critical process changes still need operator confirmation. +""" +from __future__ import annotations + +import json +import uuid +import logging +import re +import unicodedata +from datetime import datetime +from decimal import Decimal, InvalidOperation +from typing import Any, Dict, List, Optional, Tuple + +from sqlalchemy import text + +from app.action_catalog import get_action_config +from app.db import engine + +_SCHEMA_READY = False +logger = logging.getLogger(__name__) + + +RECONCILIATION_OPEN_STATUSES = {"open", "needs_review", "conflict"} +DOCUMENT_EXTERNAL_TYPES = {"jasmin_quotation", "jasmin_invoice", "jasmin_proforma", "odoo_sale_order", "packlink_shipment"} +PAYMENT_EXTERNAL_TYPES = {"payment_proof", "payment_receipt"} + + +STAGE_BY_EXTERNAL_TYPE = { + "jasmin_quotation": "QUOTE_SENT", + "jasmin_proforma": "WAITING_PAYMENT", + "jasmin_invoice": "INVOICE_SENT", + "odoo_sale_order": "ODOO_ORDER_CREATED", + "packlink_shipment": "SHIPMENT_CREATED", + "payment_proof": "WAITING_PAYMENT", + "manual_request": "QUOTE_REQUESTED", +} + +NEXT_ACTION_BY_EXTERNAL_TYPE = { + "jasmin_quotation": "SEND_PROFORMA", + "jasmin_proforma": "CONFIRM_PAYMENT", + "jasmin_invoice": "CONFIRM_PAYMENT", + "odoo_sale_order": "SEND_INVOICE", + "packlink_shipment": "REVIEW_MANUALLY", + "payment_proof": "CONFIRM_PAYMENT", + "manual_request": "SEND_QUOTE", +} + + +def _json(value: Any) -> str: + return json.dumps(value or {}, ensure_ascii=False, default=str) + + +def _clean(value: Any) -> str: + return str(value or "").strip() + + +def _uuid_or_none(value: Any) -> Optional[str]: + value = _clean(value) + return value or None + + +def _money_or_none(value: Any) -> Optional[str]: + raw = _clean(value).replace("€", "").replace(" ", "").replace(",", ".") + if not raw: + return None + try: + return str(Decimal(raw).quantize(Decimal("0.01"))) + except (InvalidOperation, ValueError): + return None + + +def _normalize_tax_id(value: Any) -> str: + """Normalize NIF/VAT values for exact matching across external systems.""" + raw = _clean(value).upper().replace(" ", "").replace("-", "").replace(".", "") + if raw.startswith("PT"): + raw = raw[2:] + return raw + + +def _strip_accents(value: Any) -> str: + text_value = unicodedata.normalize("NFKD", _clean(value)) + return "".join(ch for ch in text_value if not unicodedata.combining(ch)) + + +def _normalize_company_name(value: Any) -> str: + """Normalize fiscal/company names for identity matching. + + Business rule: company fiscal names and NIFs are unique. Therefore an + exact/contained normalized company name is a strong identity signal when a + source omits NIF (common in Odoo). Weak token overlap is still treated only + as a helper for ranking, never as a process bridge. + """ + text_value = _strip_accents(value).upper() + text_value = re.sub(r"[^A-Z0-9]+", " ", text_value) + stop_words = { + "LDA", "LTDA", "LIMITADA", "UNIPESSOAL", "SA", "S", "A", + "SOCIEDADE", "COMERCIAL", "PORTUGAL", "PT", "THE", "COMPANY", + } + tokens = [tok for tok in text_value.split() if tok and tok not in stop_words] + return " ".join(tokens) + + +def _name_match_score(external_name: Any, *candidate_names: Any) -> int: + external_key = _normalize_company_name(external_name) + if not external_key or len(external_key) < 3: + return 0 + external_tokens = set(external_key.split()) + best = 0 + for candidate in candidate_names: + candidate_key = _normalize_company_name(candidate) + if not candidate_key or len(candidate_key) < 3: + continue + if candidate_key == external_key: + best = max(best, 75) + continue + if external_key in candidate_key or candidate_key in external_key: + best = max(best, 60) + continue + candidate_tokens = set(candidate_key.split()) + if external_tokens and candidate_tokens: + overlap = len(external_tokens & candidate_tokens) + ratio = overlap / max(len(external_tokens), len(candidate_tokens)) + if overlap >= 2 and ratio >= 0.50: + best = max(best, 30) + elif overlap >= 1 and ratio >= 0.50: + best = max(best, 20) + return best + + +def _looks_like_company_fiscal_name(value: Any) -> bool: + """Return true when a name has company/legal-name signals. + + Company names are unique in this project, but not every free-text name is a + company fiscal name. This keeps personal contacts such as "Bruno Oliveira" + out of the high-confidence path while allowing "ACZCO BRAGA ENERGY, LDA" + or Odoo display names that include the fiscal name plus location suffix. + """ + text_value = _strip_accents(value).upper() + tokens = set(re.sub(r"[^A-Z0-9]+", " ", text_value).split()) + company_tokens = { + "LDA", "LTDA", "LIMITADA", "UNIPESSOAL", "SA", "S", "A", + "SOCIEDADE", "COMERCIAL", "EMPRESA", "ENERGY", "SOLUTIONS", + } + if tokens & company_tokens: + return True + # Typical official/commercial company names often have three or more + # non-address tokens even when the legal suffix was omitted by the source. + normalized = _normalize_company_name(value) + return len(normalized.split()) >= 3 + + +def _process_group_identity_label(match_key: str, items: List[Dict[str, Any]]) -> Tuple[str, str]: + """Return UI confidence + explanation for a reconstructed process group. + + NIF remains absolute. Company fiscal name is also high-confidence by the + project rule that company names are unique; personal/free-text names stay at + medium confidence. + """ + if match_key == "nif": + has_name_bridge = any( + not _normalize_tax_id(item.get("customer_tax_id") or "") + and _looks_like_company_fiscal_name(item.get("customer_name") or item.get("linked_customer_name")) + for item in items + ) + if has_name_bridge: + return "alta", "NIF + nome fiscal" + return "alta", "NIF exato" + if match_key == "email": + return "média", "email" + if match_key == "name": + if any(_looks_like_company_fiscal_name(item.get("customer_name") or item.get("linked_customer_name")) for item in items): + return "alta", "nome fiscal" + return "média", "nome" + return "média", match_key or "identidade" + + +def recent_window_start(days: int = 3) -> str: + """Inclusive start date for a short operational reconciliation window. + + "Últimos 3 dias" means today and the previous two calendar days. + This keeps Reconciliação as a compact work queue instead of a historical import. + """ + from datetime import datetime, timedelta, timezone + + days = max(int(days or 1), 1) + return (datetime.now(timezone.utc).date() - timedelta(days=days - 1)).isoformat() + + +def _priority_for_action(action_code: str, *, fallback: str = "normal") -> str: + code = str(action_code or "").upper() + if code in {"SEND_PROFORMA", "SEND_INVOICE", "CONFIRM_PAYMENT", "PREPARE_ORDER", "CREATE_SHIPMENT"}: + return "alta" + if code in {"REVIEW_MANUALLY", "REMOVE_FROM_LIST", "MARK_NO_INTEREST", "IGNORE_SPAM", "NO_ACTION", "IGNORE_BOUNCE"}: + return "baixa" + return fallback + + +def ensure_reconciliation_schema() -> None: + """Create additive reconciliation tables. + + This is intentionally not a destructive migration: it only creates tables, + indexes and optional columns that allow ClientFlow to stage external data + before an operator links or turns it into a commercial process. + """ + global _SCHEMA_READY + if _SCHEMA_READY: + return + + with engine.begin() as conn: + conn.execute(text(""" + CREATE TABLE IF NOT EXISTS reconciliation_items ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + source_system TEXT NOT NULL, + external_type TEXT NOT NULL, + external_id TEXT, + title TEXT NOT NULL, + description TEXT, + status TEXT NOT NULL DEFAULT 'open', + priority TEXT NOT NULL DEFAULT 'normal', + suggested_action TEXT, + confidence NUMERIC(4,3), + opportunity_id UUID REFERENCES opportunities(id) ON DELETE SET NULL, + customer_id UUID REFERENCES customers(id) ON DELETE SET NULL, + customer_name TEXT, + customer_email TEXT, + customer_tax_id TEXT, + document_number TEXT, + document_date DATE, + amount NUMERIC(12,2), + currency TEXT NOT NULL DEFAULT 'EUR', + payload JSONB NOT NULL DEFAULT '{}'::jsonb, + resolution_note TEXT, + idempotency_key TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + resolved_at TIMESTAMPTZ + ) + """)) + for stmt in [ + "ALTER TABLE reconciliation_items ADD COLUMN IF NOT EXISTS suggested_action TEXT", + "ALTER TABLE reconciliation_items ADD COLUMN IF NOT EXISTS confidence NUMERIC(4,3)", + "ALTER TABLE reconciliation_items ADD COLUMN IF NOT EXISTS resolution_note TEXT", + "ALTER TABLE reconciliation_items ADD COLUMN IF NOT EXISTS idempotency_key TEXT", + "ALTER TABLE reconciliation_items ADD COLUMN IF NOT EXISTS customer_tax_id TEXT", + ]: + conn.execute(text(stmt)) + conn.execute(text(""" + CREATE UNIQUE INDEX IF NOT EXISTS ux_reconciliation_items_source_external + ON reconciliation_items(source_system, external_type, external_id) + WHERE external_id IS NOT NULL AND external_id <> '' + """)) + conn.execute(text(""" + CREATE UNIQUE INDEX IF NOT EXISTS ux_reconciliation_items_idempotency_key + ON reconciliation_items(idempotency_key) + WHERE idempotency_key IS NOT NULL AND idempotency_key <> '' + """)) + conn.execute(text("CREATE INDEX IF NOT EXISTS idx_reconciliation_status ON reconciliation_items(status, created_at DESC)")) + conn.execute(text("CREATE INDEX IF NOT EXISTS idx_reconciliation_type ON reconciliation_items(external_type, status)")) + conn.execute(text("CREATE INDEX IF NOT EXISTS idx_reconciliation_opportunity ON reconciliation_items(opportunity_id)")) + conn.execute(text("CREATE INDEX IF NOT EXISTS idx_reconciliation_customer_tax_id ON reconciliation_items(customer_tax_id) WHERE customer_tax_id IS NOT NULL AND customer_tax_id <> ''")) + + conn.execute(text(""" + CREATE TABLE IF NOT EXISTS operation_links ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + opportunity_id UUID NOT NULL REFERENCES opportunities(id) ON DELETE CASCADE, + system TEXT NOT NULL, + external_type TEXT NOT NULL, + external_id TEXT, + external_name TEXT, + external_url TEXT, + status TEXT NOT NULL DEFAULT 'pending', + payload JSONB NOT NULL DEFAULT '{}'::jsonb, + last_synced_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + """)) + conn.execute(text("CREATE UNIQUE INDEX IF NOT EXISTS ux_operation_links_key ON operation_links(opportunity_id, system, external_type)")) + conn.execute(text("CREATE INDEX IF NOT EXISTS idx_operation_links_external_id ON operation_links(opportunity_id, system, external_type, external_id)")) + + conn.execute(text(""" + CREATE TABLE IF NOT EXISTS external_customer_mappings ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + source_system TEXT NOT NULL, + external_party_id TEXT NOT NULL, + external_party_name TEXT, + customer_id UUID REFERENCES customers(id) ON DELETE CASCADE, + confidence TEXT NOT NULL DEFAULT 'confirmed', + created_by TEXT NOT NULL DEFAULT 'operator', + payload JSONB NOT NULL DEFAULT '{}'::jsonb, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + """)) + conn.execute(text(""" + CREATE UNIQUE INDEX IF NOT EXISTS ux_external_customer_mappings_source_party + ON external_customer_mappings(source_system, external_party_id) + """)) + conn.execute(text("CREATE INDEX IF NOT EXISTS idx_external_customer_mappings_customer ON external_customer_mappings(customer_id)")) + + # v4.9.24: store operator decisions so rebuilds do not erase manual + # knowledge about merge/split/link/ignore/history choices. + conn.execute(text(""" + CREATE TABLE IF NOT EXISTS reconciliation_decisions ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + decision_type TEXT NOT NULL, + status TEXT, + item_ids TEXT[] NOT NULL DEFAULT ARRAY[]::TEXT[], + opportunity_id UUID REFERENCES opportunities(id) ON DELETE SET NULL, + customer_id UUID REFERENCES customers(id) ON DELETE SET NULL, + note TEXT, + actor TEXT NOT NULL DEFAULT 'operator', + payload JSONB NOT NULL DEFAULT '{}'::jsonb, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + """)) + conn.execute(text("CREATE INDEX IF NOT EXISTS idx_reconciliation_decisions_type ON reconciliation_decisions(decision_type, created_at DESC)")) + conn.execute(text("CREATE INDEX IF NOT EXISTS idx_reconciliation_decisions_opportunity ON reconciliation_decisions(opportunity_id)")) + + # v4.9.13: reconstructed Odoo processes can import the commercial + # lines into the opportunity. Keep the schema guard additive so older + # installs where the product module has not run yet still work. + conn.execute(text(""" + CREATE TABLE IF NOT EXISTS opportunity_items ( + id UUID PRIMARY KEY, + opportunity_id UUID NOT NULL REFERENCES opportunities(id) ON DELETE CASCADE, + product_id UUID, + sku TEXT, + jasmin_sales_item TEXT, + product_name TEXT NOT NULL, + description TEXT, + quantity NUMERIC(12,2) NOT NULL DEFAULT 1, + unit_price NUMERIC(12,2) NOT NULL DEFAULT 0, + discount_amount NUMERIC(12,2) NOT NULL DEFAULT 0, + total_price NUMERIC(12,2) NOT NULL DEFAULT 0, + status TEXT NOT NULL DEFAULT 'INTERESTED', + metadata JSONB NOT NULL DEFAULT '{}'::jsonb, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + """)) + for stmt in [ + "ALTER TABLE opportunity_items ADD COLUMN IF NOT EXISTS metadata JSONB NOT NULL DEFAULT '{}'::jsonb", + "ALTER TABLE opportunity_items ADD COLUMN IF NOT EXISTS status TEXT NOT NULL DEFAULT 'INTERESTED'", + "CREATE INDEX IF NOT EXISTS idx_opportunity_items_opp ON opportunity_items(opportunity_id)", + ]: + conn.execute(text(stmt)) + + conn.execute(text(""" + CREATE TABLE IF NOT EXISTS payment_proofs ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + opportunity_id UUID REFERENCES opportunities(id) ON DELETE SET NULL, + customer_id UUID REFERENCES customers(id) ON DELETE SET NULL, + reconciliation_item_id UUID REFERENCES reconciliation_items(id) ON DELETE SET NULL, + source_system TEXT NOT NULL DEFAULT 'manual', + source_ref TEXT, + filename TEXT, + file_url TEXT, + amount NUMERIC(12,2), + currency TEXT NOT NULL DEFAULT 'EUR', + proof_date DATE, + status TEXT NOT NULL DEFAULT 'pending_validation', + note TEXT, + payload JSONB NOT NULL DEFAULT '{}'::jsonb, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + validated_at TIMESTAMPTZ, + validated_by TEXT + ) + """)) + conn.execute(text("CREATE INDEX IF NOT EXISTS idx_payment_proofs_status ON payment_proofs(status, created_at DESC)")) + conn.execute(text("CREATE INDEX IF NOT EXISTS idx_payment_proofs_opportunity ON payment_proofs(opportunity_id)")) + + # v4.9.7: keep the reconciliation page resilient on installs where + # the commercial customer/opportunity link migration has not run yet. + # These are additive guards only; they do not change existing data. + conn.execute(text(""" + CREATE TABLE IF NOT EXISTS customers ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + name TEXT NOT NULL DEFAULT '', + tax_id TEXT, + email TEXT, + phone TEXT, + street_name TEXT, + postal_zone TEXT, + city_name TEXT, + country TEXT NOT NULL DEFAULT 'PT', + jasmin_customer_party_key TEXT, + jasmin_customer_id TEXT, + metadata JSONB NOT NULL DEFAULT '{}'::jsonb, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + """)) + for stmt in [ + "ALTER TABLE customers ADD COLUMN IF NOT EXISTS tax_id TEXT", + "ALTER TABLE customers ADD COLUMN IF NOT EXISTS email TEXT", + "ALTER TABLE opportunities ADD COLUMN IF NOT EXISTS local_customer_id UUID REFERENCES customers(id) ON DELETE SET NULL", + # v4.9.14: the process-link action touches several columns that may + # be missing on long-lived installs where the table was created by + # an older ClientFlow version. Keep all guards additive. + "ALTER TABLE opportunities ADD COLUMN IF NOT EXISTS stage TEXT NOT NULL DEFAULT 'NEW_LEAD'", + "ALTER TABLE opportunities ADD COLUMN IF NOT EXISTS status TEXT NOT NULL DEFAULT 'open'", + "ALTER TABLE opportunities ADD COLUMN IF NOT EXISTS value_amount NUMERIC(12,2)", + "ALTER TABLE opportunities ADD COLUMN IF NOT EXISTS currency TEXT NOT NULL DEFAULT 'EUR'", + "ALTER TABLE opportunities ADD COLUMN IF NOT EXISTS last_action_code TEXT", + "ALTER TABLE opportunities ADD COLUMN IF NOT EXISTS metadata JSONB NOT NULL DEFAULT '{}'::jsonb", + "ALTER TABLE opportunities ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ NOT NULL DEFAULT now()", + "ALTER TABLE operation_links ADD COLUMN IF NOT EXISTS external_id TEXT", + "ALTER TABLE operation_links ADD COLUMN IF NOT EXISTS external_name TEXT", + "ALTER TABLE operation_links ADD COLUMN IF NOT EXISTS external_url TEXT", + "ALTER TABLE operation_links ADD COLUMN IF NOT EXISTS payload JSONB NOT NULL DEFAULT '{}'::jsonb", + "ALTER TABLE operation_links ADD COLUMN IF NOT EXISTS last_synced_at TIMESTAMPTZ", + "ALTER TABLE operation_links ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ NOT NULL DEFAULT now()", + "ALTER TABLE operation_links ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ NOT NULL DEFAULT now()", + "ALTER TABLE opportunity_events ADD COLUMN IF NOT EXISTS task_id UUID", + "ALTER TABLE opportunity_events ADD COLUMN IF NOT EXISTS action_code TEXT", + "ALTER TABLE opportunity_events ADD COLUMN IF NOT EXISTS from_stage TEXT", + "ALTER TABLE opportunity_events ADD COLUMN IF NOT EXISTS to_stage TEXT", + "ALTER TABLE opportunity_events ADD COLUMN IF NOT EXISTS note TEXT", + "ALTER TABLE opportunity_events ADD COLUMN IF NOT EXISTS payload JSONB NOT NULL DEFAULT '{}'::jsonb", + "ALTER TABLE opportunity_events ADD COLUMN IF NOT EXISTS created_by TEXT NOT NULL DEFAULT 'system'", + "ALTER TABLE opportunity_events ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ NOT NULL DEFAULT now()", + "ALTER TABLE opportunity_items ADD COLUMN IF NOT EXISTS product_id UUID", + "ALTER TABLE opportunity_items ADD COLUMN IF NOT EXISTS sku TEXT", + "ALTER TABLE opportunity_items ADD COLUMN IF NOT EXISTS jasmin_sales_item TEXT", + "ALTER TABLE opportunity_items ADD COLUMN IF NOT EXISTS product_name TEXT", + "ALTER TABLE opportunity_items ADD COLUMN IF NOT EXISTS description TEXT", + "ALTER TABLE opportunity_items ADD COLUMN IF NOT EXISTS quantity NUMERIC(12,2) NOT NULL DEFAULT 1", + "ALTER TABLE opportunity_items ADD COLUMN IF NOT EXISTS unit_price NUMERIC(12,2) NOT NULL DEFAULT 0", + "ALTER TABLE opportunity_items ADD COLUMN IF NOT EXISTS discount_amount NUMERIC(12,2) NOT NULL DEFAULT 0", + "ALTER TABLE opportunity_items ADD COLUMN IF NOT EXISTS total_price NUMERIC(12,2) NOT NULL DEFAULT 0", + "ALTER TABLE opportunity_items ADD COLUMN IF NOT EXISTS status TEXT NOT NULL DEFAULT 'INTERESTED'", + "ALTER TABLE opportunity_items ADD COLUMN IF NOT EXISTS metadata JSONB NOT NULL DEFAULT '{}'::jsonb", + "ALTER TABLE opportunity_items ADD COLUMN IF NOT EXISTS created_at TIMESTAMPTZ NOT NULL DEFAULT now()", + "ALTER TABLE opportunity_items ADD COLUMN IF NOT EXISTS updated_at TIMESTAMPTZ NOT NULL DEFAULT now()", + ]: + conn.execute(text(stmt)) + conn.execute(text("CREATE INDEX IF NOT EXISTS idx_opportunities_local_customer ON opportunities(local_customer_id)")) + + _SCHEMA_READY = True + + +def upsert_reconciliation_item( + *, + source_system: str, + external_type: str, + external_id: Optional[str] = None, + title: str, + description: str = "", + status: str = "open", + priority: str = "normal", + suggested_action: Optional[str] = None, + confidence: Optional[float] = None, + opportunity_id: Optional[str] = None, + customer_id: Optional[str] = None, + customer_name: Optional[str] = None, + customer_email: Optional[str] = None, + customer_tax_id: Optional[str] = None, + document_number: Optional[str] = None, + document_date: Optional[str] = None, + amount: Any = None, + currency: str = "EUR", + payload: Optional[Dict[str, Any]] = None, + idempotency_key: Optional[str] = None, +) -> Dict[str, Any]: + ensure_reconciliation_schema() + source_system = _clean(source_system) or "external" + external_type = _clean(external_type) or "external_record" + external_id = _clean(external_id) or None + idempotency_key = _clean(idempotency_key) or ( + f"{source_system}:{external_type}:{external_id}" if external_id else None + ) + amount_value = _money_or_none(amount) + customer_tax_id = _normalize_tax_id(customer_tax_id) or None + + with engine.begin() as conn: + row = conn.execute(text(""" + INSERT INTO reconciliation_items ( + source_system, external_type, external_id, title, description, status, + priority, suggested_action, confidence, opportunity_id, customer_id, + customer_name, customer_email, customer_tax_id, document_number, document_date, amount, + currency, payload, idempotency_key, updated_at + ) VALUES ( + :source_system, :external_type, :external_id, :title, :description, :status, + :priority, :suggested_action, :confidence, CAST(:opportunity_id AS UUID), + CAST(:customer_id AS UUID), :customer_name, :customer_email, :customer_tax_id, :document_number, + CAST(:document_date AS DATE), :amount, :currency, CAST(:payload AS JSONB), + :idempotency_key, now() + ) + ON CONFLICT (idempotency_key) WHERE idempotency_key IS NOT NULL AND idempotency_key <> '' + DO UPDATE SET + title = EXCLUDED.title, + description = EXCLUDED.description, + priority = EXCLUDED.priority, + suggested_action = EXCLUDED.suggested_action, + confidence = EXCLUDED.confidence, + opportunity_id = COALESCE(reconciliation_items.opportunity_id, EXCLUDED.opportunity_id), + customer_id = COALESCE(reconciliation_items.customer_id, EXCLUDED.customer_id), + customer_name = COALESCE(EXCLUDED.customer_name, reconciliation_items.customer_name), + customer_email = COALESCE(EXCLUDED.customer_email, reconciliation_items.customer_email), + customer_tax_id = COALESCE(EXCLUDED.customer_tax_id, reconciliation_items.customer_tax_id), + document_number = COALESCE(EXCLUDED.document_number, reconciliation_items.document_number), + document_date = COALESCE(EXCLUDED.document_date, reconciliation_items.document_date), + amount = COALESCE(EXCLUDED.amount, reconciliation_items.amount), + currency = COALESCE(EXCLUDED.currency, reconciliation_items.currency), + status = CASE + WHEN reconciliation_items.status = 'ignored' AND reconciliation_items.payload ? 'window_cleanup' THEN EXCLUDED.status + ELSE reconciliation_items.status + END, + resolution_note = CASE + WHEN reconciliation_items.status = 'ignored' AND reconciliation_items.payload ? 'window_cleanup' THEN NULL + ELSE reconciliation_items.resolution_note + END, + resolved_at = CASE + WHEN reconciliation_items.status = 'ignored' AND reconciliation_items.payload ? 'window_cleanup' THEN NULL + ELSE reconciliation_items.resolved_at + END, + payload = (CASE + WHEN reconciliation_items.status = 'ignored' AND reconciliation_items.payload ? 'window_cleanup' THEN reconciliation_items.payload - 'window_cleanup' + ELSE reconciliation_items.payload + END) || EXCLUDED.payload, + updated_at = now() + RETURNING id::text, source_system, external_type, external_id, title, description, + status, priority, suggested_action, confidence, opportunity_id::text, + customer_id::text, customer_name, customer_email, customer_tax_id, document_number, + document_date, amount, currency, payload, resolution_note, created_at, + updated_at, resolved_at + """), { + "source_system": source_system, + "external_type": external_type, + "external_id": external_id, + "title": _clean(title) or "Item de reconciliação", + "description": _clean(description), + "status": _clean(status) or "open", + "priority": _clean(priority) or "normal", + "suggested_action": _clean(suggested_action) or None, + "confidence": confidence, + "opportunity_id": _uuid_or_none(opportunity_id), + "customer_id": _uuid_or_none(customer_id), + "customer_name": _clean(customer_name) or None, + "customer_email": _clean(customer_email) or None, + "customer_tax_id": customer_tax_id, + "document_number": _clean(document_number) or None, + "document_date": _clean(document_date) or None, + "amount": amount_value, + "currency": _clean(currency) or "EUR", + "payload": _json(payload or {}), + "idempotency_key": idempotency_key, + }).mappings().first() + return dict(row or {}) + + +def list_reconciliation_items(*, status: str = "open", external_type: Optional[str] = None, limit: int = 200, days: Optional[int] = None) -> List[Dict[str, Any]]: + ensure_reconciliation_schema() + where = [] + params: Dict[str, Any] = {"limit": int(limit)} + status = _clean(status) or "open" + if status != "all": + where.append("ri.status = :status") + params["status"] = status + if external_type: + where.append("ri.external_type = :external_type") + params["external_type"] = external_type + if days and status in {"open", "needs_review", "all"}: + # Keep the default page focused on the recent operational window. + # Manual/payment-proof items without a document_date stay visible; + # dated external API candidates outside the window move out of view + # until the operator chooses a wider historical filter/export. + where.append("(ri.document_date IS NULL OR ri.document_date >= CAST(:recent_cutoff AS DATE))") + params["recent_cutoff"] = recent_window_start(days) + where_sql = "WHERE " + " AND ".join(where) if where else "" + with engine.begin() as conn: + rows = conn.execute(text(f""" + SELECT ri.id::text, ri.source_system, ri.external_type, ri.external_id, + ri.title, ri.description, ri.status, ri.priority, ri.suggested_action, + ri.confidence, ri.opportunity_id::text, ri.customer_id::text, + ri.customer_name, ri.customer_email, ri.customer_tax_id, ri.document_number, ri.document_date, + ri.amount, ri.currency, ri.payload, ri.resolution_note, ri.created_at, + ri.updated_at, ri.resolved_at, + o.title AS opportunity_title, + c.name AS linked_customer_name + FROM reconciliation_items ri + LEFT JOIN opportunities o ON o.id = ri.opportunity_id + LEFT JOIN customers c ON c.id = ri.customer_id + {where_sql} + ORDER BY + CASE ri.priority WHEN 'alta' THEN 0 WHEN 'high' THEN 0 WHEN 'normal' THEN 1 ELSE 2 END, + ri.updated_at DESC, + ri.created_at DESC + LIMIT :limit + """), params).mappings().all() + items = [dict(row) for row in rows] + for item in items: + if str(item.get("status") or "") in RECONCILIATION_OPEN_STATUSES: + try: + item["operation_suggestions"] = find_open_operation_suggestions_for_reconciliation(item, limit=2) + except Exception as exc: # pragma: no cover - production safety guard + # Suggestion lookup is helpful, but it must never break the + # reconciliation page. If an older database/schema has a gap, + # show the items without suggestions and log the cause. + logger.warning("reconciliation suggestions failed for item %s: %s", item.get("id"), exc) + item["operation_suggestions"] = [] + item["operation_suggestions_error"] = str(exc) + else: + item["operation_suggestions"] = [] + return items + + +def get_reconciliation_item(item_id: str) -> Optional[Dict[str, Any]]: + ensure_reconciliation_schema() + with engine.begin() as conn: + row = conn.execute(text(""" + SELECT id::text, source_system, external_type, external_id, title, description, + status, priority, suggested_action, confidence, opportunity_id::text, + customer_id::text, customer_name, customer_email, customer_tax_id, document_number, + document_date, amount, currency, payload, resolution_note, created_at, + updated_at, resolved_at + FROM reconciliation_items + WHERE id = CAST(:id AS UUID) + LIMIT 1 + """), {"id": item_id}).mappings().first() + return dict(row) if row else None + + +def reconciliation_summary(*, days: Optional[int] = None) -> Dict[str, int]: + ensure_reconciliation_schema() + where_sql = "" + params: Dict[str, Any] = {} + if days: + where_sql = "WHERE document_date IS NULL OR document_date >= CAST(:recent_cutoff AS DATE)" + params["recent_cutoff"] = recent_window_start(days) + with engine.begin() as conn: + rows = conn.execute(text(f""" + SELECT status, external_type, COUNT(*) AS total + FROM reconciliation_items + {where_sql} + GROUP BY status, external_type + """), params).mappings().all() + result: Dict[str, int] = { + "open": 0, + "needs_review": 0, + "conflict": 0, + "historical": 0, + "documents": 0, + "payments": 0, + "external_sales": 0, + "shipments": 0, + "resolved": 0, + "ignored": 0, + } + for row in rows: + status = str(row.get("status") or "") + typ = str(row.get("external_type") or "") + total = int(row.get("total") or 0) + if status in RECONCILIATION_OPEN_STATUSES: + result["open"] += total + if status == "needs_review": + result["needs_review"] += total + if status == "conflict": + result["conflict"] += total + if typ in DOCUMENT_EXTERNAL_TYPES: + result["documents"] += total + if typ in PAYMENT_EXTERNAL_TYPES: + result["payments"] += total + if typ == "odoo_sale_order": + result["external_sales"] += total + if typ == "packlink_shipment": + result["shipments"] += total + if status in {"resolved", "linked"}: + result["resolved"] += total + if status == "ignored": + result["ignored"] += total + if status == "historical": + result["historical"] += total + return result + + + +def cleanup_reconciliation_outside_window( + *, + days: int = 3, + source_system: Optional[str] = None, + limit: int = 1000, + actor: str = "cleanup_reconciliation_window", +) -> Dict[str, Any]: + """Mark open reconciliation candidates older than the working window as ignored. + + The reconciliation page is meant to be a short operational queue, not a + historical import list. This helper keeps recent candidates visible and + moves older API candidates out of the operator workflow without deleting + source records or external documents. + """ + ensure_reconciliation_schema() + from datetime import datetime, timedelta, timezone + + days = max(int(days or 3), 1) + limit = min(max(int(limit or 1000), 1), 5000) + source_system = _clean(source_system) or None + cutoff = recent_window_start(days) + params: Dict[str, Any] = {"cutoff": cutoff, "limit": limit} + source_sql = "" + if source_system: + source_sql = "AND source_system = :source_system" + params["source_system"] = source_system + + with engine.begin() as conn: + rows = conn.execute(text(f""" + SELECT id::text, source_system, external_type, document_number, + customer_name, document_date, title + FROM reconciliation_items + WHERE status IN ('open', 'needs_review') + {source_sql} + AND document_date IS NOT NULL + AND document_date < CAST(:cutoff AS DATE) + ORDER BY document_date ASC, updated_at DESC + LIMIT :limit + """), params).mappings().all() + ids = [str(r["id"]) for r in rows] + if ids: + conn.execute(text(""" + UPDATE reconciliation_items + SET status = 'ignored', + resolution_note = COALESCE(NULLIF(resolution_note, ''), 'Ignorado automaticamente: fora da janela operacional de reconciliação.'), + resolved_at = now(), + updated_at = now(), + payload = COALESCE(payload, '{}'::jsonb) || CAST(:payload AS JSONB) + WHERE id = ANY(CAST(:ids AS UUID[])) + """), { + "ids": ids, + "payload": _json({"window_cleanup": {"days": days, "cutoff": cutoff, "actor": actor, "source_system": source_system or "all"}}), + }) + return { + "days": days, + "cutoff": cutoff, + "source_system": source_system or "all", + "matched": len(rows), + "ignored": len(ids), + "items": [dict(row) for row in rows[:50]], + } + + + +def reset_generated_reconciliation_items( + *, + days: Optional[int] = None, + sources: Optional[list[str]] = None, + statuses: Optional[list[str]] = None, + limit: int = 1000, + actor: str = "operator_ui", + apply: bool = False, +) -> Dict[str, Any]: + """Preview or delete generated reconciliation staging rows safely. + + This is the UI-safe version of scripts/reset_reconciliation_generated.py. + It only targets generated external candidates and always protects rows that + are already linked to an opportunity. When apply=True, a timestamped backup + table is created before deleting rows. + """ + ensure_reconciliation_schema() + from datetime import datetime, timezone + + allowed_sources = {"jasmin", "odoo", "packlink"} + allowed_statuses = {"open", "needs_review", "ignored"} + selected_sources = [str(s) for s in (sources or ["jasmin", "odoo", "packlink"]) if str(s) in allowed_sources] + selected_statuses = [str(s) for s in (statuses or ["open", "needs_review", "ignored"]) if str(s) in allowed_statuses] + if not selected_sources: + selected_sources = ["jasmin", "odoo", "packlink"] + if not selected_statuses: + selected_statuses = ["open", "needs_review", "ignored"] + + clauses = [ + "source_system = ANY(CAST(:sources AS TEXT[]))", + "status = ANY(CAST(:statuses AS TEXT[]))", + "opportunity_id IS NULL", + ] + params: Dict[str, Any] = {"sources": selected_sources, "statuses": selected_statuses} + if days is not None: + cutoff = recent_window_start(max(int(days or 1), 1)) + params["cutoff"] = cutoff + clauses.append("(document_date IS NULL OR document_date >= CAST(:cutoff AS DATE))") + where_sql = " AND ".join(clauses) + sample_limit = min(max(int(limit or 1000), 1), 5000) + + with engine.begin() as conn: + counts = conn.execute(text(f""" + SELECT source_system, external_type, status, COUNT(*) AS total + FROM reconciliation_items + WHERE {where_sql} + GROUP BY source_system, external_type, status + ORDER BY source_system, external_type, status + """), params).mappings().all() + rows = conn.execute(text(f""" + SELECT id::text, source_system, external_type, status, document_number, + customer_name, customer_tax_id, document_date, amount, title + FROM reconciliation_items + WHERE {where_sql} + ORDER BY updated_at DESC, created_at DESC + LIMIT :limit + """), {**params, "limit": sample_limit}).mappings().all() + matched = conn.execute(text(f"SELECT COUNT(*) FROM reconciliation_items WHERE {where_sql}"), params).scalar() or 0 + + backup_table = None + deleted = 0 + if apply and int(matched) > 0: + backup_table = "reconciliation_items_reset_backup_" + datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S") + conn.execute(text(f"CREATE TABLE {backup_table} AS SELECT * FROM reconciliation_items WHERE {where_sql}"), params) + deleted = conn.execute(text(f"DELETE FROM reconciliation_items WHERE {where_sql}"), params).rowcount or 0 + + return { + "days": days, + "sources": selected_sources, + "statuses": selected_statuses, + "matched": int(matched), + "deleted": int(deleted), + "backup_table": backup_table, + "counts": [dict(r) for r in counts], + "items": [dict(r) for r in rows], + "actor": actor, + "applied": bool(apply), + } + +def _norm_match_value(value: Any) -> str: + """Normalize loose names/emails for conservative matching suggestions.""" + return _clean(value).lower() + + +def find_open_operation_suggestions_for_reconciliation(item: Dict[str, Any], *, limit: int = 3) -> List[Dict[str, Any]]: + """Suggest open ClientFlow operations/opportunities that may match an external item. + + v4.9.11 makes this lookup resilient and less dependent on NIF. Exact NIF + remains the strongest signal, but if Odoo/Jasmin do not provide VAT/NIF the + system can still suggest by normalized company name or email. It never + links automatically. + """ + ensure_reconciliation_schema() + if not item or item.get("opportunity_id"): + return [] + + customer_id = _uuid_or_none(item.get("customer_id")) + customer_email = _norm_match_value(item.get("customer_email")) + payload = item.get("payload") if isinstance(item.get("payload"), dict) else {} + customer_tax_id = _normalize_tax_id( + item.get("customer_tax_id") + or payload.get("customer_tax_id") + or payload.get("tax_id") + or "" + ) + customer_name = _clean(item.get("customer_name") or item.get("linked_customer_name")) + odoo_partner_external_id = _odoo_partner_external_id_from_item(item) if str(item.get("source_system") or "") == "odoo" else "" + mapped_customer_id = "" + if odoo_partner_external_id: + try: + with engine.begin() as conn: + mapped_customer_id = _clean(conn.execute(text(""" + SELECT customer_id::text + FROM external_customer_mappings + WHERE source_system = 'odoo' AND external_party_id = :external_party_id + LIMIT 1 + """), {"external_party_id": odoo_partner_external_id}).scalar()) + except Exception: + mapped_customer_id = "" + amount_value = _money_or_none(item.get("amount")) + amount_decimal: Optional[Decimal] = None + if amount_value is not None: + try: + amount_decimal = Decimal(amount_value) + except (InvalidOperation, ValueError): + amount_decimal = None + limit = min(max(int(limit or 3), 1), 5) + + # Keep the old broad guard semantics: if not any([customer_id, customer_tax_id, customer_email, customer_name, amount_value]) + if not any([customer_id, customer_tax_id, customer_email, customer_name, amount_decimal is not None]): + return [] + + # Fetch a bounded set of open opportunities and score in Python. This is + # more robust than a large SQL CASE expression across installations with + # slightly different schemas/data types, and avoids a single bad record + # breaking /reconciliation. + with engine.begin() as conn: + rows = conn.execute(text(""" + SELECT + o.id::text AS opportunity_id, + o.title AS opportunity_title, + o.stage, + o.status, + o.customer_name, + o.customer_email, + o.value_amount, + o.local_customer_id::text AS local_customer_id, + o.updated_at, + lc.name AS linked_customer_name, + lc.email AS linked_customer_email, + lc.tax_id AS linked_customer_tax_id, + t.id::text AS task_id, + t.action_code, + t.action, + t.route, + t.created_at AS task_created_at + FROM opportunities o + LEFT JOIN customers lc ON lc.id = o.local_customer_id + LEFT JOIN LATERAL ( + SELECT id, action_code, action, route, created_at + FROM tasks + WHERE opportunity_id = o.id AND status = 'pending' + ORDER BY + CASE WHEN action_code IN ('SEND_INVOICE','SEND_PROFORMA','CONFIRM_PAYMENT','SEND_QUOTE') THEN 0 ELSE 1 END, + created_at DESC + LIMIT 1 + ) t ON true + WHERE o.status = 'open' + ORDER BY + CASE WHEN t.id IS NOT NULL THEN 0 ELSE 1 END, + o.updated_at DESC + LIMIT 250 + """)).mappings().all() + + suggestions: List[Dict[str, Any]] = [] + for row in rows: + score = 0 + reasons: List[str] = [] + if mapped_customer_id and str(row.get("local_customer_id") or "") == mapped_customer_id: + score += 120 + reasons.append("mapeamento Odoo confirmado") + if customer_id and str(row.get("local_customer_id") or "") == customer_id: + score += 60 + reasons.append("cliente fiscal") + linked_tax = _normalize_tax_id(row.get("linked_customer_tax_id") or "") + # Equivalent of the old SQL signal: COALESCE(lc.tax_id, '') = :customer_tax_id THEN 90 ELSE 0 + # but normalized in Python so PT prefixes/spaces do not break matching. + if customer_tax_id and linked_tax and linked_tax == customer_tax_id: + score += 100 + reasons.append("NIF exato") + row_emails = { + _norm_match_value(row.get("customer_email")), + _norm_match_value(row.get("linked_customer_email")), + } + if customer_email and customer_email in row_emails: + score += 45 + reasons.append("email") + name_score = _name_match_score( + customer_name, + row.get("linked_customer_name"), + row.get("customer_name"), + row.get("opportunity_title"), + ) + name_is_strong = name_score >= 35 + if name_score: + score += name_score + reasons.append("nome fiscal" if name_is_strong else "nome parcial") + if amount_decimal is not None and row.get("value_amount") is not None: + try: + row_amount = Decimal(str(row.get("value_amount"))) + if row_amount >= (amount_decimal * Decimal("0.90")) and row_amount <= (amount_decimal * Decimal("1.10")): + score += 10 + reasons.append("valor aproximado") + except (InvalidOperation, ValueError, TypeError): + pass + if row.get("task_id"): + score += 20 + reasons.append("operação aberta") + + # NIF/email/customer-id are strong. By project rule, a strong normalized + # company fiscal-name match is also a useful identity signal. Weak token + # overlap must not become a suggestion just because there is a pending + # task or a similar amount. + strong_identity = bool(mapped_customer_id and str(row.get("local_customer_id") or "") == mapped_customer_id) or bool(customer_tax_id and linked_tax == customer_tax_id) or bool(customer_email and customer_email in row_emails) or bool(customer_id and str(row.get("local_customer_id") or "") == customer_id) + if not strong_identity and name_score and not name_is_strong: + continue + threshold = 35 if strong_identity else 40 + if score < threshold: + continue + + suggestions.append({ + "opportunity_id": row.get("opportunity_id"), + "opportunity_title": row.get("opportunity_title") or "Oportunidade", + "customer_name": row.get("linked_customer_name") or row.get("customer_name") or "", + "task_id": row.get("task_id"), + "action_code": row.get("action_code") or "", + "action": row.get("action") or "", + "score": int(score), + "reason": ", ".join(dict.fromkeys(reasons)) or "possível correspondência", + }) + + suggestions.sort(key=lambda s: int(s.get("score") or 0), reverse=True) + return suggestions[:limit] + + +# v4.9.9 — process timeline reconstruction +# ------------------------------------------------------------ +# Reconciliação is more useful when loose external items are shown as a +# possible commercial process instead of isolated rows. These helpers group +# open candidates by strong identity keys (NIF first, then email/name) and infer +# a conservative timeline/state proposal. They never link or create anything +# until the operator explicitly confirms the action in the UI. + +def _payload_record(item: Dict[str, Any]) -> Dict[str, Any]: + payload = item.get("payload") if isinstance(item.get("payload"), dict) else {} + record = payload.get("record") if isinstance(payload.get("record"), dict) else {} + return record + + +def _odoo_partner_external_id_from_item(item: Dict[str, Any]) -> str: + record = _payload_record(item) + partner_id = record.get("partner_external_id") + if partner_id: + return _clean(partner_id) + value = record.get("partner_id") + if isinstance(value, (list, tuple)) and value: + return _clean(value[0]) + return _clean(value) + + +def _odoo_fulfilment_from_item(item: Dict[str, Any]) -> Dict[str, Any]: + record = _payload_record(item) + fulfilment = record.get("fulfilment") + if isinstance(fulfilment, dict): + return fulfilment + # Compatibility with older payloads: infer only the invoice pending flag. + invoice_status = _clean(record.get("invoice_status")) + return { + "invoice_status": invoice_status, + "invoice_pending": invoice_status in {"to invoice", "no"}, + "physical_status": "order_created", + "label": "Venda criada", + "stage": "ODOO_ORDER_CREATED", + "lines": record.get("order_lines") if isinstance(record.get("order_lines"), list) else [], + "outgoing_pickings": record.get("pickings") if isinstance(record.get("pickings"), list) else [], + } + + +def _odoo_sale_link_payload(item: Dict[str, Any]) -> Dict[str, Any]: + record = _payload_record(item) + return { + "sale_order": { + "id": record.get("id") or item.get("external_id"), + "name": record.get("name") or item.get("document_number"), + "state": record.get("state"), + "partner_id": record.get("partner_id"), + "partner_external_id": record.get("partner_external_id"), + "partner_name": record.get("partner_name") or item.get("customer_name"), + "amount_total": record.get("amount_total") or item.get("amount"), + "date_order": record.get("date_order") or item.get("document_date"), + "invoice_status": record.get("invoice_status"), + }, + "order_lines": record.get("order_lines") if isinstance(record.get("order_lines"), list) else [], + "fulfilment": _odoo_fulfilment_from_item(item), + } + + +def _upsert_external_customer_mapping_from_process(conn: Any, item: Dict[str, Any], opportunity_id: str, *, actor: str) -> None: + if str(item.get("source_system") or "") != "odoo": + return + external_party_id = _odoo_partner_external_id_from_item(item) + if not external_party_id: + return + row = conn.execute(text(""" + SELECT o.local_customer_id::text AS customer_id, c.name AS customer_name + FROM opportunities o + LEFT JOIN customers c ON c.id = o.local_customer_id + WHERE o.id = CAST(:opportunity_id AS UUID) + LIMIT 1 + """), {"opportunity_id": opportunity_id}).mappings().first() + customer_id = _clean(row.get("customer_id") if row else "") + if not customer_id: + return + record = _payload_record(item) + external_party_name = _clean(record.get("partner_name") or item.get("customer_name")) + conn.execute(text(""" + INSERT INTO external_customer_mappings (source_system, external_party_id, external_party_name, customer_id, confidence, created_by, payload, updated_at) + VALUES ('odoo', :external_party_id, :external_party_name, CAST(:customer_id AS UUID), 'confirmed', :actor, CAST(:payload AS JSONB), now()) + ON CONFLICT (source_system, external_party_id) + DO UPDATE SET customer_id = EXCLUDED.customer_id, external_party_name = EXCLUDED.external_party_name, confidence = 'confirmed', created_by = EXCLUDED.created_by, payload = external_customer_mappings.payload || EXCLUDED.payload, updated_at = now() + """), { + "external_party_id": external_party_id, + "external_party_name": external_party_name, + "customer_id": customer_id, + "actor": actor, + "payload": _json({"confirmed_from_opportunity_id": opportunity_id, "customer_name": row.get("customer_name") if row else ""}), + }) + + +def _upsert_odoo_operation_links_from_item(conn: Any, item: Dict[str, Any], opportunity_id: str) -> None: + if str(item.get("external_type") or "") != "odoo_sale_order": + return + record = _payload_record(item) + fulfilment = _odoo_fulfilment_from_item(item) + sale_name = _clean(record.get("name") or item.get("document_number") or item.get("external_id")) + sale_id = _clean(record.get("id") or item.get("external_id")) + conn.execute(text(""" + INSERT INTO operation_links (opportunity_id, system, external_type, external_id, external_name, external_url, status, payload, last_synced_at, updated_at) + VALUES (CAST(:opportunity_id AS UUID), 'odoo', 'sale_order', :external_id, :external_name, NULL, 'created', CAST(:payload AS JSONB), now(), now()) + ON CONFLICT (opportunity_id, system, external_type) + DO UPDATE SET external_id = EXCLUDED.external_id, external_name = EXCLUDED.external_name, status = EXCLUDED.status, payload = operation_links.payload || EXCLUDED.payload, last_synced_at = now(), updated_at = now() + """), { + "opportunity_id": opportunity_id, + "external_id": sale_id, + "external_name": sale_name, + "payload": _json(_odoo_sale_link_payload(item)), + }) + conn.execute(text(""" + INSERT INTO operation_links (opportunity_id, system, external_type, external_id, external_name, external_url, status, payload, last_synced_at, updated_at) + VALUES (CAST(:opportunity_id AS UUID), 'odoo', 'physical_status', :external_id, :external_name, NULL, :status, CAST(:payload AS JSONB), now(), now()) + ON CONFLICT (opportunity_id, system, external_type) + DO UPDATE SET external_id = EXCLUDED.external_id, external_name = EXCLUDED.external_name, status = EXCLUDED.status, payload = operation_links.payload || EXCLUDED.payload, last_synced_at = now(), updated_at = now() + """), { + "opportunity_id": opportunity_id, + "external_id": sale_id, + "external_name": sale_name, + "status": _clean(fulfilment.get("physical_status")) or "order_created", + "payload": _json({"sale_order": sale_name, **fulfilment}), + }) + + productions = fulfilment.get("productions") if isinstance(fulfilment.get("productions"), list) else [] + if productions: + done_count = sum(1 for production in productions if _clean(production.get("state")) == "done") + status = "done" if done_count == len(productions) else "in_progress" + label = f"{done_count}/{len(productions)} produções concluídas" if done_count != len(productions) else f"{done_count} produções concluídas" + conn.execute(text(""" + INSERT INTO operation_links (opportunity_id, system, external_type, external_id, external_name, external_url, status, payload, last_synced_at, updated_at) + VALUES (CAST(:opportunity_id AS UUID), 'odoo', 'production', :external_id, :external_name, NULL, :status, CAST(:payload AS JSONB), now(), now()) + ON CONFLICT (opportunity_id, system, external_type) + DO UPDATE SET external_id = EXCLUDED.external_id, external_name = EXCLUDED.external_name, status = EXCLUDED.status, payload = operation_links.payload || EXCLUDED.payload, last_synced_at = now(), updated_at = now() + """), { + "opportunity_id": opportunity_id, + "external_id": sale_id, + "external_name": label, + "status": status, + "payload": _json({"sale_order": sale_name, "productions": productions}), + }) + + if fulfilment.get("delivery_done") or fulfilment.get("delivery_ready"): + conn.execute(text(""" + INSERT INTO operation_links (opportunity_id, system, external_type, external_id, external_name, external_url, status, payload, last_synced_at, updated_at) + VALUES (CAST(:opportunity_id AS UUID), 'odoo', 'physical_validation', :external_id, :external_name, NULL, :status, CAST(:payload AS JSONB), now(), now()) + ON CONFLICT (opportunity_id, system, external_type) + DO UPDATE SET external_id = EXCLUDED.external_id, external_name = EXCLUDED.external_name, status = EXCLUDED.status, payload = operation_links.payload || EXCLUDED.payload, last_synced_at = now(), updated_at = now() + """), { + "opportunity_id": opportunity_id, + "external_id": sale_id, + "external_name": "Entrega concluída" if fulfilment.get("delivery_done") else "Entrega pronta", + "status": "validated" if fulfilment.get("delivery_done") else "ready_to_ship", + "payload": _json({"sale_order": sale_name, **fulfilment}), + }) + + +def _odoo_importable_lines(item: Dict[str, Any]) -> List[Dict[str, Any]]: + fulfilment = _odoo_fulfilment_from_item(item) + lines = fulfilment.get("lines") if isinstance(fulfilment.get("lines"), list) else [] + result: List[Dict[str, Any]] = [] + for line in lines: + if not isinstance(line, dict): + continue + name = _clean(line.get("product_name") or line.get("description")) + if not name: + continue + total = _money_or_none(line.get("price_total")) or "0.00" + unit = _money_or_none(line.get("price_unit")) or "0.00" + qty = _money_or_none(line.get("qty_ordered") or line.get("quantity") or "1") or "1.00" + # Zero-value Odoo delivery carrier lines are operational context, not + # commercial product lines for Jasmin invoicing. Keep them in the + # operation link payload but do not pollute Produtos. + if Decimal(total) == Decimal("0.00") and ("delivery" in name.lower() or "shipping" in name.lower()): + continue + result.append({**line, "product_name": name, "price_total": total, "price_unit": unit, "qty_ordered": qty}) + return result + + +def _odoo_product_sku_from_line(line: Dict[str, Any]) -> Optional[str]: + """Return the ClientFlow SKU used to map Odoo products to Jasmin items. + + The Odoo API usually returns numeric product ids in sale order lines, while + the ClientFlow catalogue stores them as ``ODOO-``. Using the raw id as + SKU (``3``) made imported opportunity lines look unmapped even when the + catalogue had ``ODOO-3 -> CARREGADOR_TRIF_22KW`` configured. + """ + raw = _clean(line.get("product_id") or line.get("odoo_product_id") or line.get("sku")) + if not raw: + return None + if raw.upper().startswith("ODOO-"): + return raw.upper() + if raw.isdigit(): + return f"ODOO-{raw}" + return raw + + +def _normalize_product_name(value: Any) -> str: + text_value = unicodedata.normalize("NFKD", _clean(value)).encode("ascii", "ignore").decode("ascii") + return re.sub(r"[^A-Z0-9]+", "", text_value.upper()) + + +def _resolve_product_mapping_for_odoo_line(conn: Any, line: Dict[str, Any]) -> Dict[str, Any]: + """Resolve an imported Odoo line against the ClientFlow product catalogue. + + Priority: + 1. ``products.sku = ODOO-`` + 2. ``products.metadata->odoo->product_id`` + 3. exact normalized product name fallback + """ + sku = _odoo_product_sku_from_line(line) + product_id_raw = _clean(line.get("product_id")) + row = None + if sku: + row = conn.execute(text(""" + SELECT id::text, sku, jasmin_sales_item, name + FROM products + WHERE sku = :sku + LIMIT 1 + """), {"sku": sku}).mappings().first() + if row is None and product_id_raw: + row = conn.execute(text(""" + SELECT id::text, sku, jasmin_sales_item, name + FROM products + WHERE metadata->'odoo'->>'product_id' = :product_id + LIMIT 1 + """), {"product_id": product_id_raw}).mappings().first() + if row is None: + wanted = _normalize_product_name(line.get("product_name") or line.get("description")) + if wanted: + for candidate in conn.execute(text(""" + SELECT id::text, sku, jasmin_sales_item, name + FROM products + WHERE active IS TRUE + LIMIT 500 + """)).mappings().all(): + if _normalize_product_name(candidate.get("name")) == wanted: + row = candidate + break + if row is None: + return {"product_id": None, "sku": sku, "jasmin_sales_item": None, "catalog_name": None, "mapping_status": "missing"} + return { + "product_id": row.get("id"), + "sku": row.get("sku") or sku, + "jasmin_sales_item": row.get("jasmin_sales_item"), + "catalog_name": row.get("name"), + "mapping_status": "mapped" if row.get("jasmin_sales_item") else "missing_jasmin", + } + + +def _upsert_opportunity_items_from_odoo_item(conn: Any, item: Dict[str, Any], opportunity_id: str) -> None: + if str(item.get("external_type") or "") != "odoo_sale_order": + return + record = _payload_record(item) + sale_name = _clean(record.get("name") or item.get("document_number") or item.get("external_id")) + for line in _odoo_importable_lines(item): + line_id = _clean(line.get("id") or line.get("product_id") or line.get("product_name")) + qty = _money_or_none(line.get("qty_ordered") or "1") or "1.00" + unit_price = _money_or_none(line.get("price_unit") or "0") or "0.00" + total_price = _money_or_none(line.get("price_total") or "0") or "0.00" + status = "DELIVERED" if float(line.get("qty_delivered") or 0) >= float(line.get("qty_ordered") or 0 or 0) and float(line.get("qty_ordered") or 0) > 0 else "ODOO_IMPORTED" + product_mapping = _resolve_product_mapping_for_odoo_line(conn, line) + conn.execute(text(""" + INSERT INTO opportunity_items ( + id, opportunity_id, product_id, sku, jasmin_sales_item, product_name, description, + quantity, unit_price, discount_amount, total_price, status, metadata + ) + SELECT + CAST(:id AS UUID), CAST(:opportunity_id AS UUID), CAST(:product_id AS UUID), :sku, :jasmin_sales_item, :product_name, :description, + :quantity, :unit_price, 0, :total_price, :status, CAST(:metadata AS JSONB) + WHERE NOT EXISTS ( + SELECT 1 FROM opportunity_items + WHERE opportunity_id = CAST(:opportunity_id AS UUID) + AND metadata->>'source_system' = 'odoo' + AND metadata->>'source_line_id' = :source_line_id + ) + """), { + "id": str(uuid.uuid4()), + "opportunity_id": opportunity_id, + "product_id": product_mapping.get("product_id"), + "sku": product_mapping.get("sku"), + "jasmin_sales_item": product_mapping.get("jasmin_sales_item"), + "product_name": line.get("product_name"), + "description": line.get("description") or line.get("product_name"), + "quantity": qty, + "unit_price": unit_price, + "total_price": total_price, + "status": status, + "source_line_id": line_id, + "metadata": _json({ + "source_system": "odoo", + "source_document": sale_name, + "source_line_id": line_id, + "qty_delivered": line.get("qty_delivered"), + "qty_invoiced": line.get("qty_invoiced"), + "product_id": line.get("product_id"), + "resolved_sku": product_mapping.get("sku"), + "resolved_jasmin_sales_item": product_mapping.get("jasmin_sales_item"), + "product_mapping_status": product_mapping.get("mapping_status"), + "catalog_name": product_mapping.get("catalog_name"), + }), + }) + + +# v4.9.26 — Jasmin document reconstruction +# ------------------------------------------------------------ +# When a reconciliation candidate is created from a Jasmin quotation/pro-forma/ +# invoice, the opportunity must not become an empty reminder. These helpers +# import the external document itself, copy its commercial lines into +# commercial_document_lines/opportunity_items and update the opportunity value. + +def _first_value(record: Dict[str, Any], *keys: str) -> Any: + for key in keys: + if key in record and record.get(key) not in (None, ""): + return record.get(key) + return None + + +def _date_or_none_value(value: Any) -> Optional[str]: + if value is None: + return None + if isinstance(value, datetime): + return value.date().isoformat() + text_value = _clean(value) + if len(text_value) >= 10 and text_value[4:5] == "-" and text_value[7:8] == "-": + return text_value[:10] + return None + + +def _jasmin_money_value(value: Any) -> Any: + """Return a numeric-like value from Jasmin money objects. + + Jasmin documents commonly encode money as either a scalar field + (for example ``grossValueAmount``) or as an object like + ``{"amount": 199.0, "symbol": "€", ...}``. The previous importer + passed those objects directly to Decimal, which produced null/0 values + even though the payload had prices. + """ + if isinstance(value, dict): + for key in ("amount", "baseAmount", "reportingAmount", "value"): + if value.get(key) not in (None, ""): + return value.get(key) + return None + return value + + +def _decimal_string(value: Any, *, default: Optional[str] = None, places: str = "0.01") -> Optional[str]: + value = _jasmin_money_value(value) + raw = _clean(value).replace("€", "").replace(" ", "").replace(",", ".") + if raw == "": + return default + try: + return str(Decimal(raw).quantize(Decimal(places))) + except (InvalidOperation, ValueError): + return default + + +def _jasmin_document_kind_for_item(item: Dict[str, Any]) -> str: + external_type = _clean(item.get("external_type")) + return { + "jasmin_quotation": "quotation", + "jasmin_proforma": "proforma", + "jasmin_invoice": "invoice", + }.get(external_type, "document") + + +def _jasmin_document_status_for_item(item: Dict[str, Any]) -> str: + record = _payload_record(item) + status = _clean(_first_value(record, "status", "documentStatus", "state", "statusDescription")) + return status or "imported" + + +def _jasmin_first_money(record: Dict[str, Any], *keys: str) -> Any: + for key in keys: + value = _first_value(record, key) + if value not in (None, ""): + return _jasmin_money_value(value) + return None + + +def _jasmin_document_totals(item: Dict[str, Any]) -> Dict[str, Optional[str]]: + record = _payload_record(item) + amount = _decimal_string(_jasmin_first_money( + record, + "taxExclusiveAmountAmount", + "grossValueAmount", + "netAmount", + "taxExclusiveAmount", + "goodsAmount", + "grossValue", + "amount", + )) + tax_amount = _decimal_string(_jasmin_first_money( + record, + "taxTotalAmount", + "taxAmountAmount", + "taxAmount", + "taxTotal", + "vatAmount", + )) + total_amount = _decimal_string(_jasmin_first_money( + record, + "payableAmountAmount", + "lineExtensionAmountAmount", + "totalAmountAmount", + "payableAmount", + "totalAmount", + "total", + "grossAmount", + "lineExtensionAmount", + "amount", + ) or item.get("amount")) + if not amount and total_amount: + amount = total_amount + return {"amount": amount, "tax_amount": tax_amount, "total_amount": total_amount} + + +def _jasmin_line_lists(record: Dict[str, Any]) -> List[Dict[str, Any]]: + result: List[Dict[str, Any]] = [] + for key in ( + "lines", + "documentLines", + "document_lines", + "documentLine", + "salesLines", + "orderLines", + "quotationLines", + "invoiceLines", + "items", + ): + value = record.get(key) + if isinstance(value, list): + result.extend([line for line in value if isinstance(line, dict)]) + return result + + +def _jasmin_sales_item_from_line(line: Dict[str, Any]) -> str: + value = _first_value( + line, + "salesItem", + "salesItemId", + "salesItemKey", + "itemKey", + "itemCode", + "item", + "article", + "productCode", + "sku", + ) + if isinstance(value, dict): + value = _first_value(value, "key", "id", "code", "name") + return _clean(value) + + +def _jasmin_description_from_line(line: Dict[str, Any]) -> str: + value = _first_value( + line, + "description", + "itemDescription", + "salesItemDescription", + "productName", + "name", + "articleDescription", + ) + if isinstance(value, dict): + value = _first_value(value, "description", "name", "key") + description = _clean(value) + return description or _jasmin_sales_item_from_line(line) or "Linha Jasmin" + + +def _jasmin_document_lines_from_item(item: Dict[str, Any]) -> List[Dict[str, Any]]: + record = _payload_record(item) + lines = [] + for idx, line in enumerate(_jasmin_line_lists(record)): + description = _jasmin_description_from_line(line) + jasmin_sales_item = _jasmin_sales_item_from_line(line) + quantity = _decimal_string(_first_value(line, "quantity", "qty", "orderedQuantity", "invoicedQuantity"), default="1.000", places="0.001") or "1.000" + unit = _clean(_first_value(line, "unit", "unitKey", "unitOfMeasure", "unitOfMeasureCode")) or "UN" + unit_price = _decimal_string(_jasmin_first_money( + line, + "unitPriceAmount", + "unitPrice", + "priceAmount", + "price", + "unitAmount", + ), default="0.00") or "0.00" + total_amount = _decimal_string(_jasmin_first_money( + line, + "lineExtensionAmountAmount", + "totalAmountAmount", + "grossValueAmount", + "taxExclusiveAmountAmount", + "totalAmount", + "lineExtensionAmount", + "grossValue", + "netAmount", + "amount", + "grossAmount", + )) + if total_amount is None: + try: + total_amount = str((Decimal(quantity) * Decimal(unit_price)).quantize(Decimal("0.01"))) + except (InvalidOperation, ValueError): + total_amount = "0.00" + line_id = _clean(_first_value(line, "id", "lineId", "lineNumber", "naturalKey", "key")) or f"{idx}:{jasmin_sales_item or description}" + tax_schema = _clean(_first_value(line, "taxSchema", "itemTaxSchema", "taxSchemaKey", "vatCode")) or "NORMAL" + lines.append({ + "source_line_id": line_id, + "line_index": idx, + "jasmin_sales_item": jasmin_sales_item, + "description": description, + "quantity": quantity, + "unit": unit, + "unit_price": unit_price, + "tax_schema": tax_schema, + "total_amount": total_amount, + "payload": line, + }) + return lines + + +def _resolve_product_mapping_for_jasmin_line(conn: Any, line: Dict[str, Any]) -> Dict[str, Any]: + jasmin_sales_item = _clean(line.get("jasmin_sales_item")) + row = None + if jasmin_sales_item: + row = conn.execute(text(""" + SELECT id::text, sku, jasmin_sales_item, name + FROM products + WHERE jasmin_sales_item = :jasmin_sales_item + OR sku = :jasmin_sales_item + LIMIT 1 + """), {"jasmin_sales_item": jasmin_sales_item}).mappings().first() + if row is None: + wanted = _normalize_product_name(line.get("description")) + if wanted: + for candidate in conn.execute(text(""" + SELECT id::text, sku, jasmin_sales_item, name + FROM products + WHERE active IS TRUE + LIMIT 500 + """)).mappings().all(): + if _normalize_product_name(candidate.get("name")) == wanted: + row = candidate + break + if row is None: + return {"product_id": None, "sku": None, "jasmin_sales_item": jasmin_sales_item or None, "catalog_name": None, "mapping_status": "missing"} + return { + "product_id": row.get("id"), + "sku": row.get("sku"), + "jasmin_sales_item": row.get("jasmin_sales_item") or jasmin_sales_item, + "catalog_name": row.get("name"), + "mapping_status": "mapped" if row.get("id") else "missing", + } + + +def _upsert_jasmin_document_from_item(conn: Any, item: Dict[str, Any], opportunity_id: str) -> Optional[str]: + if _clean(item.get("source_system")) != "jasmin" or not _clean(item.get("external_type")).startswith("jasmin_"): + return None + try: + from app.commercial_service import ensure_commercial_schema + ensure_commercial_schema() + except Exception as exc: # pragma: no cover - safety guard + logger.warning("failed to ensure commercial schema for Jasmin import: %s", exc) + return None + + record = _payload_record(item) + document_kind = _jasmin_document_kind_for_item(item) + totals = _jasmin_document_totals(item) + external_id = _clean(item.get("external_id") or _first_value(record, "id", "key", "documentKey", "naturalKey")) + document_number = _clean(item.get("document_number") or _first_value(record, "documentNumber", "number", "naturalKey", "name", "reference") or external_id) + customer_id = _uuid_or_none(item.get("customer_id")) + existing = conn.execute(text(""" + SELECT id::text + FROM commercial_documents + WHERE system = 'jasmin' + AND ( + (CAST(:external_id AS TEXT) <> '' AND external_id = CAST(:external_id AS TEXT)) + OR (CAST(:document_number AS TEXT) <> '' AND document_number = CAST(:document_number AS TEXT)) + ) + AND (opportunity_id = CAST(:opportunity_id AS UUID) OR opportunity_id IS NULL) + ORDER BY opportunity_id NULLS LAST, created_at DESC + LIMIT 1 + """), {"external_id": external_id, "document_number": document_number, "opportunity_id": opportunity_id}).scalar() + + payload = { + "source": "reconciliation_jasmin_import", + "reconciliation_item_id": item.get("id"), + "external_type": item.get("external_type"), + "record": record, + } + params = { + "id": existing or str(uuid.uuid4()), + "customer_id": customer_id, + "opportunity_id": opportunity_id, + "document_kind": document_kind, + "external_id": external_id or None, + "company": _clean(_first_value(record, "company", "companyKey")) or None, + "document_type": _clean(_first_value(record, "documentType", "documentTypeKey")) or None, + "serie": _clean(_first_value(record, "serie", "serieKey", "series")) or None, + "series_number": _clean(_first_value(record, "seriesNumber", "sequenceNumber")) or None, + "document_number": document_number or None, + "customer_party_key": _clean(_first_value(record, "buyerCustomerParty", "customerParty", "customerPartyKey", "partyKey")) or None, + "status": _jasmin_document_status_for_item(item), + "amount": totals.get("amount"), + "tax_amount": totals.get("tax_amount"), + "total_amount": totals.get("total_amount"), + "currency": item.get("currency") or _clean(_first_value(record, "currency", "currencyKey", "currencyCode")) or "EUR", + "document_date": _date_or_none_value(item.get("document_date") or _first_value(record, "documentDate", "date", "creationDate", "postingDate")), + "due_date": _date_or_none_value(_first_value(record, "dueDate", "paymentDueDate")), + "payload": _json(payload), + "role": "current" if document_kind in {"quotation", "proforma", "invoice"} else "related", + "is_primary": True, + } + if params["role"] in {"current", "accepted"}: + conn.execute(text(""" + UPDATE commercial_documents + SET role = CASE WHEN COALESCE(role, 'current') = 'current' THEN 'historical' ELSE role END, + is_primary = FALSE, + is_active = CASE WHEN COALESCE(role, 'current') = 'current' THEN FALSE ELSE COALESCE(is_active, TRUE) END, + updated_at = now() + WHERE opportunity_id = CAST(:opportunity_id AS UUID) + AND system = 'jasmin' + AND document_kind = :document_kind + AND id <> CAST(:id AS UUID) + AND COALESCE(role, 'current') IN ('current', 'accepted') + AND COALESCE(is_primary, TRUE) = TRUE + """), params) + if existing: + conn.execute(text(""" + UPDATE commercial_documents + SET customer_id = COALESCE(CAST(:customer_id AS UUID), customer_id), + opportunity_id = CAST(:opportunity_id AS UUID), + document_kind = :document_kind, + external_id = COALESCE(:external_id, external_id), + company = COALESCE(:company, company), + document_type = COALESCE(:document_type, document_type), + serie = COALESCE(:serie, serie), + series_number = CASE WHEN CAST(:series_number AS TEXT) ~ '^[0-9]+$' THEN CAST(:series_number AS INTEGER) ELSE series_number END, + document_number = COALESCE(:document_number, document_number), + customer_party_key = COALESCE(:customer_party_key, customer_party_key), + status = COALESCE(:status, status), + amount = COALESCE(CAST(:amount AS NUMERIC), amount), + tax_amount = COALESCE(CAST(:tax_amount AS NUMERIC), tax_amount), + total_amount = COALESCE(CAST(:total_amount AS NUMERIC), total_amount), + currency = COALESCE(:currency, currency), + document_date = COALESCE(CAST(:document_date AS DATE), document_date), + due_date = COALESCE(CAST(:due_date AS DATE), due_date), + role = :role, + is_primary = :is_primary, + is_active = TRUE, + payload = COALESCE(payload, '{}'::jsonb) || CAST(:payload AS JSONB), + updated_at = now() + WHERE id = CAST(:id AS UUID) + """), params) + document_id = existing + conn.execute(text("DELETE FROM commercial_document_lines WHERE document_id = CAST(:document_id AS UUID)"), {"document_id": document_id}) + else: + conn.execute(text(""" + INSERT INTO commercial_documents ( + id, customer_id, opportunity_id, system, document_kind, external_id, company, + document_type, serie, series_number, document_number, customer_party_key, + status, amount, tax_amount, total_amount, currency, role, is_primary, document_date, due_date, + payload, updated_at + ) VALUES ( + CAST(:id AS UUID), CAST(:customer_id AS UUID), CAST(:opportunity_id AS UUID), 'jasmin', + :document_kind, :external_id, :company, :document_type, :serie, + CASE WHEN CAST(:series_number AS TEXT) ~ '^[0-9]+$' THEN CAST(:series_number AS INTEGER) ELSE NULL END, + :document_number, :customer_party_key, :status, CAST(:amount AS NUMERIC), + CAST(:tax_amount AS NUMERIC), CAST(:total_amount AS NUMERIC), :currency, + :role, :is_primary, CAST(:document_date AS DATE), CAST(:due_date AS DATE), CAST(:payload AS JSONB), now() + ) + """), params) + document_id = params["id"] + + for line in _jasmin_document_lines_from_item(item): + mapping = _resolve_product_mapping_for_jasmin_line(conn, line) + conn.execute(text(""" + INSERT INTO commercial_document_lines ( + document_id, line_index, local_product_id, jasmin_sales_item, description, + quantity, unit, unit_price, tax_schema, total_amount, payload + ) VALUES ( + CAST(:document_id AS UUID), :line_index, CAST(:local_product_id AS UUID), :jasmin_sales_item, + :description, CAST(:quantity AS NUMERIC), :unit, CAST(:unit_price AS NUMERIC), + :tax_schema, CAST(:total_amount AS NUMERIC), CAST(:payload AS JSONB) + ) + """), { + "document_id": document_id, + "line_index": line.get("line_index"), + "local_product_id": mapping.get("product_id"), + "jasmin_sales_item": mapping.get("jasmin_sales_item") or line.get("jasmin_sales_item"), + "description": line.get("description"), + "quantity": line.get("quantity"), + "unit": line.get("unit"), + "unit_price": line.get("unit_price"), + "tax_schema": line.get("tax_schema"), + "total_amount": line.get("total_amount"), + "payload": _json({"source_line_id": line.get("source_line_id"), "mapping": mapping, "raw": line.get("payload")}), + }) + return document_id + + +def _upsert_opportunity_items_from_jasmin_item(conn: Any, item: Dict[str, Any], opportunity_id: str) -> int: + if _clean(item.get("source_system")) != "jasmin" or not _clean(item.get("external_type")).startswith("jasmin_"): + return 0 + document_ref = _clean(item.get("document_number") or item.get("external_id")) + upserted = 0 + for line in _jasmin_document_lines_from_item(item): + mapping = _resolve_product_mapping_for_jasmin_line(conn, line) + source_line_id = _clean(line.get("source_line_id")) + params = { + "id": str(uuid.uuid4()), + "opportunity_id": opportunity_id, + "product_id": mapping.get("product_id"), + "sku": mapping.get("sku"), + "jasmin_sales_item": mapping.get("jasmin_sales_item") or line.get("jasmin_sales_item"), + "product_name": mapping.get("catalog_name") or line.get("description"), + "description": line.get("description"), + "quantity": line.get("quantity"), + "unit_price": line.get("unit_price"), + "total_price": line.get("total_amount"), + "status": "JASMIN_IMPORTED", + "source_document": document_ref, + "source_line_id": source_line_id, + "metadata": _json({ + "source_system": "jasmin", + "source_document": document_ref, + "source_line_id": source_line_id, + "source_external_id": item.get("external_id"), + "source_external_type": item.get("external_type"), + "resolved_sku": mapping.get("sku"), + "resolved_jasmin_sales_item": mapping.get("jasmin_sales_item"), + "product_mapping_status": mapping.get("mapping_status"), + "catalog_name": mapping.get("catalog_name"), + "price_source": "jasmin_document_line", + }), + } + updated = conn.execute(text(""" + UPDATE opportunity_items + SET product_id = COALESCE(CAST(:product_id AS UUID), product_id), + sku = COALESCE(:sku, sku), + jasmin_sales_item = COALESCE(:jasmin_sales_item, jasmin_sales_item), + product_name = COALESCE(:product_name, product_name), + description = COALESCE(:description, description), + quantity = CAST(:quantity AS NUMERIC), + unit_price = CAST(:unit_price AS NUMERIC), + total_price = CAST(:total_price AS NUMERIC), + status = CASE WHEN status IN ('REJECTED', 'CANCELLED') THEN status ELSE :status END, + metadata = COALESCE(metadata, '{}'::jsonb) || CAST(:metadata AS JSONB), + updated_at = now() + WHERE opportunity_id = CAST(:opportunity_id AS UUID) + AND metadata->>'source_system' = 'jasmin' + AND metadata->>'source_document' = :source_document + AND metadata->>'source_line_id' = :source_line_id + """), params).rowcount or 0 + if updated: + upserted += updated + continue + + inserted = conn.execute(text(""" + INSERT INTO opportunity_items ( + id, opportunity_id, product_id, sku, jasmin_sales_item, product_name, description, + quantity, unit_price, discount_amount, total_price, status, metadata + ) + SELECT + CAST(:id AS UUID), CAST(:opportunity_id AS UUID), CAST(:product_id AS UUID), :sku, + :jasmin_sales_item, :product_name, :description, CAST(:quantity AS NUMERIC), + CAST(:unit_price AS NUMERIC), 0, CAST(:total_price AS NUMERIC), :status, + CAST(:metadata AS JSONB) + WHERE NOT EXISTS ( + SELECT 1 FROM opportunity_items + WHERE opportunity_id = CAST(:opportunity_id AS UUID) + AND metadata->>'source_system' = 'jasmin' + AND metadata->>'source_document' = :source_document + AND metadata->>'source_line_id' = :source_line_id + ) + """), params).rowcount or 0 + upserted += inserted + return upserted + +def _apply_jasmin_documents_to_opportunity(conn: Any, items: List[Dict[str, Any]], opportunity_id: str, *, actor: str) -> Dict[str, Any]: + jasmin_items = [item for item in items if _clean(item.get("source_system")) == "jasmin" and _clean(item.get("external_type")).startswith("jasmin_")] + if not jasmin_items: + return {"documents": 0, "lines": 0} + document_count = 0 + line_count = 0 + product_names: List[str] = [] + best_amount = None + for item in jasmin_items: + if _upsert_jasmin_document_from_item(conn, item, opportunity_id): + document_count += 1 + line_count += _upsert_opportunity_items_from_jasmin_item(conn, item, opportunity_id) + for line in _jasmin_document_lines_from_item(item): + name = _clean(line.get("description")) + if name and name not in product_names: + product_names.append(name) + totals = _jasmin_document_totals(item) + best_amount = best_amount or totals.get("total_amount") or item.get("amount") + product_interest = ", ".join(product_names[:3]) + if len(product_names) > 3: + product_interest += f" +{len(product_names)-3} linha(s)" + conn.execute(text(""" + UPDATE opportunities + SET value_amount = CASE + WHEN CAST(:amount AS NUMERIC) IS NOT NULL AND (value_amount IS NULL OR value_amount = 0) THEN CAST(:amount AS NUMERIC) + ELSE value_amount + END, + product_interest = CASE + WHEN :product_interest <> '' AND (product_interest IS NULL OR product_interest = '' OR product_interest ILIKE 'Processo importado%%') THEN :product_interest + ELSE product_interest + END, + metadata = COALESCE(metadata, '{}'::jsonb) || CAST(:metadata AS JSONB), + updated_at = now() + WHERE id = CAST(:opportunity_id AS UUID) + """), { + "opportunity_id": opportunity_id, + "amount": _money_or_none(best_amount), + "product_interest": product_interest, + "metadata": _json({ + "jasmin_reconciliation_imported": True, + "jasmin_documents_imported": document_count, + "jasmin_lines_imported": line_count, + "jasmin_import_actor": actor, + }), + }) + conn.execute(text(""" + UPDATE opportunities + SET value_amount = COALESCE(( + SELECT SUM(total_price) + FROM opportunity_items + WHERE opportunity_id = CAST(:opportunity_id AS UUID) + AND status NOT IN ('REJECTED', 'CANCELLED') + ), value_amount, 0), + updated_at = now() + WHERE id = CAST(:opportunity_id AS UUID) + AND (value_amount IS NULL OR value_amount = 0) + AND EXISTS (SELECT 1 FROM opportunity_items WHERE opportunity_id = CAST(:opportunity_id AS UUID)) + """), {"opportunity_id": opportunity_id}) + return {"documents": document_count, "lines": line_count, "product_interest": product_interest} + + +def _ensure_pending_task_for_reconstruction(conn: Any, opportunity_id: str, action_code: str, *, note: str, actor: str) -> None: + action_code = _clean(action_code).upper() or "REVIEW_MANUALLY" + exists = conn.execute(text(""" + SELECT id::text FROM tasks + WHERE opportunity_id = CAST(:opportunity_id AS UUID) + AND status = 'pending' + AND action_code = :action_code + LIMIT 1 + """), {"opportunity_id": opportunity_id, "action_code": action_code}).scalar() + if exists: + return + try: + config = get_action_config(action_code) + conn.execute(text(""" + INSERT INTO tasks ( + id, opportunity_id, action_code, route, action, note, action_required, + safe_to_post, status, source_system, source_event_id, idempotency_key, metadata, created_at, updated_at + ) VALUES ( + CAST(:id AS UUID), CAST(:opportunity_id AS UUID), :action_code, :route, :action, :note, TRUE, + FALSE, 'pending', 'reconciliation_process', :source_event_id, :idempotency_key, CAST(:metadata AS JSONB), now(), now() + ) + ON CONFLICT (idempotency_key) DO NOTHING + """), { + "id": str(uuid.uuid4()), + "opportunity_id": opportunity_id, + "action_code": action_code, + "route": config.get("route") or "financeiro", + "action": config.get("action") or action_code, + "note": note, + "source_event_id": f"apply:{opportunity_id}:{action_code}", + "idempotency_key": f"task:reconciliation_apply:{opportunity_id}:{action_code}", + "metadata": _json({"source": "reconciliation_apply", "actor": actor}), + }) + except Exception as exc: # pragma: no cover - production safety guard + logger.warning("failed to create reconstruction task for %s: %s", opportunity_id, exc) + +def _apply_reconstructed_process_to_opportunity(conn: Any, items: List[Dict[str, Any]], opportunity_id: str, *, actor: str) -> Dict[str, Any]: + """Apply reconstructed evidence to the opportunity summary/pipeline. + + Linking is no longer just a loose reference: Odoo/Jasmin evidence updates + operation_links, imports Odoo lines into Produtos, writes timeline events + and moves the visual stage to the most advanced reconstructed step. + """ + state = infer_reconciliation_process_state(items) + stage = _clean(state.get("stage")) or "REVIEW" + action_code = _clean(state.get("action_code")) or "REVIEW_MANUALLY" + amount = next((item.get("amount") for item in items if item.get("amount") is not None), None) + amount_value = _money_or_none(amount) + odoo_items = [item for item in items if str(item.get("external_type") or "") == "odoo_sale_order"] + for item in items: + _upsert_external_customer_mapping_from_process(conn, item, opportunity_id, actor=actor) + _upsert_odoo_operation_links_from_item(conn, item, opportunity_id) + _upsert_opportunity_items_from_odoo_item(conn, item, opportunity_id) + jasmin_import = _apply_jasmin_documents_to_opportunity(conn, items, opportunity_id, actor=actor) + + metadata_payload = { + "reconstruction_applied": True, + "actor": actor, + "suggested_stage": stage, + "suggested_action": action_code, + "item_ids": [str(item.get("id")) for item in items if item.get("id")], + "jasmin_import": jasmin_import, + } + conn.execute(text(""" + UPDATE opportunities + SET stage = :stage, + last_action_code = :action_code, + value_amount = CASE + WHEN CAST(:amount AS NUMERIC) IS NOT NULL AND (value_amount IS NULL OR value_amount = 0) THEN CAST(:amount AS NUMERIC) + ELSE value_amount + END, + metadata = COALESCE(metadata, '{}'::jsonb) || CAST(:metadata AS JSONB), + updated_at = now() + WHERE id = CAST(:opportunity_id AS UUID) + """), { + "opportunity_id": opportunity_id, + "stage": stage, + "action_code": action_code, + "amount": amount_value, + "metadata": _json(metadata_payload), + }) + + # If Odoo lines were imported and the opportunity value was still zero, + # recalculate from opportunity_items. This preserves an existing manual + # value when one has already been set. + conn.execute(text(""" + UPDATE opportunities + SET value_amount = COALESCE(( + SELECT SUM(total_price) + FROM opportunity_items + WHERE opportunity_id = CAST(:opportunity_id AS UUID) + AND status NOT IN ('REJECTED', 'CANCELLED') + ), value_amount, 0), + updated_at = now() + WHERE id = CAST(:opportunity_id AS UUID) + AND (value_amount IS NULL OR value_amount = 0) + AND EXISTS (SELECT 1 FROM opportunity_items WHERE opportunity_id = CAST(:opportunity_id AS UUID)) + """), {"opportunity_id": opportunity_id}) + + if odoo_items and action_code == "SEND_INVOICE": + _ensure_pending_task_for_reconstruction( + conn, + opportunity_id, + action_code, + note="Processo Odoo reconstruído: encomenda/entrega encontrada e fatura por emitir.", + actor=actor, + ) + return {"stage": stage, "action_code": action_code, "amount": amount_value} + + +PROCESS_STEP_BY_EXTERNAL_TYPE = { + "manual_request": (5, "Pedido externo registado", "QUOTE_REQUESTED", "SEND_QUOTE"), + "jasmin_quotation": (20, "Orçamento encontrado no Jasmin", "QUOTE_SENT", "SEND_PROFORMA"), + "jasmin_proforma": (30, "Pró-forma encontrada no Jasmin", "WAITING_PAYMENT", "CONFIRM_PAYMENT"), + "payment_proof": (40, "Comprovativo de pagamento recebido", "WAITING_PAYMENT", "CONFIRM_PAYMENT"), + "jasmin_invoice": (50, "Fatura encontrada no Jasmin", "INVOICE_SENT", "CONFIRM_PAYMENT"), + "odoo_sale_order": (60, "Venda/encomenda encontrada no Odoo", "ODOO_ORDER_CREATED", "SEND_INVOICE"), + "packlink_shipment": (80, "Envio encontrado na Packlink", "SHIPMENT_CREATED", "REVIEW_MANUALLY"), +} + + +def _process_identity_keys(item: Dict[str, Any]) -> List[Tuple[str, str]]: + """Return all usable identities for grouping external evidence. + + The fiscal customer is still the strongest identity, but real-world data is + uneven: Jasmin usually has NIF while Odoo may only have the partner/company + name. v4.9.17 therefore keeps all identities for the same item and lets the + grouping layer merge evidence by any shared key. + + Example: a Jasmin quote with NIF + name and an Odoo sale with only the same + normalized name become one process candidate instead of a quote candidate + with the Odoo sale shown merely as a suggestion. + """ + keys: List[Tuple[str, str]] = [] + tax_id = _normalize_tax_id(item.get("customer_tax_id") or "") + if tax_id: + keys.append(("nif", tax_id)) + email = _norm_match_value(item.get("customer_email")) + if email: + keys.append(("email", email)) + name = _normalize_company_name(item.get("customer_name") or item.get("linked_customer_name")) + if name and len(name) >= 5: + keys.append(("name", name)) + return keys + + +def _process_identity_key(item: Dict[str, Any]) -> Optional[Tuple[str, str]]: + """Backward-compatible strongest identity for older callers/tests.""" + keys = _process_identity_keys(item) + return keys[0] if keys else None + + +def _best_process_group_key(items: List[Dict[str, Any]]) -> Optional[Tuple[str, str]]: + """Pick the strongest display key for a merged process group.""" + priority = {"nif": 0, "email": 1, "name": 2} + keys: List[Tuple[str, str]] = [] + for item in items: + keys.extend(_process_identity_keys(item)) + if not keys: + return None + # Deduplicate while preserving deterministic ordering by strength/value. + unique = sorted(set(keys), key=lambda x: (priority.get(x[0], 99), x[1])) + return unique[0] + + +def _base_process_step(item: Dict[str, Any]) -> Dict[str, Any]: + external_type = str(item.get("external_type") or "external_record") + rank, label, stage, action_code = PROCESS_STEP_BY_EXTERNAL_TYPE.get( + external_type, + (10, "Evidência externa encontrada", STAGE_BY_EXTERNAL_TYPE.get(external_type, "REVIEW"), NEXT_ACTION_BY_EXTERNAL_TYPE.get(external_type, "REVIEW_MANUALLY")), + ) + return { + "rank": rank, + "label": label, + "stage": stage, + "action_code": item.get("suggested_action") or action_code, + "item_id": item.get("id"), + "source_system": item.get("source_system"), + "external_type": external_type, + "document_number": item.get("document_number") or item.get("external_id"), + "document_date": item.get("document_date"), + "amount": item.get("amount"), + "currency": item.get("currency") or "EUR", + "title": item.get("title"), + } + + +def _process_steps_for_item(item: Dict[str, Any]) -> List[Dict[str, Any]]: + base = _base_process_step(item) + if str(item.get("external_type") or "") != "odoo_sale_order": + return [base] + steps = [base] + fulfilment = _odoo_fulfilment_from_item(item) + lines = fulfilment.get("lines") if isinstance(fulfilment.get("lines"), list) else [] + if lines: + steps.append({**base, "rank": 62, "label": "Linhas/produtos importados do Odoo", "stage": "ODOO_ORDER_CREATED", "action_code": "SEND_INVOICE", "external_type": "odoo_sale_lines", "products": [line.get("product_name") or line.get("description") for line in lines if line.get("product_name") or line.get("description")]}) + outgoing = fulfilment.get("outgoing_pickings") if isinstance(fulfilment.get("outgoing_pickings"), list) else [] + done_pickings = [p for p in outgoing if _clean(p.get("state")) == "done"] + ready_pickings = [p for p in outgoing if _clean(p.get("state")) == "assigned"] + if done_pickings: + for picking in done_pickings: + steps.append({**base, "rank": 75, "label": "Entrega Odoo concluída", "stage": "SHIPMENT_CREATED", "action_code": "SEND_INVOICE", "external_type": "odoo_delivery", "document_number": picking.get("name"), "document_date": _clean(picking.get("date_done"))[:10] or base.get("document_date"), "picking_state": picking.get("state")}) + elif ready_pickings: + for picking in ready_pickings: + steps.append({**base, "rank": 70, "label": "Entrega Odoo pronta para despacho", "stage": "READY_TO_SHIP", "action_code": "SEND_INVOICE", "external_type": "odoo_delivery", "document_number": picking.get("name"), "document_date": _clean(picking.get("scheduled_date"))[:10] or base.get("document_date"), "picking_state": picking.get("state")}) + if fulfilment.get("invoice_pending"): + steps.append({**base, "rank": 85, "label": "Fatura por emitir", "stage": "SHIPMENT_CREATED" if done_pickings else "ODOO_ORDER_CREATED", "action_code": "SEND_INVOICE", "external_type": "odoo_invoice_pending", "document_number": base.get("document_number")}) + return steps + + +def _process_step(item: Dict[str, Any]) -> Dict[str, Any]: + return _process_steps_for_item(item)[0] + + +def _operation_document_ref(item: Dict[str, Any]) -> str: + """Stable document/order reference used to avoid merging two orders. + + Customer identity tells us who the company is; this reference tells us which + commercial operation the evidence belongs to. It is deliberately based on + external document/order numbers rather than the customer name. + """ + record = _payload_record(item) + return _clean( + record.get("name") + or item.get("document_number") + or record.get("id") + or item.get("external_id") + ) + + +def _operation_primary_anchor(item: Dict[str, Any]) -> str: + """Return a strong operation anchor for a specific purchase/process. + + Fiscal identity answers "who is the customer". This anchor answers "which + purchase/process is this evidence about". Odoo sales are always anchors, + and Jasmin commercial documents are anchors too so two quotations for the + same fiscal customer do not collapse into one giant opportunity. + """ + external_type = str(item.get("external_type") or "") + ref = _operation_document_ref(item) + anchor_types = { + "odoo_sale_order", + "jasmin_quotation", + "jasmin_proforma", + "jasmin_invoice", + } + if external_type in anchor_types and ref: + return f"{external_type}:{ref}" + return "" + + +def _date_prefix(value: Any) -> str: + return _clean(value)[:10] + + +def _date_distance_days(a: Any, b: Any) -> Optional[int]: + a_value = _date_prefix(a) + b_value = _date_prefix(b) + if not a_value or not b_value: + return None + try: + return abs((datetime.fromisoformat(a_value).date() - datetime.fromisoformat(b_value).date()).days) + except Exception: + return None + + +def _operation_match_score(item: Dict[str, Any], anchor: Dict[str, Any]) -> int: + """Score whether an evidence item belongs to a specific purchase anchor.""" + if item is anchor or item.get("id") == anchor.get("id"): + return 1000 + score = 0 + item_ref = _operation_document_ref(item) + anchor_ref = _operation_document_ref(anchor) + if item_ref and anchor_ref and item_ref == anchor_ref: + score += 100 + + item_amount = _money_or_none(item.get("amount")) + anchor_amount = _money_or_none(anchor.get("amount")) + if item_amount is not None and anchor_amount is not None: + try: + item_dec = Decimal(item_amount) + anchor_dec = Decimal(anchor_amount) + if item_dec == anchor_dec: + score += 70 + elif anchor_dec and item_dec >= (anchor_dec * Decimal("0.90")) and item_dec <= (anchor_dec * Decimal("1.10")): + score += 35 + except (InvalidOperation, ValueError, TypeError): + pass + + distance = _date_distance_days(item.get("document_date"), anchor.get("document_date")) + if distance is not None: + if distance == 0: + score += 25 + elif distance <= 3: + score += 20 + elif distance <= 10: + score += 12 + elif distance <= 30: + score += 5 + + # Odoo fulfilment lines are often the best differentiator when the same + # company places two orders close together. Only count explicit overlap. + item_products = { + _normalize_product_name(line.get("product_name") or line.get("description")) + for line in _odoo_importable_lines(item) + if _normalize_product_name(line.get("product_name") or line.get("description")) + } + anchor_products = { + _normalize_product_name(line.get("product_name") or line.get("description")) + for line in _odoo_importable_lines(anchor) + if _normalize_product_name(line.get("product_name") or line.get("description")) + } + if item_products and anchor_products and item_products & anchor_products: + score += 25 + return score + + +def _operation_anchors_can_merge(left: Dict[str, Any], right: Dict[str, Any]) -> bool: + """Return true when two purchase anchors are clearly the same process. + + Conservative rules: + - Same external document/order reference is a hard match. + - Two Odoo sale orders with different refs are always different purchases. + - Two anchors of the same document class with different refs remain separate. + - Different document classes may merge only with strong amount/date/product + evidence, e.g. quote ORC2026.154 + sale S00279 for the same amount. + """ + left_ref = _operation_document_ref(left) + right_ref = _operation_document_ref(right) + if left_ref and right_ref and left_ref == right_ref: + return True + + left_type = str(left.get("external_type") or "") + right_type = str(right.get("external_type") or "") + if left_type == "odoo_sale_order" and right_type == "odoo_sale_order": + return False + if left_type == right_type: + return False + + score = _operation_match_score(left, right) + if score >= 90: + return True + + # Backward-compatible fiscal-name bridge: when Jasmin gives the NIF/name + # and Odoo only gives the name, a single Odoo sale close to the Jasmin + # commercial document should still reconstruct one process even when the + # Jasmin amount was not imported. Multiple Odoo orders remain separated by + # their own anchors and loose items must still match unambiguously. + if {left_type, right_type} & {"odoo_sale_order"}: + distance = _date_distance_days(left.get("document_date"), right.get("document_date")) + left_amount = _money_or_none(left.get("amount")) + right_amount = _money_or_none(right.get("amount")) + if distance is not None and distance <= 3 and (left_amount is None or right_amount is None): + return True + return False + + +def _split_items_by_operation_identity(items: List[Dict[str, Any]]) -> List[List[Dict[str, Any]]]: + """Split one fiscal customer group into separate commercial purchases. + + The first pass intentionally merges evidence by fiscal identity + (NIF/email/fiscal name). This second pass prevents "one customer = one + process" mistakes. Every Odoo sale and every Jasmin commercial document is + a purchase anchor; anchors are merged only when there is strong evidence + they are the same purchase. Payments/shipments/manual evidence are assigned + only when the best amount/date/reference/product match is unambiguous. + """ + anchors = [item for item in items if _operation_primary_anchor(item)] + if len(anchors) <= 1: + return [items] + + parent = list(range(len(anchors))) + + def find(idx: int) -> int: + while parent[idx] != idx: + parent[idx] = parent[parent[idx]] + idx = parent[idx] + return idx + + def union(a: int, b: int) -> None: + ra, rb = find(a), find(b) + if ra != rb: + parent[rb] = ra + + for i, left in enumerate(anchors): + for j, right in enumerate(anchors[i + 1:], start=i + 1): + if _operation_anchors_can_merge(left, right): + union(i, j) + + grouped_anchor_indexes: Dict[int, List[int]] = {} + for idx in range(len(anchors)): + grouped_anchor_indexes.setdefault(find(idx), []).append(idx) + + buckets: List[Dict[str, Any]] = [] + for indexes in grouped_anchor_indexes.values(): + anchor_items = [anchors[idx] for idx in indexes] + # Prefer Odoo sale order as the visible purchase key, then quotation, + # proforma and invoice. This keeps UI labels stable for real orders. + anchor_items.sort(key=lambda item: { + "odoo_sale_order": 0, + "jasmin_quotation": 1, + "jasmin_proforma": 2, + "jasmin_invoice": 3, + }.get(str(item.get("external_type") or ""), 9)) + buckets.append({"anchor": anchor_items[0], "items": anchor_items, "key": _operation_primary_anchor(anchor_items[0])}) + + anchor_ids = {str(anchor.get("id") or id(anchor)) for anchor in anchors} + standalone: List[List[Dict[str, Any]]] = [] + + for item in items: + if str(item.get("id") or id(item)) in anchor_ids: + continue + scored = sorted( + ((_operation_match_score(item, bucket["anchor"]), idx) for idx, bucket in enumerate(buckets)), + key=lambda pair: pair[0], + reverse=True, + ) + best_score, best_idx = scored[0] + second_score = scored[1][0] if len(scored) > 1 else -1 + if best_score >= 70 and best_score > second_score: + buckets[best_idx]["items"].append(item) + else: + standalone.append([item]) + + return [bucket["items"] for bucket in buckets] + standalone + +def _operation_group_suffix(items: List[Dict[str, Any]]) -> str: + preferred_types = ["odoo_sale_order", "jasmin_quotation", "jasmin_proforma", "jasmin_invoice"] + for preferred_type in preferred_types: + for item in items: + if str(item.get("external_type") or "") != preferred_type: + continue + anchor = _operation_primary_anchor(item) + if anchor: + return anchor + for item in items: + ref = _operation_document_ref(item) + if ref: + return f"document:{ref}" + return "" + + +def infer_reconciliation_process_state(items: List[Dict[str, Any]]) -> Dict[str, Any]: + """Infer a conservative process stage/next action from grouped evidence.""" + steps = sorted([step for item in items for step in _process_steps_for_item(item)], key=lambda x: (x.get("rank") or 0, str(x.get("document_date") or ""))) + if not steps: + return {"stage": "REVIEW", "action_code": "REVIEW_MANUALLY", "label": "Processo por rever"} + strongest = max(steps, key=lambda x: int(x.get("rank") or 0)) + stage = strongest.get("stage") or "REVIEW" + action_code = strongest.get("action_code") or "REVIEW_MANUALLY" + + # If a sale/order already exists, the important next check is usually + # invoice/payment status, not creating another commercial document. + external_types = {str(item.get("external_type") or "") for item in items} + if "odoo_sale_order" in external_types and "jasmin_invoice" not in external_types: + action_code = "SEND_INVOICE" + if any(step.get("external_type") == "odoo_delivery" for step in steps): + stage = "SHIPMENT_CREATED" + if "payment_proof" in external_types: + action_code = "CONFIRM_PAYMENT" + if "packlink_shipment" in external_types and "jasmin_invoice" not in external_types: + action_code = "SEND_INVOICE" + + return { + "stage": stage, + "action_code": action_code, + "label": strongest.get("label") or "Processo reconstruído", + "steps": steps, + } + + +def _process_amount_values(items: List[Dict[str, Any]]) -> List[Decimal]: + values: List[Decimal] = [] + for item in items: + value = _money_or_none(item.get("amount")) + if value is None: + continue + try: + values.append(Decimal(value)) + except (InvalidOperation, ValueError, TypeError): + continue + return values + + +def _candidate_reasons_and_risks(group: Dict[str, Any], items: List[Dict[str, Any]], *, identity_reason: str, confidence_label: str) -> Dict[str, Any]: + """Explain why a process candidate was proposed and what needs review. + + The operator should never have to trust an opaque confidence label. This + helper produces short, UI-ready reasons and risks separated from the final + state/action inference. It is deliberately conservative: customer identity + can be high-confidence while the purchase grouping still needs review. + """ + reasons: List[str] = [] + risks: List[str] = [] + external_types = {str(item.get("external_type") or "") for item in items} + + if identity_reason: + reasons.append(f"cliente fiscal por {identity_reason}") + if group.get("operation_key"): + reasons.append("compra/processo separado por referência documental") + if group.get("split_from_multi_purchase"): + reasons.append("cliente com várias compras detectadas; este cartão foi separado") + risks.append("mesmo cliente fiscal tem outros processos na janela") + if len(items) > 1: + reasons.append(f"{len(items)} evidências agrupadas") + if group.get("suggestions"): + reasons.append("existe sugestão de oportunidade aberta") + + amounts = _process_amount_values(items) + if len(amounts) >= 2: + min_amount, max_amount = min(amounts), max(amounts) + if min_amount == max_amount: + reasons.append("valor igual entre documentos") + elif min_amount and max_amount <= (min_amount * Decimal("1.10")): + reasons.append("valor aproximado entre documentos") + else: + risks.append("valores diferentes entre documentos") + + dated_items = [item for item in items if item.get("document_date")] + if len(dated_items) >= 2: + distances = [ + _date_distance_days(left.get("document_date"), right.get("document_date")) + for pos, left in enumerate(dated_items) + for right in dated_items[pos + 1:] + ] + distances = [d for d in distances if d is not None] + if distances: + if min(distances) <= 3: + reasons.append("datas próximas") + if max(distances) > 30: + risks.append("datas afastadas; pode ser histórico ou compra diferente") + + if "odoo_sale_order" in external_types and "jasmin_invoice" not in external_types: + risks.append("venda Odoo sem fatura Jasmin associada") + if "jasmin_invoice" in external_types and not ({"odoo_sale_order", "jasmin_quotation", "jasmin_proforma"} & external_types): + risks.append("fatura solta; confirmar se é histórico") + if "payment_proof" in external_types and "jasmin_invoice" not in external_types: + risks.append("comprovativo sem fatura associada") + if not any(_normalize_tax_id(item.get("customer_tax_id") or "") for item in items): + risks.append("cliente sem NIF; validar nome fiscal") + if confidence_label != "alta": + risks.append("confiança de cliente/processo não é alta") + + # Deterministic de-duplication with stable order. + reasons = list(dict.fromkeys([r for r in reasons if r])) + risks = list(dict.fromkeys([r for r in risks if r])) + if risks and any("valores diferentes" in risk or "compra diferente" in risk for risk in risks): + review_status = "conflict" + elif risks: + review_status = "needs_review" + else: + review_status = "ready" + return {"reasons": reasons, "risks": risks, "review_status": review_status} + + +def _record_reconciliation_decision( + conn: Any, + *, + decision_type: str, + item_ids: Optional[List[str]] = None, + status: Optional[str] = None, + opportunity_id: Optional[str] = None, + customer_id: Optional[str] = None, + note: str = "", + actor: str = "operator", + payload: Optional[Dict[str, Any]] = None, +) -> None: + """Persist an operator/system decision without failing core actions. + + Long-lived installations may receive this table through the additive schema + guard. If a local database is inconsistent, the original link/create/ignore + action should still complete, so failures are logged only. + """ + try: + conn.execute(text(""" + INSERT INTO reconciliation_decisions ( + decision_type, status, item_ids, opportunity_id, customer_id, + note, actor, payload + ) VALUES ( + :decision_type, :status, CAST(:item_ids AS TEXT[]), + CAST(:opportunity_id AS UUID), CAST(:customer_id AS UUID), + :note, :actor, CAST(:payload AS JSONB) + ) + """), { + "decision_type": _clean(decision_type) or "unknown", + "status": _clean(status) or None, + "item_ids": [str(x) for x in (item_ids or []) if str(x)], + "opportunity_id": _uuid_or_none(opportunity_id), + "customer_id": _uuid_or_none(customer_id), + "note": _clean(note) or None, + "actor": _clean(actor) or "operator", + "payload": _json(payload or {}), + }) + except Exception as exc: # pragma: no cover - production safety guard + logger.warning("failed to record reconciliation decision %s: %s", decision_type, exc) + + +def list_reconciliation_process_candidates(*, status: str = "open", days: int = 3, limit: int = 20) -> List[Dict[str, Any]]: + """Return grouped process candidates built from recent open items. + + A candidate is shown when at least two pieces of evidence share the same + NIF/email/name, when one item already has a strong open-operation + suggestion, or when one important external item is actionable by itself + (for example an Odoo sale order without ClientFlow). + """ + try: + items = list_reconciliation_items(status=status, limit=300, days=days) + except Exception as exc: # pragma: no cover - production safety guard + logger.warning("failed to list reconciliation items for process candidates: %s", exc) + return [] + # Build connected components from every usable identity key. NIF is absolute + # and normalized company fiscal name is also a strong business key in this + # project. This avoids splitting one real process into separate cards when + # Jasmin has NIF + name and Odoo has only the company name. + indexed_items: List[Dict[str, Any]] = [] + parent: List[int] = [] + key_owner: Dict[str, int] = {} + + def find(idx: int) -> int: + while parent[idx] != idx: + parent[idx] = parent[parent[idx]] + idx = parent[idx] + return idx + + def union(a: int, b: int) -> None: + ra, rb = find(a), find(b) + if ra != rb: + parent[rb] = ra + + for item in items: + keys = _process_identity_keys(item) + if not keys: + continue + idx = len(indexed_items) + indexed_items.append(item) + parent.append(idx) + for key_name, key_value in keys: + key = f"{key_name}:{key_value}" + if key in key_owner: + union(idx, key_owner[key]) + else: + key_owner[key] = idx + + # Fiscal-name bridge for cross-source reconstruction. Jasmin often carries + # the fiscal NIF + canonical company name, while Odoo may only carry a + # partner/display name with extra location text and no NIF. Exact identity + # keys above do not merge those records when the names are not identical. + # If at least one side has a NIF and the normalized names are a strong + # substring/overlap match, merge them into the same process. The merged + # group will still use the NIF as display key through _best_process_group_key. + name_candidates: List[Tuple[int, str, bool, str]] = [] + for idx, item in enumerate(indexed_items): + name_key = _normalize_company_name(item.get("customer_name") or item.get("linked_customer_name")) + if len(name_key) < 5: + continue + has_tax = bool(_normalize_tax_id(item.get("customer_tax_id") or "")) + source = str(item.get("source_system") or "") + name_candidates.append((idx, name_key, has_tax, source)) + + for pos, (idx_a, name_a, has_tax_a, source_a) in enumerate(name_candidates): + for idx_b, name_b, has_tax_b, source_b in name_candidates[pos + 1:]: + if find(idx_a) == find(idx_b): + continue + if not (has_tax_a or has_tax_b): + continue + # Prefer bridging different sources; same-source fuzzy merges are + # riskier and can accidentally collapse unrelated customer records. + if source_a and source_b and source_a == source_b: + continue + score = _name_match_score(name_a, name_b) + if score >= 35: + union(idx_a, idx_b) + + grouped_items: Dict[int, List[Dict[str, Any]]] = {} + for idx, item in enumerate(indexed_items): + grouped_items.setdefault(find(idx), []).append(item) + + groups: Dict[str, Dict[str, Any]] = {} + for identity_items in grouped_items.values(): + split_groups = _split_items_by_operation_identity(identity_items) + multiple_operation_groups = len(split_groups) > 1 + for raw_items in split_groups: + best_key = _best_process_group_key(raw_items) + if not best_key: + continue + key_name, key_value = best_key + identity_group_key = f"{key_name}:{key_value}" + operation_suffix = _operation_group_suffix(raw_items) if multiple_operation_groups else "" + group_key = f"{identity_group_key}|{operation_suffix}" if operation_suffix else identity_group_key + group = { + "process_key": group_key, + "match_key": key_name, + "match_value": key_value, + "operation_key": operation_suffix, + "split_from_multi_purchase": multiple_operation_groups, + "items": [], + "suggestions": [], + } + for item in raw_items: + group["items"].append(item) + for suggestion in item.get("operation_suggestions") or []: + if suggestion.get("opportunity_id") and all(existing.get("opportunity_id") != suggestion.get("opportunity_id") for existing in group["suggestions"]): + group["suggestions"].append(suggestion) + groups[group_key] = group + + candidates: List[Dict[str, Any]] = [] + for group in groups.values(): + group_items = sorted( + group["items"], + key=lambda x: ( + str(x.get("document_date") or x.get("updated_at") or ""), + min((int(step.get("rank") or 0) for step in _process_steps_for_item(x)), default=0), + ), + ) + external_types = {str(item.get("external_type") or "") for item in group_items} + actionable_singleton = bool(external_types & { + "odoo_sale_order", + "jasmin_invoice", + "jasmin_proforma", + "payment_proof", + "packlink_shipment", + }) + split_purchase_singleton = bool(group.get("split_from_multi_purchase")) and bool(external_types & { + "jasmin_quotation", + "manual_request", + "external_record", + }) + if len(group_items) < 2 and not group.get("suggestions") and not actionable_singleton and not split_purchase_singleton: + continue + state = infer_reconciliation_process_state(group_items) + first = group_items[0] + customer_name = next((item.get("customer_name") for item in group_items if item.get("customer_name")), None) or first.get("linked_customer_name") or "Cliente externo" + customer_email = next((item.get("customer_email") for item in group_items if item.get("customer_email")), None) or "" + customer_tax_id = next((item.get("customer_tax_id") for item in group_items if item.get("customer_tax_id")), None) or "" + total_amount = next((item.get("amount") for item in group_items if item.get("amount") is not None), None) + confidence_label, identity_reason = _process_group_identity_label(str(group.get("match_key") or ""), group_items) + explanation = _candidate_reasons_and_risks(group, group_items, identity_reason=identity_reason, confidence_label=confidence_label) + candidates.append({ + "process_key": group["process_key"], + "match_key": group["match_key"], + "match_value": group["match_value"], + "operation_key": group.get("operation_key") or "", + "customer_name": customer_name, + "customer_email": customer_email, + "customer_tax_id": customer_tax_id, + "amount": total_amount, + "currency": first.get("currency") or "EUR", + "item_ids": [str(item.get("id")) for item in group_items if item.get("id")], + "items": group_items, + "timeline": state.get("steps") or [], + "suggested_stage": state.get("stage"), + "suggested_action": state.get("action_code"), + "suggested_label": state.get("label"), + "suggestions": group.get("suggestions") or [], + "confidence": confidence_label, + "identity_reason": identity_reason, + "reasons": explanation.get("reasons") or [], + "risks": explanation.get("risks") or [], + "review_status": explanation.get("review_status") or "needs_review", + }) + candidates.sort(key=lambda x: (0 if x.get("confidence") == "alta" else 1, -len(x.get("items") or []), str(x.get("customer_name") or ""))) + return candidates[: max(int(limit or 20), 1)] + + +def _get_reconciliation_items_by_ids(item_ids: List[str]) -> List[Dict[str, Any]]: + ensure_reconciliation_schema() + cleaned = [str(x).strip() for x in item_ids if str(x).strip()] + if not cleaned: + return [] + with engine.begin() as conn: + rows = conn.execute(text(""" + SELECT id::text, source_system, external_type, external_id, title, description, + status, priority, suggested_action, confidence, opportunity_id::text, + customer_id::text, customer_name, customer_email, customer_tax_id, document_number, + document_date, amount, currency, payload, resolution_note, created_at, + updated_at, resolved_at + FROM reconciliation_items + WHERE id = ANY(CAST(:ids AS UUID[])) + AND status IN ('open','needs_review','conflict') + ORDER BY document_date NULLS FIRST, created_at + """), {"ids": cleaned}).mappings().all() + return [dict(row) for row in rows] + + +def link_reconciliation_process_to_opportunity(item_ids: List[str], opportunity_id: str, *, actor: str = "operator") -> int: + """Link several reconciliation items to the same opportunity as one process.""" + items = _get_reconciliation_items_by_ids(item_ids) + if not items or not _uuid_or_none(opportunity_id): + return 0 + state = infer_reconciliation_process_state(items) + with engine.begin() as conn: + for item in items: + conn.execute(text(""" + UPDATE reconciliation_items + SET opportunity_id = CAST(:opportunity_id AS UUID), + status = 'linked', + resolved_at = now(), + updated_at = now(), + payload = COALESCE(payload, '{}'::jsonb) || CAST(:payload AS JSONB) + WHERE id = CAST(:id AS UUID) + """), { + "id": item.get("id"), + "opportunity_id": opportunity_id, + "payload": _json({"process_reconstruction": {"linked_by": actor, "suggested_stage": state.get("stage"), "suggested_action": state.get("action_code")}}), + }) + _upsert_external_customer_mapping_from_process(conn, item, opportunity_id, actor=actor) + _upsert_odoo_operation_links_from_item(conn, item, opportunity_id) + _upsert_opportunity_items_from_odoo_item(conn, item, opportunity_id) + _apply_jasmin_documents_to_opportunity(conn, [item], opportunity_id, actor=actor) + for step in _process_steps_for_item(item): + conn.execute(text(""" + INSERT INTO opportunity_events ( + id, opportunity_id, event_type, action_code, to_stage, note, payload, created_by + ) VALUES ( + CAST(:event_id AS UUID), CAST(:opportunity_id AS UUID), + 'reconciliation_evidence_imported', :action_code, :to_stage, + :note, CAST(:payload AS JSONB), :created_by + ) + """), { + "event_id": str(uuid.uuid4()), + "opportunity_id": opportunity_id, + "action_code": step.get("action_code") or state.get("action_code"), + "to_stage": step.get("stage") or state.get("stage"), + "note": f"{step.get('label') or 'Evidência externa ligada'}: {step.get('document_number') or item.get('document_number') or item.get('external_id')}", + "payload": _json({"reconciliation_item_id": item.get("id"), "source_system": item.get("source_system"), "external_type": step.get("external_type") or item.get("external_type"), "external_id": item.get("external_id"), "document_number": step.get("document_number") or item.get("document_number"), "document_date": step.get("document_date") or item.get("document_date"), "amount": item.get("amount"), "step": step}), + "created_by": actor, + }) + _apply_reconstructed_process_to_opportunity(conn, items, opportunity_id, actor=actor) + _record_reconciliation_decision( + conn, + decision_type="link_process", + item_ids=[str(item.get("id")) for item in items if item.get("id")], + status="linked", + opportunity_id=opportunity_id, + actor=actor, + payload={"suggested_stage": state.get("stage"), "suggested_action": state.get("action_code")}, + ) + return len(items) + + +def create_opportunity_from_reconciliation_process(item_ids: List[str], *, actor: str = "operator") -> Optional[str]: + """Create one reconstructed opportunity from several external items.""" + items = _get_reconciliation_items_by_ids(item_ids) + if not items: + return None + state = infer_reconciliation_process_state(items) + first = items[0] + customer_name = next((item.get("customer_name") for item in items if item.get("customer_name")), None) or "Processo reconstruído" + customer_email = next((item.get("customer_email") for item in items if item.get("customer_email")), None) or None + customer_id = next((item.get("customer_id") for item in items if item.get("customer_id")), None) + amount_value = next((item.get("amount") for item in items if item.get("amount") is not None), None) + currency = next((item.get("currency") for item in items if item.get("currency")), None) or "EUR" + action_code = state.get("action_code") or "REVIEW_MANUALLY" + stage = state.get("stage") or "REVIEW" + evidence = [{ + "reconciliation_item_id": item.get("id"), + "source_system": item.get("source_system"), + "external_type": item.get("external_type"), + "external_id": item.get("external_id"), + "document_number": item.get("document_number"), + "document_date": item.get("document_date"), + "amount": item.get("amount"), + } for item in items] + opportunity_id = str(uuid.uuid4()) + metadata = { + "created_from_reconciliation_process": True, + "source_system": "reconciliation", + "evidence": evidence, + "suggested_stage": stage, + "suggested_action": action_code, + } + title = f"Processo reconstruído · {customer_name}" + with engine.begin() as conn: + conn.execute(text(""" + INSERT INTO opportunities ( + id, title, stage, status, local_customer_id, customer_name, + customer_email, product_interest, value_amount, currency, + source_system, source_event_id, last_action_code, metadata + ) VALUES ( + CAST(:id AS UUID), :title, :stage, 'open', CAST(:customer_id AS UUID), + :customer_name, :customer_email, :product_interest, :value_amount, + :currency, 'reconciliation', :source_event_id, :last_action_code, + CAST(:metadata AS JSONB) + ) + """), { + "id": opportunity_id, + "title": title, + "stage": stage, + "customer_id": _uuid_or_none(customer_id), + "customer_name": customer_name, + "customer_email": customer_email, + "product_interest": "Processo importado/reconstruído", + "value_amount": _money_or_none(amount_value), + "currency": currency, + "source_event_id": first.get("external_id") or first.get("id"), + "last_action_code": action_code, + "metadata": _json(metadata), + }) + conn.execute(text(""" + INSERT INTO opportunity_events ( + id, opportunity_id, event_type, action_code, to_stage, note, payload, created_by + ) VALUES ( + CAST(:event_id AS UUID), CAST(:opportunity_id AS UUID), + 'opportunity_reconstructed_from_reconciliation', :action_code, :to_stage, + :note, CAST(:payload AS JSONB), :created_by + ) + """), { + "event_id": str(uuid.uuid4()), + "opportunity_id": opportunity_id, + "action_code": action_code, + "to_stage": stage, + "note": "Oportunidade reconstruída a partir de múltiplas evidências externas.", + "payload": _json(metadata), + "created_by": actor, + }) + for item in items: + conn.execute(text(""" + UPDATE reconciliation_items + SET opportunity_id = CAST(:opportunity_id AS UUID), + status = 'linked', + resolved_at = now(), + updated_at = now(), + resolution_note = 'Ligado por reconstrução de processo', + payload = COALESCE(payload, '{}'::jsonb) || CAST(:payload AS JSONB) + WHERE id = CAST(:id AS UUID) + """), {"opportunity_id": opportunity_id, "id": item.get("id"), "payload": _json({"process_reconstruction": {"opportunity_id": opportunity_id, "actor": actor}})}) + _upsert_external_customer_mapping_from_process(conn, item, opportunity_id, actor=actor) + _upsert_odoo_operation_links_from_item(conn, item, opportunity_id) + _upsert_opportunity_items_from_odoo_item(conn, item, opportunity_id) + _apply_jasmin_documents_to_opportunity(conn, [item], opportunity_id, actor=actor) + for step in _process_steps_for_item(item): + conn.execute(text(""" + INSERT INTO opportunity_events ( + id, opportunity_id, event_type, action_code, to_stage, note, payload, created_by + ) VALUES ( + CAST(:event_id AS UUID), CAST(:opportunity_id AS UUID), + 'reconciliation_evidence_imported', :action_code, :to_stage, + :note, CAST(:payload AS JSONB), :created_by + ) + """), { + "event_id": str(uuid.uuid4()), + "opportunity_id": opportunity_id, + "action_code": step.get("action_code") or action_code, + "to_stage": step.get("stage") or stage, + "note": f"{step.get('label') or 'Evidência externa importada'}: {step.get('document_number') or item.get('document_number') or item.get('external_id')}", + "payload": _json({"reconciliation_item_id": item.get("id"), "source_system": item.get("source_system"), "external_type": step.get("external_type") or item.get("external_type"), "external_id": item.get("external_id"), "document_number": step.get("document_number") or item.get("document_number"), "document_date": step.get("document_date") or item.get("document_date"), "amount": item.get("amount"), "step": step}), + "created_by": actor, + }) + _record_reconciliation_decision( + conn, + decision_type="create_process_opportunity", + item_ids=[str(item.get("id")) for item in items if item.get("id")], + status="linked", + opportunity_id=opportunity_id, + customer_id=_uuid_or_none(customer_id), + note="Oportunidade reconstruída a partir de reconciliação", + actor=actor, + payload={"suggested_stage": stage, "suggested_action": action_code, "evidence_count": len(evidence)}, + ) + task_id = _create_task_for_opportunity( + opportunity_id=opportunity_id, + action_code=action_code, + note="Continuar processo reconstruído a partir de documentos/vendas/comprovativos externos. Validar antes de executar ações fiscais ou financeiras.", + source_system="reconciliation_process", + source_event_id=opportunity_id, + metadata=metadata, + ) + if task_id: + with engine.begin() as conn: + conn.execute(text(""" + UPDATE opportunities + SET last_task_id = CAST(:task_id AS UUID), updated_at = now() + WHERE id = CAST(:opportunity_id AS UUID) + """), {"task_id": task_id, "opportunity_id": opportunity_id}) + return opportunity_id + +def set_reconciliation_status(item_id: str, *, status: str, note: str = "", actor: str = "operator") -> None: + ensure_reconciliation_schema() + with engine.begin() as conn: + conn.execute(text(""" + UPDATE reconciliation_items + SET status = :status, + resolution_note = COALESCE(NULLIF(:note, ''), resolution_note), + resolved_at = CASE WHEN :status IN ('resolved','ignored','linked','historical') THEN now() ELSE resolved_at END, + updated_at = now(), + payload = COALESCE(payload, '{}'::jsonb) || CAST(:payload AS JSONB) + WHERE id = CAST(:id AS UUID) + """), { + "id": item_id, + "status": status, + "note": note, + "payload": _json({"last_action": {"actor": actor, "status": status}}), + }) + _record_reconciliation_decision( + conn, + decision_type="status", + item_ids=[item_id], + status=status, + note=note, + actor=actor, + payload={"status": status}, + ) + + +def link_reconciliation_to_opportunity(item_id: str, opportunity_id: str, *, actor: str = "operator") -> None: + # Treat a single item link as a one-item process link. This ensures Odoo + # sales/fulfilment evidence updates the opportunity pipeline instead of + # creating only a loose reference. + if link_reconciliation_process_to_opportunity([item_id], opportunity_id, actor=actor): + return + ensure_reconciliation_schema() + with engine.begin() as conn: + conn.execute(text(""" + UPDATE reconciliation_items + SET opportunity_id = CAST(:opportunity_id AS UUID), + status = 'linked', + resolved_at = now(), + updated_at = now(), + payload = COALESCE(payload, '{}'::jsonb) || CAST(:payload AS JSONB) + WHERE id = CAST(:id AS UUID) + """), { + "id": item_id, + "opportunity_id": opportunity_id, + "payload": _json({"linked_by": actor}), + }) + conn.execute(text(""" + INSERT INTO opportunity_events ( + id, opportunity_id, event_type, note, payload, created_by + ) VALUES ( + CAST(:event_id AS UUID), CAST(:opportunity_id AS UUID), + 'reconciliation_item_linked', :note, CAST(:payload AS JSONB), :created_by + ) + """), { + "event_id": str(uuid.uuid4()), + "opportunity_id": opportunity_id, + "note": "Item de reconciliação ligado à oportunidade.", + "payload": _json({"reconciliation_item_id": item_id}), + "created_by": actor, + }) + _record_reconciliation_decision( + conn, + decision_type="link_item", + item_ids=[item_id], + status="linked", + opportunity_id=opportunity_id, + actor=actor, + ) + + +def _create_task_for_opportunity( + *, + opportunity_id: str, + action_code: str, + note: str, + source_system: str, + source_event_id: Optional[str], + metadata: Optional[Dict[str, Any]] = None, +) -> Optional[str]: + config = get_action_config(action_code) + route = config.get("route") or "rever" + action = config.get("action") or action_code + action_required = bool(config.get("action_required", True)) + safe_to_post = bool(config.get("safe_to_post", False)) + status = "pending" if action_required else "skipped" + idempotency_key = f"task:{source_system}:opportunity:{opportunity_id}:{source_event_id or action_code}:{action_code}" + with engine.begin() as conn: + row = conn.execute(text(""" + INSERT INTO tasks ( + opportunity_id, action_code, route, action, note, action_required, + safe_to_post, status, source_system, source_event_id, idempotency_key, + metadata, priority + ) VALUES ( + CAST(:opportunity_id AS UUID), :action_code, :route, :action, :note, + :action_required, :safe_to_post, :status, :source_system, :source_event_id, + :idempotency_key, CAST(:metadata AS JSONB), :priority + ) + ON CONFLICT (idempotency_key) DO NOTHING + RETURNING id::text + """), { + "opportunity_id": opportunity_id, + "action_code": action_code, + "route": route, + "action": action, + "note": note, + "action_required": action_required, + "safe_to_post": safe_to_post, + "status": status, + "source_system": source_system, + "source_event_id": source_event_id, + "idempotency_key": idempotency_key, + "metadata": _json(metadata or {}), + "priority": _priority_for_action(action_code), + }).fetchone() + return row[0] if row else None + + +def create_opportunity_from_reconciliation(item_id: str, *, actor: str = "operator") -> Optional[str]: + """Promote a reconciliation candidate into a ClientFlow opportunity. + + This is a deliberate operator action. It does not auto-confirm payment or + close the opportunity; it creates the process container and a next task. + """ + ensure_reconciliation_schema() + item = get_reconciliation_item(item_id) + if not item: + return None + + external_type = str(item.get("external_type") or "external_record") + stage = STAGE_BY_EXTERNAL_TYPE.get(external_type, "REVIEW") + action_code = item.get("suggested_action") or NEXT_ACTION_BY_EXTERNAL_TYPE.get(external_type, "REVIEW_MANUALLY") + title = item.get("title") or item.get("customer_name") or "Oportunidade importada" + customer_name = item.get("customer_name") or title + metadata = { + "created_from_reconciliation_item_id": item_id, + "source_system": item.get("source_system"), + "external_type": external_type, + "external_id": item.get("external_id"), + "document_number": item.get("document_number"), + } + + opportunity_id = str(uuid.uuid4()) + with engine.begin() as conn: + conn.execute(text(""" + INSERT INTO opportunities ( + id, title, stage, status, local_customer_id, customer_name, + customer_email, product_interest, value_amount, currency, + source_system, source_event_id, last_action_code, metadata + ) VALUES ( + CAST(:id AS UUID), :title, :stage, 'open', CAST(:customer_id AS UUID), + :customer_name, :customer_email, :product_interest, :value_amount, + :currency, :source_system, :source_event_id, :last_action_code, + CAST(:metadata AS JSONB) + ) + """), { + "id": opportunity_id, + "title": title, + "stage": stage, + "customer_id": _uuid_or_none(item.get("customer_id")), + "customer_name": customer_name, + "customer_email": item.get("customer_email"), + "product_interest": (item.get("payload") or {}).get("product_interest") if isinstance(item.get("payload"), dict) else None, + "value_amount": _money_or_none(item.get("amount")), + "currency": item.get("currency") or "EUR", + "source_system": item.get("source_system") or "reconciliation", + "source_event_id": item.get("external_id"), + "last_action_code": action_code, + "metadata": _json(metadata), + }) + conn.execute(text(""" + INSERT INTO opportunity_events ( + id, opportunity_id, event_type, action_code, to_stage, note, payload, created_by + ) VALUES ( + CAST(:event_id AS UUID), CAST(:opportunity_id AS UUID), + 'opportunity_created_from_reconciliation', :action_code, :to_stage, + :note, CAST(:payload AS JSONB), :created_by + ) + """), { + "event_id": str(uuid.uuid4()), + "opportunity_id": opportunity_id, + "action_code": action_code, + "to_stage": stage, + "note": "Oportunidade criada a partir de informação externa reconciliada.", + "payload": _json(metadata), + "created_by": actor, + }) + conn.execute(text(""" + UPDATE reconciliation_items + SET opportunity_id = CAST(:opportunity_id AS UUID), status = 'linked', + resolved_at = now(), updated_at = now(), resolution_note = :note + WHERE id = CAST(:item_id AS UUID) + """), { + "opportunity_id": opportunity_id, + "item_id": item_id, + "note": "Criada oportunidade a partir deste item.", + }) + _upsert_external_customer_mapping_from_process(conn, item, opportunity_id, actor=actor) + _upsert_odoo_operation_links_from_item(conn, item, opportunity_id) + _upsert_opportunity_items_from_odoo_item(conn, item, opportunity_id) + _apply_jasmin_documents_to_opportunity(conn, [item], opportunity_id, actor=actor) + _record_reconciliation_decision( + conn, + decision_type="create_item_opportunity", + item_ids=[item_id], + status="linked", + opportunity_id=opportunity_id, + customer_id=_uuid_or_none(item.get("customer_id")), + note="Criada oportunidade a partir deste item.", + actor=actor, + payload=metadata, + ) + + task_id = _create_task_for_opportunity( + opportunity_id=opportunity_id, + action_code=action_code, + note=item.get("description") or f"Continuar processo importado de {item.get('source_system') or 'fonte externa'}.", + source_system="reconciliation", + source_event_id=item_id, + metadata=metadata, + ) + if task_id: + with engine.begin() as conn: + conn.execute(text(""" + UPDATE opportunities + SET last_task_id = CAST(:task_id AS UUID), updated_at = now() + WHERE id = CAST(:opportunity_id AS UUID) + """), {"task_id": task_id, "opportunity_id": opportunity_id}) + return opportunity_id + + +def create_external_request( + *, + source_channel: str, + customer_name: str, + customer_email: str = "", + customer_phone: str = "", + product_interest: str = "", + request_text: str = "", + action_code: str = "SEND_QUOTE", + actor: str = "operator", +) -> Dict[str, Any]: + """Create an opportunity and task for WhatsApp/phone/direct-email intake.""" + ensure_reconciliation_schema() + action_code = str(action_code or "SEND_QUOTE").upper() + config = get_action_config(action_code) + opportunity_id = str(uuid.uuid4()) + title_parts = [p for p in [_clean(customer_name), _clean(product_interest)] if p] + title = " · ".join(title_parts) or "Pedido externo" + stage = STAGE_BY_EXTERNAL_TYPE.get("manual_request", "QUOTE_REQUESTED") + if action_code == "SEND_INFO": + stage = "INFO_REQUESTED" + elif action_code == "SEND_INVOICE": + stage = "INVOICE_REQUESTED" + elif action_code == "SEND_PROFORMA": + stage = "PROFORMA_REQUESTED" + metadata = { + "manual_intake": True, + "source_channel": source_channel, + "request_text": request_text, + "created_by": actor, + } + with engine.begin() as conn: + conn.execute(text(""" + INSERT INTO opportunities ( + id, title, stage, status, customer_name, customer_email, customer_phone, + product_interest, source_system, last_action_code, metadata + ) VALUES ( + CAST(:id AS UUID), :title, :stage, 'open', :customer_name, + :customer_email, :customer_phone, :product_interest, :source_system, + :action_code, CAST(:metadata AS JSONB) + ) + """), { + "id": opportunity_id, + "title": title, + "stage": stage, + "customer_name": _clean(customer_name) or "Contacto externo", + "customer_email": _clean(customer_email), + "customer_phone": _clean(customer_phone), + "product_interest": _clean(product_interest), + "source_system": _clean(source_channel) or "manual", + "action_code": action_code, + "metadata": _json(metadata), + }) + conn.execute(text(""" + INSERT INTO opportunity_events ( + id, opportunity_id, event_type, action_code, to_stage, note, payload, created_by + ) VALUES ( + CAST(:event_id AS UUID), CAST(:opportunity_id AS UUID), 'manual_external_intake', + :action_code, :stage, :note, CAST(:payload AS JSONB), :created_by + ) + """), { + "event_id": str(uuid.uuid4()), + "opportunity_id": opportunity_id, + "action_code": action_code, + "stage": stage, + "note": f"Pedido registado manualmente via {_clean(source_channel) or 'canal externo'}.", + "payload": _json(metadata), + "created_by": actor, + }) + task_id = _create_task_for_opportunity( + opportunity_id=opportunity_id, + action_code=action_code, + note=_clean(request_text) or config.get("action") or "Continuar pedido externo.", + source_system=_clean(source_channel) or "manual", + source_event_id=f"manual:{opportunity_id}", + metadata=metadata, + ) + if task_id: + with engine.begin() as conn: + conn.execute(text(""" + UPDATE opportunities + SET last_task_id = CAST(:task_id AS UUID), updated_at = now() + WHERE id = CAST(:opportunity_id AS UUID) + """), {"task_id": task_id, "opportunity_id": opportunity_id}) + return {"opportunity_id": opportunity_id, "task_id": task_id} + + +def create_payment_proof( + *, + opportunity_id: Optional[str] = None, + customer_id: Optional[str] = None, + source_system: str = "manual", + source_ref: str = "", + filename: str = "", + file_url: str = "", + amount: Any = None, + currency: str = "EUR", + proof_date: str = "", + note: str = "", + payload: Optional[Dict[str, Any]] = None, + actor: str = "operator", +) -> Dict[str, Any]: + """Store proof as evidence and create a CONFIRM_PAYMENT task if linked. + + A payment proof is never a payment confirmation. It only opens or supports + a finance validation task. + """ + ensure_reconciliation_schema() + amount_value = _money_or_none(amount) + with engine.begin() as conn: + row = conn.execute(text(""" + INSERT INTO payment_proofs ( + opportunity_id, customer_id, source_system, source_ref, filename, + file_url, amount, currency, proof_date, status, note, payload + ) VALUES ( + CAST(:opportunity_id AS UUID), CAST(:customer_id AS UUID), :source_system, + :source_ref, :filename, :file_url, :amount, :currency, + CAST(:proof_date AS DATE), 'pending_validation', :note, CAST(:payload AS JSONB) + ) + RETURNING id::text, opportunity_id::text, customer_id::text, source_system, + source_ref, filename, file_url, amount, currency, proof_date, + status, note, payload, created_at, updated_at + """), { + "opportunity_id": _uuid_or_none(opportunity_id), + "customer_id": _uuid_or_none(customer_id), + "source_system": _clean(source_system) or "manual", + "source_ref": _clean(source_ref), + "filename": _clean(filename), + "file_url": _clean(file_url), + "amount": amount_value, + "currency": _clean(currency) or "EUR", + "proof_date": _clean(proof_date) or None, + "note": _clean(note), + "payload": _json(payload or {"created_by": actor}), + }).mappings().first() + proof = dict(row or {}) + proof_id = proof.get("id") + + if opportunity_id: + task_id = _create_task_for_opportunity( + opportunity_id=opportunity_id, + action_code="CONFIRM_PAYMENT", + note=f"Comprovativo de pagamento recebido. Validar no banco/Jasmin antes de confirmar. {note}".strip(), + source_system="payment_proof", + source_event_id=proof_id, + metadata={"payment_proof_id": proof_id, "amount": amount_value, "source_system": source_system}, + ) + proof["task_id"] = task_id + else: + item = upsert_reconciliation_item( + source_system=_clean(source_system) or "manual", + external_type="payment_proof", + external_id=proof_id, + title="Comprovativo de pagamento por associar", + description="Comprovativo recebido, mas ainda sem oportunidade associada. Não confirma pagamento.", + priority="alta", + suggested_action="CONFIRM_PAYMENT", + amount=amount_value, + currency=currency, + payload={"payment_proof_id": proof_id, "filename": filename, "note": note}, + ) + proof["reconciliation_item_id"] = item.get("id") + return proof + + +def sync_local_documents_without_opportunity(limit: int = 200) -> Dict[str, int]: + """Create reconciliation candidates for local Jasmin documents not linked to opportunities.""" + ensure_reconciliation_schema() + with engine.begin() as conn: + rows = conn.execute(text(""" + SELECT cd.id::text, cd.system, cd.document_kind, cd.external_id, cd.document_number, + cd.customer_id::text, c.name AS customer_name, c.email AS customer_email, c.tax_id AS customer_tax_id, + cd.status, cd.total_amount, cd.amount, cd.currency, cd.document_date, + cd.payload + FROM commercial_documents cd + LEFT JOIN customers c ON c.id = cd.customer_id + WHERE cd.opportunity_id IS NULL + AND COALESCE(cd.system, 'jasmin') = 'jasmin' + AND COALESCE(cd.status, '') NOT IN ('ignored','cancelled','superseded') + ORDER BY cd.updated_at DESC, cd.created_at DESC + LIMIT :limit + """), {"limit": int(limit)}).mappings().all() + created = 0 + for row in rows: + data = dict(row) + kind = str(data.get("document_kind") or "document") + external_type = "jasmin_invoice" if "invoice" in kind else "jasmin_quotation" + document_number = data.get("document_number") or data.get("external_id") or data.get("id") + title = f"Documento Jasmin sem oportunidade · {document_number}" + item = upsert_reconciliation_item( + source_system="jasmin", + external_type=external_type, + external_id=data.get("external_id") or data.get("id"), + title=title, + description="Documento Jasmin existente sem ligação a oportunidade ClientFlow.", + priority="normal" if external_type == "jasmin_quotation" else "alta", + suggested_action=NEXT_ACTION_BY_EXTERNAL_TYPE.get(external_type), + customer_id=data.get("customer_id"), + customer_name=data.get("customer_name"), + customer_email=data.get("customer_email"), + customer_tax_id=data.get("customer_tax_id"), + document_number=document_number, + document_date=data.get("document_date"), + amount=data.get("total_amount") or data.get("amount"), + currency=data.get("currency") or "EUR", + payload={"commercial_document_id": data.get("id"), "document_kind": kind, "status": data.get("status")}, + idempotency_key=f"reconcile:commercial_document:{data.get('id')}", + ) + if item: + created += 1 + return {"seen": len(rows), "created_or_updated": created} diff --git a/app/schemas.py b/app/schemas.py new file mode 100644 index 0000000..e21bdee --- /dev/null +++ b/app/schemas.py @@ -0,0 +1,60 @@ +from typing import Any, Dict, Optional + +from pydantic import BaseModel, Field + + +class CurrentState(BaseModel): + last_action_code: str = "desconhecido" + last_route: str = "desconhecido" + last_task_status: str = "desconhecido" + metadata: Dict[str, Any] = Field(default_factory=dict) + + +class AnalyzeRequest(BaseModel): + last_customer_message: str = Field(..., min_length=1) + previous_context: str = "" + current_state: CurrentState = Field(default_factory=CurrentState) + + source: Optional[str] = "manual" + conversation_id: Optional[str] = None + contact_id: Optional[str] = None + + +class ActionDecision(BaseModel): + action_code: str + note: str = "" + confidence: float = Field(ge=0.0, le=1.0, default=0.0) + + +class ActionResult(BaseModel): + action_code: str + route: str + action_required: bool = False + action: str + note: str = "" + safe_to_post: bool = False + + +class UsageInfo(BaseModel): + id: Optional[str] = None + model: Optional[str] = None + provider: Optional[str] = None + prompt_tokens: int = 0 + completion_tokens: int = 0 + total_tokens: int = 0 + cost: float = 0.0 + + +class AnalyzeResponse(BaseModel): + app: str = "ClientFlow" + model: str + + action_decision: ActionDecision + action_result: ActionResult + + usage: UsageInfo + needs_review: bool = False + + action_run_id: Optional[str] = None + message_id: Optional[str] = None + task_id: Optional[str] = None diff --git a/app/task_service.py b/app/task_service.py new file mode 100644 index 0000000..247d942 --- /dev/null +++ b/app/task_service.py @@ -0,0 +1,1679 @@ +import json +import os +from typing import Any, Dict, List, Optional + +from sqlalchemy import text + +from app.action_catalog import get_action_config +from app.db import engine +from app.schemas import ActionResult + + +def _dump(value: Any) -> Dict[str, Any]: + if value is None: + return {} + if hasattr(value, "model_dump"): + return value.model_dump() + if isinstance(value, dict): + return value + return dict(value) + + +def _json(value: Any) -> str: + return json.dumps(value or {}, ensure_ascii=False) + + +def _uuid_or_none(value: Optional[str]) -> Optional[str]: + value = str(value or "").strip() + return value or None + + +def _initial_task_status(action_code: str, route: str, action_required: bool) -> str: + """Define se uma decisão de triagem gera trabalho humano. + + v4.6: REVIEW_MANUALLY e REMOVE_FROM_LIST têm de aparecer como trabalho + pendente. Só spam e NO_ACTION são realmente ignorados/skipped. + """ + code = str(action_code or "").strip().upper() + route = str(route or "").strip().lower() + + if code == "IGNORE_BOUNCE": + return "ignored" + if code in {"IGNORE_SPAM", "NO_ACTION"} or route == "spam": + return "skipped" + if code in {"REVIEW_MANUALLY", "REMOVE_FROM_LIST"}: + return "pending" + return "pending" if action_required else "skipped" + + +def _priority_for_action(action_code: str, route: str) -> str: + """Initial task priority for Operations. + + v4.9.0: generic review/marketing tasks must not compete with real + document/payment/fulfilment work. Financial/document tasks remain high. + """ + code = str(action_code or "").strip().upper() + route = str(route or "").strip().lower() + if code in {"SEND_PROFORMA", "SEND_INVOICE", "CONFIRM_PAYMENT", "PREPARE_ORDER", "CREATE_SHIPMENT"}: + return "alta" + if code in {"REVIEW_MANUALLY", "REMOVE_FROM_LIST", "MARK_NO_INTEREST", "IGNORE_SPAM", "NO_ACTION", "IGNORE_BOUNCE"}: + return "baixa" + if route == "financeiro": + return "alta" + if route in {"marketing", "rever"}: + return "baixa" + return "normal" + + +def _timeline_payload_preview(metadata: Optional[Dict[str, Any]]) -> Dict[str, Any]: + metadata = metadata or {} + preview = dict(metadata) + # Evita payloads demasiado grandes em eventos de timeline. + if "message_preview" in preview: + preview["message_preview"] = str(preview.get("message_preview") or "")[:500] + return preview + + +def _record_task_timeline_event(task_id: str, event_type: str = "task_created") -> None: + """Regista timeline quando a task já está associada a uma oportunidade. + + Se ainda não houver oportunidade, não força nada. O histórico principal + continua em task_events/opportunity_events. + """ + try: + from app.communication_service import create_timeline_event + + task = get_task_detail(task_id) + if not task or not task.get("opportunity_id"): + return + + title_prefix = { + "task_created": "Task criada", + "task_done": "Task concluída", + "task_auto_completed": "Task concluída automaticamente", + "task_skipped": "Task ignorada", + }.get(str(event_type or ""), "Task") + + create_timeline_event( + opportunity_id=task.get("opportunity_id"), + event_type=event_type, + title=f"{title_prefix}: {task.get('action') or task.get('action_code')}", + description=(task.get("note") or "")[:500], + source=task.get("source_system") or "clientflow", + related_type="task", + related_id=task_id, + payload={ + "action_code": task.get("action_code"), + "route": task.get("route"), + "status": task.get("status"), + "conversation_id": task.get("conversation_id"), + "contact_id": task.get("contact_id"), + "source_system": task.get("source_system"), + }, + created_by="system", + ) + except Exception as exc: + print(f"ClientFlow timeline event failed for task {task_id}: {exc}", flush=True) + + +def create_task_from_action_result( + *, + action_result: ActionResult | Dict[str, Any], + action_run_id: Optional[str] = None, + message_id: Optional[str] = None, + raw_event_id: Optional[str] = None, + conversation_id: Optional[str] = None, + contact_id: Optional[str] = None, + customer_id: Optional[str] = None, + source_system: str = "clientflow", + source_event_id: Optional[str] = None, + metadata: Optional[Dict[str, Any]] = None, +) -> Optional[str]: + data = _dump(action_result) + + action_code = data.get("action_code") or "REVIEW_MANUALLY" + if str(action_code or "").strip().upper() == "IGNORE_BOUNCE": + return None + config = get_action_config(action_code) + + route = data.get("route") or config["route"] + action = data.get("action") or config["action"] + note = data.get("note") or action + + action_required = bool(data.get("action_required", False)) + safe_to_post = bool(data.get("safe_to_post", False)) + + status = _initial_task_status(action_code, route, action_required) + priority = _priority_for_action(action_code, route) + + # Idempotência forte: + # - raw_event_id é estável quando o mesmo webhook é reenviado; + # - evita tasks duplicadas para o mesmo evento/action_code; + # - mantém fallback para source_event_id/message_id/action_run_id. + idempotency_source = ( + raw_event_id + or source_event_id + or message_id + or action_run_id + or "no_source" + ) + + idempotency_key = ":".join([ + "task", + str(source_system or "unknown"), + str(conversation_id or "no_conversation"), + str(idempotency_source), + str(action_code), + ]) + + sql = text(""" + INSERT INTO tasks ( + action_run_id, + message_id, + raw_event_id, + conversation_id, + contact_id, + customer_id, + action_code, + route, + action, + note, + action_required, + safe_to_post, + status, + source_system, + source_event_id, + idempotency_key, + metadata, + priority + ) + VALUES ( + CAST(:action_run_id AS UUID), + CAST(:message_id AS UUID), + CAST(:raw_event_id AS UUID), + :conversation_id, + :contact_id, + :customer_id, + :action_code, + :route, + :action, + :note, + :action_required, + :safe_to_post, + :status, + :source_system, + :source_event_id, + :idempotency_key, + CAST(:metadata AS JSONB), + :priority + ) + ON CONFLICT (idempotency_key) DO NOTHING + RETURNING id::text + """) + + with engine.begin() as conn: + row = conn.execute(sql, { + "action_run_id": _uuid_or_none(action_run_id), + "message_id": _uuid_or_none(message_id), + "raw_event_id": _uuid_or_none(raw_event_id), + "conversation_id": conversation_id, + "contact_id": contact_id, + "customer_id": customer_id, + "action_code": action_code, + "route": route, + "action": action, + "note": note, + "action_required": action_required, + "safe_to_post": safe_to_post, + "status": status, + "source_system": source_system, + "source_event_id": source_event_id, + "idempotency_key": idempotency_key, + "metadata": _json(metadata or {}), + "priority": priority, + }).fetchone() + + task_id = row[0] if row else None + + if task_id: + try: + from app.opportunity_service import upsert_opportunity_for_task + + upsert_opportunity_for_task( + task_id, + trigger="task_created", + created_by="system", + ) + _record_task_timeline_event(task_id, event_type="task_created") + except Exception as exc: + print(f"ClientFlow opportunity upsert failed for task {task_id}: {exc}", flush=True) + + return task_id + + +def list_tasks( + *, + status: Optional[str] = None, + route: Optional[str] = None, + q: Optional[str] = None, + limit: int = 100, +) -> List[Dict[str, Any]]: + where = [] + params: Dict[str, Any] = {"limit": limit} + + if status: + where.append("t.status = :status") + params["status"] = status + + if route: + where.append("t.route = :route") + params["route"] = route + + if q: + params["q"] = f"%{q.strip().lower()}%" + where.append(""" + ( + lower(coalesce(t.conversation_id, '')) like :q + or lower(coalesce(t.contact_id, '')) like :q + or lower(coalesce(t.customer_id, '')) like :q + or lower(coalesce(t.action_code, '')) like :q + or lower(coalesce(t.route, '')) like :q + or lower(coalesce(t.action, '')) like :q + or lower(coalesce(t.note, '')) like :q + or lower(coalesce(m.clean_body, '')) like :q + or lower(coalesce(m.raw_body, '')) like :q + or lower(coalesce(re.payload->'sender'->>'name', '')) like :q + or lower(coalesce(re.payload->'sender'->>'email', '')) like :q + or lower(coalesce(re.payload->'sender'->>'phone_number', '')) like :q + or lower(coalesce(re.payload->'conversation'->'meta'->'sender'->>'name', '')) like :q + or lower(coalesce(re.payload->'conversation'->'meta'->'sender'->>'email', '')) like :q + or lower(coalesce(re.payload->'conversation'->'contact_inbox'->>'source_id', '')) like :q + or lower(coalesce(re.payload->>'content', '')) like :q + ) + """) + + where_sql = "" + if where: + where_sql = "WHERE " + " AND ".join(where) + + sql = text(f""" + SELECT + t.id::text, + t.action_run_id::text, + t.message_id::text, + t.raw_event_id::text, + t.opportunity_id::text, + t.conversation_id, + t.contact_id, + t.customer_id, + t.action_code, + t.route, + t.action, + t.note, + t.action_required, + t.safe_to_post, + t.status, + t.priority, + t.source_system, + t.source_event_id, + t.due_at, + t.created_at, + t.updated_at, + t.done_at, + t.done_by, + t.metadata, + + COALESCE( + NULLIF(re.payload->'sender'->>'name', ''), + NULLIF(re.payload->'conversation'->'meta'->'sender'->>'name', ''), + NULLIF(t.customer_id, ''), + NULLIF(t.contact_id, '') + ) AS customer_name, + + COALESCE( + NULLIF(re.payload->'sender'->>'email', ''), + NULLIF(re.payload->'conversation'->'meta'->'sender'->>'email', ''), + NULLIF(re.payload->'conversation'->'contact_inbox'->>'source_id', '') + ) AS customer_email, + + COALESCE( + NULLIF(re.payload->'sender'->>'phone_number', ''), + NULLIF(re.payload->'conversation'->'meta'->'sender'->>'phone_number', '') + ) AS customer_phone, + + COALESCE( + NULLIF(re.payload->'conversation'->'additional_attributes'->>'mail_subject', ''), + NULLIF(re.payload->'content_attributes'->'email'->>'subject', ''), + NULLIF(re.payload->'conversation'->'messages'->0->'content_attributes'->'email'->>'subject', '') + ) AS message_subject, + + COALESCE( + NULLIF(m.clean_body, ''), + NULLIF(m.raw_body, ''), + NULLIF(re.payload->>'content', '') + ) AS request_text + + FROM tasks t + LEFT JOIN messages m ON m.id = t.message_id + LEFT JOIN raw_events re ON re.id = t.raw_event_id + {where_sql} + ORDER BY + CASE t.status + WHEN 'pending' THEN 1 + WHEN 'failed' THEN 2 + WHEN 'skipped' THEN 3 + WHEN 'done' THEN 4 + ELSE 5 + END, + CASE COALESCE(t.priority, 'normal') + WHEN 'alta' THEN 1 + WHEN 'normal' THEN 2 + WHEN 'baixa' THEN 3 + ELSE 4 + END, + t.due_at NULLS LAST, + t.created_at DESC + LIMIT :limit + """) + + with engine.begin() as conn: + rows = conn.execute(sql, params).mappings().all() + + return [dict(row) for row in rows] + + +def get_task(task_id: str) -> Optional[Dict[str, Any]]: + sql = text(""" + SELECT + id::text, + action_run_id::text, + message_id::text, + raw_event_id::text, + conversation_id, + contact_id, + customer_id, + action_code, + route, + action, + note, + action_required, + safe_to_post, + status, + source_system, + source_event_id, + due_at, + created_at, + updated_at, + done_at, + done_by, + metadata + FROM tasks + WHERE id = CAST(:task_id AS UUID) + """) + + with engine.begin() as conn: + row = conn.execute(sql, {"task_id": task_id}).mappings().first() + + return dict(row) if row else None + + + +def get_task_detail(task_id: str) -> Optional[Dict[str, Any]]: + sql = text(""" + SELECT + t.id::text, + t.action_run_id::text, + t.message_id::text, + t.raw_event_id::text, + t.opportunity_id::text, + t.conversation_id, + t.contact_id, + t.customer_id, + t.action_code, + t.route, + t.action, + t.note, + t.action_required, + t.safe_to_post, + t.status, + t.priority, + t.source_system, + t.source_event_id, + t.idempotency_key, + t.due_at, + t.created_at, + t.updated_at, + t.done_at, + t.done_by, + t.metadata, + o.local_customer_id::text AS linked_customer_id, + cu.name AS linked_customer_name, + cu.email AS linked_customer_email, + cu.tax_id AS linked_customer_tax_id, + cu.street_name AS linked_customer_street_name, + cu.postal_zone AS linked_customer_postal_zone, + cu.city_name AS linked_customer_city_name, + cu.phone AS linked_customer_phone, + + m.raw_body, + m.clean_body, + m.previous_context, + m.metadata AS message_metadata, + + ar.provider, + ar.decision_source, + ar.action_decision, + ar.action_result, + ar.usage, + ar.needs_review, + + re.event_type, + re.payload AS raw_payload, + re.processing_error, + + COALESCE( + NULLIF(re.payload->'sender'->>'name', ''), + NULLIF(re.payload->'conversation'->'meta'->'sender'->>'name', ''), + NULLIF(t.customer_id, ''), + NULLIF(t.contact_id, '') + ) AS customer_name, + + COALESCE( + NULLIF(re.payload->'sender'->>'email', ''), + NULLIF(re.payload->'conversation'->'meta'->'sender'->>'email', ''), + NULLIF(re.payload->'conversation'->'contact_inbox'->>'source_id', '') + ) AS customer_email, + + COALESCE( + NULLIF(re.payload->'sender'->>'phone_number', ''), + NULLIF(re.payload->'conversation'->'meta'->'sender'->>'phone_number', '') + ) AS customer_phone, + + COALESCE( + NULLIF(re.payload->'conversation'->'additional_attributes'->>'mail_subject', ''), + NULLIF(re.payload->'content_attributes'->'email'->>'subject', ''), + NULLIF(re.payload->'conversation'->'messages'->0->'content_attributes'->'email'->>'subject', '') + ) AS message_subject + + FROM tasks t + LEFT JOIN messages m ON m.id = t.message_id + LEFT JOIN action_runs ar ON ar.id = t.action_run_id + LEFT JOIN raw_events re ON re.id = t.raw_event_id + LEFT JOIN opportunities o ON o.id = t.opportunity_id + LEFT JOIN customers cu ON cu.id = o.local_customer_id + WHERE t.id = CAST(:task_id AS UUID) + """) + + with engine.begin() as conn: + row = conn.execute(sql, {"task_id": task_id}).mappings().first() + + return dict(row) if row else None + + +def list_customer_task_history( + *, + contact_id: Optional[str] = None, + conversation_id: Optional[str] = None, + exclude_task_id: Optional[str] = None, + limit: int = 10, +) -> List[Dict[str, Any]]: + where = [] + params: Dict[str, Any] = {"limit": limit} + + if contact_id: + where.append("contact_id = :contact_id") + params["contact_id"] = contact_id + + if not contact_id and conversation_id: + where.append("conversation_id = :conversation_id") + params["conversation_id"] = conversation_id + + if exclude_task_id: + where.append("id <> CAST(:exclude_task_id AS UUID)") + params["exclude_task_id"] = exclude_task_id + + if not where: + return [] + + sql = text(f""" + SELECT + id::text, + created_at, + action_code, + route, + action, + note, + status, + conversation_id, + contact_id, + source_system, + done_at, + done_by + FROM tasks + WHERE {" AND ".join(where)} + ORDER BY created_at DESC + LIMIT :limit + """) + + with engine.begin() as conn: + rows = conn.execute(sql, params).mappings().all() + + return [dict(row) for row in rows] + + + +def get_external_mapping( + *, + local_system: str, + local_entity_type: str, + local_entity_id: str, + external_system: str, + external_entity_type: str, +) -> Optional[Dict[str, Any]]: + sql = text(""" + SELECT + id::text, + local_system, + local_entity_type, + local_entity_id, + external_system, + external_entity_type, + external_entity_id, + external_url, + match_key, + match_value, + confidence, + metadata, + created_at, + updated_at + FROM external_mappings + WHERE local_system = :local_system + AND local_entity_type = :local_entity_type + AND local_entity_id = :local_entity_id + AND external_system = :external_system + AND external_entity_type = :external_entity_type + ORDER BY updated_at DESC + LIMIT 1 + """) + + with engine.begin() as conn: + row = conn.execute(sql, { + "local_system": local_system, + "local_entity_type": local_entity_type, + "local_entity_id": str(local_entity_id), + "external_system": external_system, + "external_entity_type": external_entity_type, + }).mappings().first() + + return dict(row) if row else None + + +def get_customer_profile(contact_id: str) -> Optional[Dict[str, Any]]: + sql = text(""" + WITH base AS ( + SELECT + t.contact_id, + t.conversation_id, + t.created_at, + t.status, + COALESCE( + NULLIF(re.payload->'sender'->>'name', ''), + NULLIF(re.payload->'conversation'->'meta'->'sender'->>'name', ''), + NULLIF(t.customer_id, ''), + NULLIF(t.contact_id, '') + ) AS customer_name, + COALESCE( + NULLIF(re.payload->'sender'->>'email', ''), + NULLIF(re.payload->'conversation'->'meta'->'sender'->>'email', ''), + NULLIF(re.payload->'conversation'->'contact_inbox'->>'source_id', '') + ) AS customer_email, + COALESCE( + NULLIF(re.payload->'sender'->>'phone_number', ''), + NULLIF(re.payload->'conversation'->'meta'->'sender'->>'phone_number', '') + ) AS customer_phone + FROM tasks t + LEFT JOIN raw_events re ON re.id = t.raw_event_id + WHERE t.contact_id = :contact_id + ) + SELECT + :contact_id AS contact_id, + COALESCE((array_agg(customer_name ORDER BY created_at DESC) FILTER (WHERE customer_name IS NOT NULL AND customer_name <> ''))[1], :contact_id) AS customer_name, + COALESCE((array_agg(customer_email ORDER BY created_at DESC) FILTER (WHERE customer_email IS NOT NULL AND customer_email <> ''))[1], '') AS customer_email, + COALESCE((array_agg(customer_phone ORDER BY created_at DESC) FILTER (WHERE customer_phone IS NOT NULL AND customer_phone <> ''))[1], '') AS customer_phone, + COALESCE((array_agg(conversation_id ORDER BY created_at DESC) FILTER (WHERE conversation_id IS NOT NULL AND conversation_id <> ''))[1], '') AS latest_conversation_id, + COUNT(*) AS task_count, + COUNT(*) FILTER (WHERE status = 'pending') AS pending_count, + COUNT(*) FILTER (WHERE status = 'done') AS done_count, + COUNT(*) FILTER (WHERE status = 'skipped') AS skipped_count, + MIN(created_at) AS first_seen_at, + MAX(created_at) AS last_seen_at + FROM base + """) + + with engine.begin() as conn: + row = conn.execute(sql, {"contact_id": str(contact_id)}).mappings().first() + + if not row: + return None + + result = dict(row) + if not result.get("task_count"): + return None + + return result + + +def list_customer_tasks(contact_id: str, *, limit: int = 50) -> List[Dict[str, Any]]: + sql = text(""" + SELECT + id::text, + created_at, + updated_at, + action_code, + route, + action, + note, + status, + conversation_id, + contact_id, + source_system, + done_at, + done_by + FROM tasks + WHERE contact_id = :contact_id + ORDER BY created_at DESC + LIMIT :limit + """) + + with engine.begin() as conn: + rows = conn.execute(sql, { + "contact_id": str(contact_id), + "limit": limit, + }).mappings().all() + + return [dict(row) for row in rows] + + +def list_customer_messages(contact_id: str, *, limit: int = 20) -> List[Dict[str, Any]]: + sql = text(""" + SELECT + id::text, + created_at, + conversation_id, + contact_id, + source_system, + direction, + raw_body, + clean_body, + previous_context, + metadata + FROM messages + WHERE contact_id = :contact_id + ORDER BY created_at DESC + LIMIT :limit + """) + + with engine.begin() as conn: + rows = conn.execute(sql, { + "contact_id": str(contact_id), + "limit": limit, + }).mappings().all() + + return [dict(row) for row in rows] + + +def complete_task_with_note( + task_id: str, + *, + done_by: str = "admin", + done_note: str = "", +) -> Optional[Dict[str, Any]]: + sql = text(""" + UPDATE tasks + SET + status = 'done', + done_at = now(), + done_by = CAST(:done_by AS TEXT), + updated_at = now(), + metadata = COALESCE(metadata, '{}'::jsonb) + || jsonb_build_object( + 'done_note', CAST(:done_note AS TEXT), + 'done_by', CAST(:done_by AS TEXT), + 'done_at', now() + ) + WHERE id = CAST(:task_id AS UUID) + RETURNING + id::text, + status, + done_at, + done_by, + metadata + """) + + with engine.begin() as conn: + row = conn.execute(sql, { + "task_id": task_id, + "done_by": done_by, + "done_note": done_note or "", + }).mappings().first() + + if row: + try: + from app.opportunity_service import advance_opportunity_after_task_done + + advance_opportunity_after_task_done( + task_id, + event_type="task_done_with_note", + payload={"done_note": done_note or ""}, + created_by=done_by, + ) + except Exception as exc: + print(f"ClientFlow opportunity advance failed for task {task_id}: {exc}", flush=True) + + if str(done_by or "").lower() not in {"chatwoot_outgoing", "system"}: + from app.operator_audit_service import record_operator_action_best_effort + + record_operator_action_best_effort( + action="task_completed_with_note", + entity_type="task", + entity_id=task_id, + task_id=task_id, + actor=done_by, + payload={"done_note": done_note or ""}, + ) + + return dict(row) if row else None + + +def get_admin_dashboard_metrics() -> Dict[str, Any]: + sql = text(""" + WITH task_stats AS ( + SELECT + count(*) FILTER (WHERE status = 'pending') AS pending_total, + count(*) FILTER (WHERE status = 'done') AS done_total, + count(*) FILTER (WHERE status = 'skipped') AS skipped_total, + count(*) FILTER (WHERE status = 'failed') AS failed_total, + + count(*) FILTER (WHERE created_at >= date_trunc('day', now())) AS created_today, + count(*) FILTER (WHERE done_at >= date_trunc('day', now())) AS done_today, + + count(*) FILTER (WHERE status = 'pending' AND route = 'vendas') AS pending_vendas, + count(*) FILTER (WHERE status = 'pending' AND route = 'financeiro') AS pending_financeiro, + count(*) FILTER (WHERE status = 'pending' AND route = 'operacoes') AS pending_operacoes, + count(*) FILTER (WHERE status = 'pending' AND route = 'suporte') AS pending_suporte, + count(*) FILTER (WHERE status = 'pending' AND route = 'rever') AS pending_rever, + + count(*) FILTER ( + WHERE status = 'pending' + AND ( + (route = 'suporte' AND created_at < now() - interval '2 hours') + OR (route = 'vendas' AND created_at < now() - interval '4 hours') + OR (route = 'financeiro' AND created_at < now() - interval '8 hours') + OR (route = 'operacoes' AND created_at < now() - interval '24 hours') + OR (route = 'rever' AND created_at < now() - interval '24 hours') + ) + ) AS overdue_total + FROM tasks + ), + raw_stats AS ( + SELECT + count(*) FILTER (WHERE created_at >= now() - interval '24 hours') AS webhooks_24h, + count(*) FILTER (WHERE ignored = true AND created_at >= now() - interval '24 hours') AS ignored_24h, + count(*) FILTER (WHERE processing_error IS NOT NULL AND processing_error <> '' AND created_at >= now() - interval '24 hours') AS webhook_errors_24h + FROM raw_events + ), + outbox_stats AS ( + SELECT + count(*) FILTER (WHERE status = 'pending') AS outbox_pending, + count(*) FILTER (WHERE status = 'failed') AS outbox_failed, + count(*) FILTER (WHERE status = 'sent') AS outbox_sent + FROM integration_outbox + ) + SELECT * + FROM task_stats, raw_stats, outbox_stats + """) + + with engine.begin() as conn: + row = conn.execute(sql).mappings().first() + + return dict(row or {}) + + +def list_admin_recent_tasks(*, limit: int = 8) -> List[Dict[str, Any]]: + sql = text(""" + SELECT + id::text, + created_at, + action_code, + route, + action, + note, + status, + conversation_id, + contact_id, + source_system + FROM tasks + ORDER BY created_at DESC + LIMIT :limit + """) + + with engine.begin() as conn: + rows = conn.execute(sql, {"limit": limit}).mappings().all() + + return [dict(row) for row in rows] + + +def list_admin_recent_raw_events(*, limit: int = 8) -> List[Dict[str, Any]]: + sql = text(""" + SELECT + created_at, + source_system, + event_type, + source_event_id, + conversation_id, + processed, + ignored, + processing_error + FROM raw_events + ORDER BY created_at DESC + LIMIT :limit + """) + + with engine.begin() as conn: + rows = conn.execute(sql, {"limit": limit}).mappings().all() + + return [dict(row) for row in rows] + + +def list_customer_opportunities(contact_id: str, *, limit: int = 20) -> List[Dict[str, Any]]: + """Lista oportunidades internas ClientFlow associadas ao contacto. + + Substitui a antiga consulta de oportunidades mapeadas em CRM externo. + """ + sql = text(""" + SELECT + o.id::text, + o.title, + o.stage, + o.status, + o.product_interest, + o.value_amount, + o.currency, + o.conversation_id, + o.customer_name, + o.customer_email, + o.last_action_code, + o.last_task_id::text, + o.created_at, + o.updated_at, + count(t.id) AS task_count, + count(t.id) FILTER (WHERE t.status = 'pending') AS pending_task_count + FROM opportunities o + LEFT JOIN tasks t ON t.opportunity_id = o.id + WHERE o.contact_id = :contact_id + GROUP BY o.id + ORDER BY o.updated_at DESC + LIMIT :limit + """) + + with engine.begin() as conn: + rows = conn.execute(sql, { + "contact_id": str(contact_id), + "limit": limit, + }).mappings().all() + + return [dict(row) for row in rows] + + +# Compatibilidade para código antigo/imports anteriores. +list_customer_opportunity_mappings = list_customer_opportunities + +def get_system_health_metrics() -> Dict[str, Any]: + sql = text(""" + WITH task_stats AS ( + SELECT + count(*) AS tasks_total, + count(*) FILTER (WHERE status = 'pending') AS tasks_pending, + count(*) FILTER (WHERE status = 'done') AS tasks_done, + count(*) FILTER (WHERE status = 'failed') AS tasks_failed, + count(*) FILTER (WHERE created_at >= now() - interval '24 hours') AS tasks_24h + FROM tasks + ), + raw_stats AS ( + SELECT + count(*) FILTER (WHERE created_at >= now() - interval '24 hours') AS webhooks_24h, + count(*) FILTER (WHERE ignored = true AND created_at >= now() - interval '24 hours') AS webhooks_ignored_24h, + count(*) FILTER ( + WHERE processing_error IS NOT NULL + AND processing_error <> '' + AND created_at >= now() - interval '24 hours' + ) AS webhooks_errors_24h + FROM raw_events + ), + outbox_stats AS ( + SELECT + count(*) FILTER (WHERE status = 'pending') AS outbox_pending, + count(*) FILTER (WHERE status = 'failed') AS outbox_failed, + count(*) FILTER (WHERE status = 'sent') AS outbox_sent + FROM integration_outbox + WHERE target_system <> ('t' || 'wenty') + ), + opportunity_stats AS ( + SELECT + count(*) AS opportunities_total, + count(*) FILTER (WHERE status = 'open') AS opportunities_open, + count(*) FILTER (WHERE stage = 'WAITING_PAYMENT') AS opportunities_waiting_payment, + count(*) FILTER (WHERE stage IN ('ORDER_PREPARATION', 'SHIPPED')) AS opportunities_operations + FROM opportunities + ) + SELECT * + FROM task_stats, raw_stats, outbox_stats, opportunity_stats + """) + + with engine.begin() as conn: + row = conn.execute(sql).mappings().first() + + return dict(row or {}) + +def create_task_event( + *, + task_id: str, + event_type: str, + payload: Optional[Dict[str, Any]] = None, + created_by: str = "operator", +) -> None: + sql = text(""" + INSERT INTO task_events ( + task_id, + event_type, + payload, + created_by + ) + VALUES ( + CAST(:task_id AS UUID), + CAST(:event_type AS TEXT), + CAST(:payload AS JSONB), + CAST(:created_by AS TEXT) + ) + """) + + with engine.begin() as conn: + conn.execute(sql, { + "task_id": task_id, + "event_type": event_type, + "payload": _json(payload or {}), + "created_by": created_by, + }) + + +def create_business_event( + *, + event_type: str, + task: Dict[str, Any], + payload: Optional[Dict[str, Any]] = None, + created_by: str = "operator", +) -> Optional[str]: + idempotency_key = f"business_event:{task['id']}:{event_type}" + + sql = text(""" + INSERT INTO business_events ( + event_type, + task_id, + action_run_id, + message_id, + raw_event_id, + customer_id, + conversation_id, + contact_id, + payload, + idempotency_key, + created_by + ) + VALUES ( + CAST(:event_type AS TEXT), + CAST(:task_id AS UUID), + CAST(:action_run_id AS UUID), + CAST(:message_id AS UUID), + CAST(:raw_event_id AS UUID), + :customer_id, + :conversation_id, + :contact_id, + CAST(:payload AS JSONB), + CAST(:idempotency_key AS TEXT), + CAST(:created_by AS TEXT) + ) + ON CONFLICT (idempotency_key) DO NOTHING + RETURNING id::text + """) + + with engine.begin() as conn: + row = conn.execute(sql, { + "event_type": event_type, + "task_id": task["id"], + "action_run_id": _uuid_or_none(task.get("action_run_id")), + "message_id": _uuid_or_none(task.get("message_id")), + "raw_event_id": _uuid_or_none(task.get("raw_event_id")), + "customer_id": task.get("customer_id"), + "conversation_id": task.get("conversation_id"), + "contact_id": task.get("contact_id"), + "payload": _json(payload or {}), + "idempotency_key": idempotency_key, + "created_by": created_by, + }).fetchone() + + return row[0] if row else None + + +def create_next_task_after_business_event( + *, + event_type: str, + task: Dict[str, Any], + created_by: str = "system", +) -> Optional[str]: + """Não cria action_codes operacionais legados. + + A continuação após pagamento/preparação passa por `business_events`, + `operation_links` e pela página de operações da oportunidade. + """ + return None + + +def complete_task( + *, + task_id: str, + done_by: str = "operator", + payload: Optional[Dict[str, Any]] = None, +) -> Dict[str, Any]: + task = get_task(task_id) + + if task is None: + return {"ok": False, "error": "task_not_found"} + + if task["status"] == "done": + return {"ok": True, "status": "already_done", "task": task} + + with engine.begin() as conn: + conn.execute(text(""" + UPDATE tasks + SET + status = 'done', + done_at = now(), + done_by = CAST(:done_by AS TEXT), + updated_at = now() + WHERE id = CAST(:task_id AS UUID) + """), { + "task_id": task_id, + "done_by": done_by, + }) + + create_task_event( + task_id=task_id, + event_type="task_done", + payload=payload or {}, + created_by=done_by, + ) + + if str(done_by or "") == "chatwoot_outgoing": + create_task_event( + task_id=task_id, + event_type="task_auto_completed", + payload=payload or {}, + created_by=done_by, + ) + elif str(done_by or "").lower() != "system": + from app.operator_audit_service import record_operator_action_best_effort + + record_operator_action_best_effort( + action="task_completed", + entity_type="task", + entity_id=task_id, + task_id=task_id, + actor=done_by, + before={"status": task.get("status")}, + after={"status": "done"}, + payload=payload or {}, + ) + + config = get_action_config(task["action_code"]) + event_type = config.get("business_event_on_done") + + business_event_id = None + next_task_id = None + + if event_type: + business_event_id = create_business_event( + event_type=event_type, + task=task, + payload={ + "task_action_code": task["action_code"], + "task_action": task["action"], + "task_note": task["note"], + **(payload or {}), + }, + created_by=done_by, + ) + + try: + from app.integration_outbox_service import create_outbox_for_business_event + + create_outbox_for_business_event( + business_event_id=business_event_id, + event_type=event_type, + task=task, + payload={ + "task_action_code": task["action_code"], + "task_action": task["action"], + "task_note": task["note"], + **(payload or {}), + }, + ) + except Exception as exc: + print(f"ClientFlow outbox creation failed: {exc}") + + next_task_id = create_next_task_after_business_event( + event_type=event_type, + task=task, + created_by="system", + ) + + try: + from app.opportunity_service import advance_opportunity_after_task_done + + advance_opportunity_after_task_done( + task_id, + event_type=event_type or "task_done", + payload=payload or {}, + created_by=done_by, + ) + timeline_event_type = "task_auto_completed" if str(done_by) == "chatwoot_outgoing" else "task_done" + _record_task_timeline_event(task_id, event_type=timeline_event_type) + except Exception as exc: + print(f"ClientFlow opportunity advance failed for task {task_id}: {exc}", flush=True) + + return { + "ok": True, + "status": "done", + "task_id": task_id, + "business_event_id": business_event_id, + "next_task_id": next_task_id, + } + + +def skip_task( + *, + task_id: str, + skipped_by: str = "operator", + reason: str = "", +) -> Dict[str, Any]: + task = get_task(task_id) + + if task is None: + return {"ok": False, "error": "task_not_found"} + + with engine.begin() as conn: + conn.execute(text(""" + UPDATE tasks + SET + status = 'skipped', + updated_at = now() + WHERE id = CAST(:task_id AS UUID) + """), {"task_id": task_id}) + + create_task_event( + task_id=task_id, + event_type="task_skipped", + payload={"reason": reason}, + created_by=skipped_by, + ) + + from app.operator_audit_service import record_operator_action_best_effort + + record_operator_action_best_effort( + action="task_skipped", + entity_type="task", + entity_id=task_id, + task_id=task_id, + actor=skipped_by, + before={"status": task.get("status")}, + after={"status": "skipped"}, + payload={"reason": reason}, + ) + + return {"ok": True, "status": "skipped", "task_id": task_id} + + + +def auto_complete_task_from_outgoing_message( + *, + conversation_id: Optional[str], + contact_id: Optional[str] = None, + outgoing_message_id: Optional[str] = None, + outgoing_content: str = "", + sender_name: Optional[str] = None, + sender_type: Optional[str] = None, + is_private: bool = False, +) -> Dict[str, Any]: + """ + Quando um operador responde no Chatwoot, fecha automaticamente a tarefa pendente + da mesma conversa, mas só para action_codes onde responder ao cliente é a ação. + """ + + enabled = os.getenv("CHATWOOT_AUTO_COMPLETE_ON_OUTGOING", "true").lower() in { + "1", "true", "yes", "sim" + } + + if not enabled: + return {"ok": True, "status": "disabled"} + + if is_private: + return {"ok": True, "status": "ignored_private_note"} + + if not conversation_id: + return {"ok": False, "status": "missing_conversation_id"} + + content = str(outgoing_content or "").strip() + if not content: + return {"ok": True, "status": "ignored_empty_outgoing"} + + # Evitar que notas internas/automáticas do próprio ClientFlow fechem tarefas. + lowered = content.lower() + if "clientflow" in lowered and ("ação recomendada" in lowered or "acao recomendada" in lowered): + return {"ok": True, "status": "ignored_clientflow_note"} + + # Guardrail v4.6.7 / v4.6.8: + # completar automaticamente só ações em que a resposta no Chatwoot é a + # própria execução da task. Há ações que nunca devem ser fechadas por uma + # mensagem outgoing, mesmo que alguém as coloque por engano no ENV. + default_codes = [ + "SEND_INFO", + "SEND_QUOTE", + "SEND_PROFORMA", + "SEND_INVOICE", + "SUPPORT", + ] + configured = os.getenv("CHATWOOT_AUTO_COMPLETE_ACTION_CODES", "").strip() + never_auto_complete_codes = { + "CONFIRM_PAYMENT", + "CONFIRM_PAYMENT_AND_PREPARE_SHIPMENT", + "MARK_NO_INTEREST", + "REVIEW_MANUALLY", + "PREPARE_ORDER", + "CREATE_SHIPMENT", + "REMOVE_FROM_LIST", + } + requested_codes = [ + c.strip().upper() + for c in (configured.split(",") if configured else default_codes) + if c.strip() + ] + blocked_by_policy = [c for c in requested_codes if c in never_auto_complete_codes] + codes = [c for c in requested_codes if c not in never_auto_complete_codes] + + if blocked_by_policy: + print( + "ClientFlow auto-complete policy ignored action_codes=" + + ",".join(sorted(set(blocked_by_policy))), + flush=True, + ) + + if not codes: + return { + "ok": True, + "status": "no_action_codes_configured", + "blocked_by_policy": sorted(set(blocked_by_policy)), + } + + placeholders = ", ".join([f":code_{i}" for i in range(len(codes))]) + params: Dict[str, Any] = { + "conversation_id": str(conversation_id), + **{f"code_{i}": code for i, code in enumerate(codes)}, + } + + sql = text(f""" + SELECT + id::text, + action_code, + route, + action, + note, + created_at, + metadata + FROM tasks + WHERE status = 'pending' + AND conversation_id = :conversation_id + AND action_code IN ({placeholders}) + ORDER BY + CASE route + WHEN 'financeiro' THEN 1 + WHEN 'vendas' THEN 2 + WHEN 'suporte' THEN 3 + ELSE 4 + END, + created_at DESC + LIMIT 1 + """) + + with engine.begin() as conn: + task = conn.execute(sql, params).mappings().first() + + if not task: + return { + "ok": True, + "status": "no_matching_pending_task", + "conversation_id": conversation_id, + } + + metadata = task.get("metadata") or {} + if isinstance(metadata, str): + try: + metadata = json.loads(metadata) + except Exception: + metadata = {} + if str(metadata.get("opportunity_linking_status") or "") == "ambiguous": + return { + "ok": True, + "status": "blocked_ambiguous_opportunity", + "reason": "operator_must_confirm_opportunity_before_auto_complete", + "task_id": task["id"], + "action_code": task["action_code"], + "route": task["route"], + "conversation_id": conversation_id, + } + + payload = { + "auto_completed": True, + "reason": "operator_outgoing_message_in_chatwoot", + "outgoing_message_id": outgoing_message_id, + "outgoing_content_preview": content[:500], + "sender_name": sender_name, + "sender_type": sender_type, + "conversation_id": conversation_id, + "contact_id": contact_id, + } + + result = complete_task( + task_id=task["id"], + done_by="chatwoot_outgoing", + payload=payload, + ) + + return { + "ok": result.get("ok", False), + "status": "auto_completed" if result.get("ok") else "failed", + "task_id": task["id"], + "action_code": task["action_code"], + "route": task["route"], + "conversation_id": conversation_id, + "result": result, + } + + + +def get_latest_task_preparation(task_id: str) -> Optional[Dict[str, Any]]: + sql = text(""" + select + id::text, + task_id::text, + conversation_id, + contact_id, + prep_type, + status, + extracted_data, + missing_fields, + suggested_reply, + confidence, + model, + provider, + total_tokens, + cost, + created_at, + updated_at + from task_preparations + where task_id = :task_id + order by created_at desc + limit 1 + """) + + with engine.begin() as conn: + row = conn.execute(sql, {"task_id": task_id}).mappings().first() + + return dict(row) if row else None + + + +ACTION_ROUTE_MAP = { + "SEND_INFO": ("vendas", "Enviar informação ao cliente"), + "SEND_QUOTE": ("vendas", "Enviar proposta/cotação"), + "SEND_PROFORMA": ("financeiro", "Emitir fatura pró-forma"), + "SEND_INVOICE": ("financeiro", "Enviar fatura ao cliente"), + "CONFIRM_PAYMENT": ("financeiro", "Confirmar pagamento"), + "SUPPORT": ("suporte", "Responder ao pedido de suporte"), + "REMOVE_FROM_LIST": ("marketing", "Remover contacto da lista"), + "MARK_NO_INTEREST": ("vendas", "Marcar sem interesse"), + "NO_ACTION": ("rever", "Sem ação necessária"), + "REVIEW_MANUALLY": ("rever", "Rever manualmente"), + "IGNORE_SPAM": ("spam", "Ignorar spam"), +} + + +def reclassify_task( + *, + task_id: str, + new_action_code: str, + reason: str = "", + reclassified_by: str = "operator", + reopen: bool = True, +) -> Dict[str, Any]: + import json as _json + + new_action_code = str(new_action_code or "").strip().upper() + + if new_action_code not in ACTION_ROUTE_MAP: + raise ValueError(f"Unsupported action_code: {new_action_code}") + + new_route, new_action = ACTION_ROUTE_MAP[new_action_code] + + # Só spam/sem ação saem do trabalho operacional. + # REVIEW_MANUALLY e REMOVE_FROM_LIST devem ficar pendentes para o operador. + new_status = "skipped" if new_action_code in {"IGNORE_SPAM", "NO_ACTION"} else "pending" + new_note = reason.strip() or f"Reclassificado manualmente para {new_action_code}." + + with engine.begin() as conn: + old = conn.execute( + text(""" + select + id::text, + action_code, + route, + action, + note, + status + from tasks + where id = :task_id + limit 1 + """), + {"task_id": task_id}, + ).mappings().first() + + if not old: + return {"ok": False, "status": "not_found", "task_id": task_id} + + metadata_patch = { + "manual_reclassified": True, + "manual_reclassified_at": "now", + "manual_reclassified_by": reclassified_by, + "manual_reclassified_reason": reason, + "previous_action_code": old["action_code"], + "previous_route": old["route"], + "previous_status": old["status"], + } + + event_payload = { + "old": { + "action_code": old["action_code"], + "route": old["route"], + "status": old["status"], + }, + "new": { + "action_code": new_action_code, + "route": new_route, + "status": new_status, + }, + "reason": reason, + } + + conn.execute( + text(""" + update tasks + set + action_code = :action_code, + route = :route, + action = :action, + note = :note, + status = :status, + done_at = null, + done_by = null, + metadata = coalesce(metadata, '{}'::jsonb) + || cast(:metadata_patch as jsonb) + || jsonb_build_object('manual_reclassified_at', now()), + updated_at = now() + where id = :task_id + """), + { + "task_id": task_id, + "action_code": new_action_code, + "route": new_route, + "action": new_action, + "note": new_note, + "status": new_status, + "metadata_patch": _json.dumps(metadata_patch, ensure_ascii=False), + }, + ) + + conn.execute( + text(""" + insert into task_events ( + task_id, + event_type, + created_by, + payload + ) + values ( + :task_id, + 'task_reclassified', + :created_by, + cast(:payload as jsonb) + ) + """), + { + "task_id": task_id, + "created_by": reclassified_by, + "payload": _json.dumps(event_payload, ensure_ascii=False), + }, + ) + + from app.operator_audit_service import record_operator_action_best_effort + + record_operator_action_best_effort( + action="task_reclassified", + entity_type="task", + entity_id=task_id, + task_id=task_id, + actor=reclassified_by, + before={"action_code": old["action_code"], "route": old["route"], "status": old["status"]}, + after={"action_code": new_action_code, "route": new_route, "status": new_status}, + payload={"reason": reason}, + ) + + return { + "ok": True, + "status": "reclassified", + "task_id": task_id, + "old_action_code": old["action_code"], + "old_route": old["route"], + "old_status": old["status"], + "new_action_code": new_action_code, + "new_route": new_route, + "new_status": new_status, + } + + + + + +def list_tasks_board_v3( + *, + status: str = "pending", + route: str = "", + q: str = "", + limit: int = 200, +) -> List[Dict[str, Any]]: + filters = [] + params: Dict[str, Any] = {"limit": int(limit)} + + status = str(status or "").strip() + route = str(route or "").strip() + q = str(q or "").strip() + + if status and status != "all": + filters.append("t.status = :status") + params["status"] = status + + if route and route != "all": + filters.append("t.route = :route") + params["route"] = route + + if q: + filters.append(""" + ( + t.conversation_id ilike :q + or coalesce(t.contact_id, '') ilike :q + or coalesce(t.action_code, '') ilike :q + or coalesce(t.action, '') ilike :q + or coalesce(t.note, '') ilike :q + ) + """) + params["q"] = f"%{q}%" + + where_sql = " and ".join(filters) if filters else "true" + + sql = text(f""" + select + t.id::text, + t.created_at, + t.updated_at, + t.conversation_id, + t.contact_id, + t.action_code, + t.route, + t.status, + t.action, + t.note, + t.done_by, + t.done_at, + coalesce(t.metadata, '{{}}'::jsonb) as metadata + from tasks t + where {where_sql} + order by + case t.status + when 'pending' then 1 + when 'skipped' then 2 + when 'done' then 3 + else 4 + end, + case t.route + when 'financeiro' then 1 + when 'vendas' then 2 + when 'suporte' then 3 + when 'rever' then 4 + when 'spam' then 5 + else 9 + end, + t.created_at desc + limit :limit + """) + + with engine.begin() as conn: + rows = conn.execute(sql, params).mappings().all() + + return [dict(r) for r in rows] diff --git a/app/webhooks_chatwoot.py b/app/webhooks_chatwoot.py new file mode 100644 index 0000000..4ef3161 --- /dev/null +++ b/app/webhooks_chatwoot.py @@ -0,0 +1,459 @@ +import time +import json +import hmac +import hashlib +import re +from typing import Any, Dict, Optional, List +from hmac import compare_digest + +from fastapi import APIRouter, Request, HTTPException + +from app.analyzer import analyze +from app.config import settings +from app.chatwoot_client import add_private_note +from app.message_cleaner import extract_chatwoot_content +from app.persistence import ( + get_state_for_conversation, + mark_raw_event_processed, + save_raw_event, +) +from app.schemas import AnalyzeRequest +from app.task_service import auto_complete_task_from_outgoing_message + + +router = APIRouter(prefix="/webhooks", tags=["webhooks"]) + + +def as_str(value: Any) -> Optional[str]: + if value is None: + return None + return str(value) + + +def get_nested(data: Dict[str, Any], *keys: str) -> Any: + current: Any = data + + for key in keys: + if not isinstance(current, dict): + return None + current = current.get(key) + + return current + + +def compact_text(value: Any, limit: int = 450) -> str: + """Normaliza texto para contexto LLM compacto.""" + text = str(value or "") + text = re.sub(r"<[^>]+>", " ", text) + text = re.sub(r"\s+", " ", text).strip() + if len(text) > limit: + return text[: limit - 1].rstrip() + "…" + return text + + +def extract_email_subject(payload: Dict[str, Any], message: Dict[str, Any], conversation: Dict[str, Any]) -> str: + """Tenta extrair o assunto do email a partir das variantes comuns do Chatwoot.""" + candidates = [ + get_nested(message, "content_attributes", "email", "subject"), + get_nested(payload, "content_attributes", "email", "subject"), + get_nested(message, "content_attributes", "subject"), + get_nested(payload, "content_attributes", "subject"), + get_nested(conversation, "additional_attributes", "mail_subject"), + get_nested(payload, "conversation", "additional_attributes", "mail_subject"), + conversation.get("subject") if isinstance(conversation, dict) else None, + payload.get("subject"), + ] + + # Em alguns webhooks o assunto vem dentro de uma mensagem do array conversation.messages. + messages = conversation.get("messages") if isinstance(conversation, dict) else None + if isinstance(messages, list): + for item in messages: + if isinstance(item, dict): + candidates.append(get_nested(item, "content_attributes", "email", "subject")) + candidates.append(get_nested(item, "content_attributes", "subject")) + + for candidate in candidates: + cleaned = compact_text(candidate, limit=180) + if cleaned: + return cleaned + return "" + + +def message_role(message: Dict[str, Any]) -> str: + raw_type = str( + message.get("message_type") + or message.get("direction") + or message.get("sender_type") + or "" + ).lower() + if raw_type in {"outgoing", "outbound", "1"}: + return "BLIF" + return "Cliente" + + +def message_timestamp(message: Dict[str, Any]) -> Any: + return ( + message.get("created_at") + or message.get("updated_at") + or message.get("timestamp") + or message.get("id") + or 0 + ) + + +def extract_message_content_for_context(message: Dict[str, Any]) -> str: + try: + content = extract_chatwoot_content(message, message) + except Exception: + content = message.get("content") or message.get("text") or message.get("body") or "" + return compact_text(content, limit=360) + + +def extract_recent_conversation_context( + payload: Dict[str, Any], + current_message_id: Optional[str], + max_messages: int = 4, +) -> List[str]: + """Extrai as últimas mensagens relevantes antes da mensagem atual. + + Não envia a thread inteira ao LLM: usa no máximo `max_messages` linhas compactas. + """ + conversation = ( + payload.get("conversation") + if isinstance(payload.get("conversation"), dict) + else get_nested(payload, "message", "conversation") + if isinstance(get_nested(payload, "message", "conversation"), dict) + else {} + ) + messages = conversation.get("messages") if isinstance(conversation, dict) else None + if not isinstance(messages, list): + return [] + + normalized: List[Dict[str, Any]] = [] + for item in messages: + if not isinstance(item, dict): + continue + if item.get("private") or item.get("content_type") == "input_select": + continue + item_id = as_str(item.get("id") or item.get("message_id")) + if current_message_id and item_id == current_message_id: + continue + content = extract_message_content_for_context(item) + if not content: + continue + normalized.append({ + "sort": message_timestamp(item), + "role": message_role(item), + "content": content, + }) + + try: + normalized.sort(key=lambda row: row["sort"]) + except Exception: + pass + + recent = normalized[-max_messages:] + return [f"- {row['role']}: {row['content']}" for row in recent] + + +def build_triage_previous_context(payload: Dict[str, Any], extracted: Dict[str, Any]) -> str: + message = payload.get("message") if isinstance(payload.get("message"), dict) else payload + conversation = ( + payload.get("conversation") + if isinstance(payload.get("conversation"), dict) + else message.get("conversation") if isinstance(message.get("conversation"), dict) + else {} + ) + + lines: List[str] = ["Origem: webhook Chatwoot."] + + subject = extract_email_subject(payload, message, conversation) + if subject: + lines.append(f"Assunto: {subject}") + + recent_lines = extract_recent_conversation_context( + payload=payload, + current_message_id=extracted.get("source_event_id"), + max_messages=4, + ) + if recent_lines: + lines.append("Histórico recente, apenas para contexto:") + lines.extend(recent_lines) + else: + lines.append("Histórico recente: não disponível no webhook.") + + return "\n".join(lines) + + +def extract_chatwoot_event(payload: Dict[str, Any]) -> Dict[str, Any]: + event_type = ( + payload.get("event") + or payload.get("webhook_event") + or payload.get("event_type") + or payload.get("name") + or "unknown" + ) + + message = payload.get("message") if isinstance(payload.get("message"), dict) else payload + + conversation = ( + payload.get("conversation") + if isinstance(payload.get("conversation"), dict) + else message.get("conversation") if isinstance(message.get("conversation"), dict) + else {} + ) + + sender = ( + payload.get("sender") + if isinstance(payload.get("sender"), dict) + else message.get("sender") if isinstance(message.get("sender"), dict) + else {} + ) + + contact = ( + payload.get("contact") + if isinstance(payload.get("contact"), dict) + else conversation.get("contact") if isinstance(conversation.get("contact"), dict) + else get_nested(conversation, "contact_inbox", "contact") + or sender + or {} + ) + + content = extract_chatwoot_content(payload, message) + + conversation_id = ( + payload.get("conversation_id") + or message.get("conversation_id") + or conversation.get("id") + or get_nested(payload, "conversation", "id") + ) + + contact_id = ( + payload.get("contact_id") + or contact.get("id") + or sender.get("id") + or get_nested(conversation, "contact_inbox", "contact", "id") + ) + + source_event_id = ( + payload.get("id") + or message.get("id") + or payload.get("message_id") + ) + + message_type = ( + message.get("message_type") + or payload.get("message_type") + or message.get("direction") + or payload.get("direction") + or "" + ) + + message_type_text = str(message_type).lower() + is_outgoing = message_type_text in {"outgoing", "outbound", "1"} + + is_private = bool( + message.get("private") + or payload.get("private") + or message.get("content_type") == "input_select" + ) + + sender_name = ( + sender.get("name") + or sender.get("available_name") + or sender.get("display_name") + or sender.get("email") + ) + + sender_type = ( + sender.get("type") + or sender.get("role") + or sender.get("account_role") + or "" + ) + + return { + "event_type": str(event_type), + "source_event_id": as_str(source_event_id), + "conversation_id": as_str(conversation_id), + "contact_id": as_str(contact_id), + "content": str(content or "").strip(), + "message_type": message_type_text, + "is_outgoing": is_outgoing, + "is_private": is_private, + "sender_name": as_str(sender_name), + "sender_type": as_str(sender_type), + } + + +def validate_chatwoot_webhook_secret(request: Request) -> None: + expected = (settings.clientflow_webhook_secret or "").strip() + + # Em desenvolvimento mantém compatibilidade; em produção exige segredo. + if not expected: + if str(settings.env or "").strip().lower() in {"prod", "production", "staging"}: + raise HTTPException(status_code=500, detail="CLIENTFLOW_WEBHOOK_SECRET not configured") + return + + received = ( + request.query_params.get("token") + or request.headers.get("X-ClientFlow-Webhook-Secret") + or request.headers.get("X-Webhook-Secret") + or "" + ).strip() + + if not received or not compare_digest(received, expected): + raise HTTPException(status_code=401, detail="invalid webhook secret") + + + +def validate_chatwoot_signature(request: Request, raw_body: bytes) -> None: + secret = (settings.clientflow_webhook_secret or "").strip() + + # Em desenvolvimento mantém compatibilidade; em produção exige assinatura. + if not secret: + if str(settings.env or "").strip().lower() in {"prod", "production", "staging"}: + raise HTTPException(status_code=500, detail="CLIENTFLOW_WEBHOOK_SECRET not configured") + return + + received_signature = request.headers.get("X-Chatwoot-Signature", "") + timestamp = request.headers.get("X-Chatwoot-Timestamp", "") + + if not received_signature or not timestamp: + raise HTTPException(status_code=401, detail="missing chatwoot signature") + + try: + ts = int(timestamp) + except ValueError: + raise HTTPException(status_code=401, detail="invalid chatwoot timestamp") + + # Anti-replay: rejeita eventos com mais de 5 minutos. + if abs(int(time.time()) - ts) > 300: + raise HTTPException(status_code=401, detail="expired chatwoot signature") + + signed_payload = timestamp.encode("utf-8") + b"." + raw_body + expected_signature = "sha256=" + hmac.new( + secret.encode("utf-8"), + signed_payload, + hashlib.sha256, + ).hexdigest() + + if not hmac.compare_digest(expected_signature, received_signature): + raise HTTPException(status_code=401, detail="invalid chatwoot signature") + +@router.post("/chatwoot") +async def chatwoot_webhook(request: Request) -> Dict[str, Any]: + + raw_body = await request.body() + validate_chatwoot_signature(request, raw_body) + + try: + payload = json.loads(raw_body.decode('utf-8') or '{}') + except json.JSONDecodeError: + raise HTTPException(status_code=400, detail='invalid json payload') + extracted = extract_chatwoot_event(payload) + + raw_event_info = save_raw_event( + source_system="chatwoot", + event_type=extracted["event_type"], + source_event_id=extracted["source_event_id"], + conversation_id=extracted["conversation_id"], + contact_id=extracted["contact_id"], + payload=payload, + ) + raw_event_id = raw_event_info["id"] + + if raw_event_info.get("processed"): + return { + "status": "duplicate_ignored", + "raw_event_id": raw_event_id, + "message_id": raw_event_info.get("message_id"), + "action_run_id": raw_event_info.get("action_run_id"), + } + + if not extracted["content"]: + mark_raw_event_processed( + raw_event_id=raw_event_id, + ignored=True, + error="ignored: empty content", + ) + return { + "status": "ignored", + "reason": "empty_content", + "raw_event_id": raw_event_id, + } + + if extracted["is_outgoing"]: + auto_complete_result = auto_complete_task_from_outgoing_message( + conversation_id=extracted["conversation_id"], + contact_id=extracted["contact_id"], + outgoing_message_id=extracted["source_event_id"], + outgoing_content=extracted["content"], + sender_name=extracted.get("sender_name"), + sender_type=extracted.get("sender_type"), + is_private=bool(extracted.get("is_private")), + ) + + completed = auto_complete_result.get("status") == "auto_completed" + + mark_raw_event_processed( + raw_event_id=raw_event_id, + ignored=not completed, + error=None if completed else f"outgoing: {auto_complete_result.get('status')}", + ) + + return { + "status": "outgoing_processed" if completed else "ignored", + "reason": auto_complete_result.get("status"), + "raw_event_id": raw_event_id, + "conversation_id": extracted["conversation_id"], + "contact_id": extracted["contact_id"], + "auto_complete": auto_complete_result, + } + + current_state = get_state_for_conversation(extracted["conversation_id"]) + + analyze_request = AnalyzeRequest( + last_customer_message=extracted["content"], + previous_context=build_triage_previous_context(payload, extracted), + current_state=current_state, + source="chatwoot", + conversation_id=extracted["conversation_id"] or raw_event_id, + contact_id=extracted["contact_id"], + ) + + response = await analyze(analyze_request, raw_event_id=raw_event_id) + + chatwoot_note_result = { + "status": "skipped", + "reason": "no conversation_id", + } + + if extracted["conversation_id"]: + chatwoot_note_result = await add_private_note( + conversation_id=extracted["conversation_id"], + action_result=response.action_result, + ) + + mark_raw_event_processed( + raw_event_id=raw_event_id, + action_run_id=response.action_run_id, + message_id=response.message_id, + ignored=False, + error=None, + ) + + return { + "status": "processed", + "raw_event_id": raw_event_id, + "action_run_id": response.action_run_id, + "message_id": response.message_id, + "task_id": response.task_id, + "conversation_id": extracted["conversation_id"], + "contact_id": extracted["contact_id"], + "action_decision": response.action_decision.model_dump(), + "action_result": response.action_result.model_dump(), + "needs_review": response.needs_review, + "usage": response.usage.model_dump(), + "chatwoot_note": chatwoot_note_result, + } diff --git a/app/workflow_guard.py b/app/workflow_guard.py new file mode 100644 index 0000000..797b20c --- /dev/null +++ b/app/workflow_guard.py @@ -0,0 +1,299 @@ +from __future__ import annotations + +import json +from typing import Any, Dict, Optional, Set + +from sqlalchemy import text + +from app.db import engine + + +class OperationActionBlocked(RuntimeError): + pass + + +ACTION_LABELS: Dict[str, str] = { + "jasmin_quotation": "Criar orçamento Jasmin", + "jasmin_proforma": "Emitir/registar pró-forma", + "payment_confirmed": "Confirmar pagamento", + "odoo_sale_order": "Criar/registar venda Odoo", + "odoo_production": "Registar produção/preparação", + "odoo_physical_validated": "Validar encomenda física", + "jasmin_invoice": "Emitir/registar fatura", + "packlink_shipment": "Criar envio Packlink", + "tracking_sent": "Enviar/registar tracking", + "delivered": "Marcar entregue/concluir", +} + + +def _to_bool(value: Any) -> bool: + if isinstance(value, bool): + return value + return str(value or "").strip().lower() in {"1", "true", "t", "yes", "sim"} + + +def _json(value: Any) -> Dict[str, Any]: + if isinstance(value, dict): + return value + if isinstance(value, str) and value: + try: + parsed = json.loads(value) + return parsed if isinstance(parsed, dict) else {} + except Exception: + return {} + return {} + + +def get_workflow_context(opportunity_id: str) -> Dict[str, Any]: + with engine.begin() as conn: + opp = conn.execute(text(""" + SELECT id::text, stage, status + FROM opportunities + WHERE id = CAST(:opportunity_id AS UUID) + """), {"opportunity_id": str(opportunity_id)}).mappings().first() + + links = conn.execute(text(""" + SELECT system, external_type, status, external_id, external_name, external_url, payload, updated_at + FROM operation_links + WHERE opportunity_id = CAST(:opportunity_id AS UUID) + """), {"opportunity_id": str(opportunity_id)}).mappings().all() + + item_count = conn.execute(text(""" + SELECT count(*) + FROM opportunity_items + WHERE opportunity_id = CAST(:opportunity_id AS UUID) + """), {"opportunity_id": str(opportunity_id)}).scalar() or 0 + + jasmin_doc_counts = conn.execute(text(""" + SELECT + count(*) FILTER (WHERE document_kind = 'quotation') AS quotation_count, + count(*) FILTER (WHERE document_kind = 'proforma') AS proforma_count, + count(*) FILTER (WHERE document_kind = 'invoice') AS invoice_count + FROM commercial_documents + WHERE opportunity_id = CAST(:opportunity_id AS UUID) + AND system = 'jasmin' + """), {"opportunity_id": str(opportunity_id)}).mappings().first() + + by_key = {} + for row in links: + d = dict(row) + d["payload"] = _json(d.get("payload")) + by_key[f"{d.get('system')}:{d.get('external_type')}"] = d + + physical = by_key.get("odoo:physical_status") or {} + physical_payload = _json(physical.get("payload")) + + return { + "opportunity": dict(opp or {}), + "links": by_key, + "item_count": int(item_count), + "jasmin_quotation_count": int((jasmin_doc_counts or {}).get("quotation_count") or 0), + "jasmin_proforma_count": int((jasmin_doc_counts or {}).get("proforma_count") or 0), + "jasmin_invoice_count": int((jasmin_doc_counts or {}).get("invoice_count") or 0), + "physical_status": physical, + "physical_payload": physical_payload, + "ready_to_ship": _to_bool(physical_payload.get("ready_to_ship")), + } + + +def _has(ctx: Dict[str, Any], key: str, statuses: Optional[Set[str]] = None) -> bool: + link = (ctx.get("links") or {}).get(key) + if not link: + return False + if statuses is None: + return True + return str(link.get("status") or "") in statuses + + +def blocked_reason(ctx: Dict[str, Any], action_key: str) -> Optional[str]: + opp = ctx.get("opportunity") or {} + if not opp: + return "Oportunidade não encontrada." + + stage = str(opp.get("stage") or "") + opp_status = str(opp.get("status") or "") + + if opp_status == "closed" and action_key != "delivered": + return "A oportunidade já está fechada." + + item_count = int(ctx.get("item_count") or 0) + ready_to_ship = bool(ctx.get("ready_to_ship")) + + quotation = _has(ctx, "jasmin:quotation", {"created", "sent", "converted"}) or int(ctx.get("jasmin_quotation_count") or 0) > 0 + proforma = _has(ctx, "jasmin:proforma", {"issued"}) or int(ctx.get("jasmin_proforma_count") or 0) > 0 + payment = _has(ctx, "clientflow:payment", {"confirmed"}) + sale = _has(ctx, "odoo:sale_order", {"created"}) + physical_validated = _has(ctx, "odoo:physical_validation", {"validated"}) + invoice = _has(ctx, "jasmin:invoice", {"issued"}) or int(ctx.get("jasmin_invoice_count") or 0) > 0 + shipment = _has(ctx, "packlink:shipment", {"created"}) + tracking = _has(ctx, "clientflow:tracking", {"sent"}) + + if action_key == "jasmin_quotation": + if invoice: + return "Já existe fatura Jasmin associada/importada nesta oportunidade." + if proforma: + return "Já existe pró-forma Jasmin associada/importada nesta oportunidade." + if quotation: + return "Já existe orçamento Jasmin associado/importado nesta oportunidade." + if item_count <= 0: + return "Não é possível criar orçamento sem produtos na oportunidade." + if stage in {"WON", "LOST", "NO_INTEREST"}: + return "A oportunidade já está fechada/perdida." + return None + + if action_key == "jasmin_proforma": + if item_count <= 0: + return "Não é possível emitir pró-forma sem produtos na oportunidade." + if stage in {"WON", "LOST", "NO_INTEREST"}: + return "A oportunidade já está fechada/perdida." + return None + + if action_key == "payment_confirmed": + if not (proforma or stage in {"QUOTE_SENT", "PROFORMA_SENT", "WAITING_PAYMENT", "PAYMENT_CONFIRMED"}): + return "Confirmação de pagamento só deve ocorrer depois de orçamento/pró-forma/pedido de pagamento." + return None + + if action_key == "odoo_sale_order": + if not payment: + return "Só é possível criar/registar venda Odoo após pagamento confirmado." + return None + + if action_key == "odoo_production": + if not sale: + return "Produção/preparação exige venda Odoo criada." + return None + + if action_key == "odoo_physical_validated": + if not sale: + return "Validação física exige venda Odoo criada." + if not ready_to_ship: + return "Odoo ainda não indica encomenda pronta para despacho." + return None + + if action_key == "jasmin_invoice": + if not payment: + return "Fatura Jasmin exige pagamento confirmado." + if not sale: + return "Fatura Jasmin exige venda Odoo criada." + return None + + if action_key == "packlink_shipment": + if not payment: + return "Envio Packlink exige pagamento confirmado." + if not invoice: + return "Envio Packlink exige fatura emitida." + if not (ready_to_ship or physical_validated): + return "Odoo ainda não indica encomenda pronta para despacho." + return None + + if action_key == "tracking_sent": + if not shipment: + return "Só é possível enviar/registar tracking após criar envio Packlink." + return None + + if action_key == "delivered": + if not (tracking or shipment): + return "Só é possível marcar entregue depois de envio/tracking." + return None + + return f"Ação desconhecida: {action_key}" + + +def validate_operation_action(opportunity_id: str, action_key: str) -> None: + ctx = get_workflow_context(opportunity_id) + reason = blocked_reason(ctx, action_key) + if reason: + raise OperationActionBlocked(reason) + + +def get_workflow_action_plan(opportunity_id: str) -> Dict[str, Any]: + ctx = get_workflow_context(opportunity_id) + opp = ctx.get("opportunity") or {} + stage = str(opp.get("stage") or "") + status = str(opp.get("status") or "") + + item_count = int(ctx.get("item_count") or 0) + physical_payload = ctx.get("physical_payload") or {} + ready_to_ship = bool(ctx.get("ready_to_ship")) + + quotation = _has(ctx, "jasmin:quotation", {"created", "sent", "converted"}) or int(ctx.get("jasmin_quotation_count") or 0) > 0 + proforma = _has(ctx, "jasmin:proforma", {"issued"}) or int(ctx.get("jasmin_proforma_count") or 0) > 0 + payment = _has(ctx, "clientflow:payment", {"confirmed"}) + sale = _has(ctx, "odoo:sale_order", {"created"}) + physical_status = _has(ctx, "odoo:physical_status") + physical_validated = _has(ctx, "odoo:physical_validation", {"validated"}) + invoice = _has(ctx, "jasmin:invoice", {"issued"}) or int(ctx.get("jasmin_invoice_count") or 0) > 0 + shipment = _has(ctx, "packlink:shipment", {"created"}) + tracking = _has(ctx, "clientflow:tracking", {"sent"}) + delivered = _has(ctx, "packlink:delivery", {"delivered"}) + + if status == "closed" or stage in {"WON", "LOST", "NO_INTEREST"} or delivered: + if stage == "NO_INTEREST": + state_label = "Sem interesse" + reason = "Cliente indicou ausência de interesse atual/necessidade." + elif stage == "LOST": + state_label = "Perdida" + reason = "Oportunidade perdida por decisão comercial." + else: + state_label = "Concluída" + reason = "Processo concluído." + next_action = {"kind": "none", "label": "Sem ação necessária", "reason": reason} + elif invoice and not payment: + state_label = "Fatura enviada" + next_action = {"kind": "operation", "action_key": "payment_confirmed", "label": ACTION_LABELS["payment_confirmed"], "reason": "Fatura Jasmin já existe; confirmar pagamento ou rever se é registo histórico/reconstruído."} + elif item_count <= 0 and not (quotation or proforma or invoice or payment or sale): + state_label = "Preparar proposta" + next_action = {"kind": "manual", "label": "Adicionar produto/proposta", "reason": "A oportunidade ainda não tem produtos."} + elif not quotation and not proforma and not invoice and not payment: + state_label = "Proposta/pagamento" + next_action = {"kind": "operation", "action_key": "jasmin_quotation", "label": ACTION_LABELS["jasmin_quotation"], "reason": "Criar orçamento Jasmin para validação pelo cliente."} + elif not payment: + state_label = "A aguardar pagamento" + next_action = {"kind": "operation", "action_key": "payment_confirmed", "label": ACTION_LABELS["payment_confirmed"], "reason": "Confirmar quando o pagamento estiver validado."} + elif not sale: + state_label = "Pagamento confirmado" + next_action = {"kind": "operation", "action_key": "odoo_sale_order", "label": ACTION_LABELS["odoo_sale_order"], "reason": "Criar/registar venda oficial no Odoo."} + elif not physical_status: + state_label = "Venda criada no Odoo" + next_action = {"kind": "sync_odoo", "label": "Sincronizar estado Odoo", "reason": "Ler se a encomenda física está em produção, bloqueada ou pronta."} + elif not ready_to_ship: + state_label = physical_payload.get("label") or "Em preparação" + next_action = {"kind": "wait", "label": "Aguardar Odoo", "reason": physical_payload.get("next_action") or physical_payload.get("reason") or "Odoo ainda não indica pronto para despacho."} + elif not physical_validated: + state_label = "Pronta para despacho" + next_action = {"kind": "operation", "action_key": "odoo_physical_validated", "label": ACTION_LABELS["odoo_physical_validated"], "reason": "Confirmar no ClientFlow que a encomenda física está pronta."} + elif not invoice: + state_label = "Pronta para faturar" + next_action = {"kind": "operation", "action_key": "jasmin_invoice", "label": ACTION_LABELS["jasmin_invoice"], "reason": "Emitir/registar fatura antes do envio."} + elif not shipment: + state_label = "Pronta para envio" + next_action = {"kind": "operation", "action_key": "packlink_shipment", "label": ACTION_LABELS["packlink_shipment"], "reason": "Criar envio/recolha Packlink."} + elif not tracking: + state_label = "Envio criado" + next_action = {"kind": "operation", "action_key": "tracking_sent", "label": ACTION_LABELS["tracking_sent"], "reason": "Enviar tracking ao cliente."} + else: + state_label = "A acompanhar entrega" + next_action = {"kind": "operation", "action_key": "delivered", "label": ACTION_LABELS["delivered"], "reason": "Marcar como entregue/concluído quando confirmado."} + + allowed = [] + blocked = [] + for key, label in ACTION_LABELS.items(): + reason = blocked_reason(ctx, key) + item = {"action_key": key, "label": label, "reason": reason or ""} + if reason: + blocked.append(item) + else: + allowed.append(item) + + return { + "state_label": state_label, + "stage": stage, + "status": status, + "ready_to_ship": ready_to_ship, + "physical_label": physical_payload.get("label") or "", + "physical_reason": physical_payload.get("reason") or "", + "physical_next_action": physical_payload.get("next_action") or "", + "next_action": next_action, + "allowed_actions": allowed, + "blocked_actions": blocked, + } diff --git a/deploy/systemd/clientflow-fiscal-enrichment.service b/deploy/systemd/clientflow-fiscal-enrichment.service new file mode 100644 index 0000000..bc81725 --- /dev/null +++ b/deploy/systemd/clientflow-fiscal-enrichment.service @@ -0,0 +1,10 @@ +[Unit] +Description=ClientFlow Fiscal Enrichment Worker +After=network.target postgresql.service + +[Service] +Type=oneshot +User=plx +WorkingDirectory=/mnt/ssd/home/plx/clientflow_backend +EnvironmentFile=/mnt/ssd/home/plx/clientflow_backend/.env +ExecStart=/mnt/ssd/home/plx/clientflow_backend/.venv/bin/python /mnt/ssd/home/plx/clientflow_backend/scripts/enrich_fiscal_customers.py --incremental --limit 100 diff --git a/deploy/systemd/clientflow-fiscal-enrichment.timer b/deploy/systemd/clientflow-fiscal-enrichment.timer new file mode 100644 index 0000000..95fe45c --- /dev/null +++ b/deploy/systemd/clientflow-fiscal-enrichment.timer @@ -0,0 +1,11 @@ +[Unit] +Description=Run ClientFlow Fiscal Enrichment Worker every 15 minutes + +[Timer] +OnBootSec=3min +OnUnitActiveSec=15min +AccuracySec=30s +Unit=clientflow-fiscal-enrichment.service + +[Install] +WantedBy=timers.target diff --git a/deploy/systemd/clientflow-outbox-jasmin.service b/deploy/systemd/clientflow-outbox-jasmin.service new file mode 100644 index 0000000..f0e4e59 --- /dev/null +++ b/deploy/systemd/clientflow-outbox-jasmin.service @@ -0,0 +1,13 @@ +[Unit] +Description=ClientFlow Jasmin Outbox Worker +After=network.target postgresql.service + +[Service] +Type=oneshot +User=plx +WorkingDirectory=/mnt/ssd/home/plx/clientflow_backend +EnvironmentFile=/mnt/ssd/home/plx/clientflow_backend/.env +Environment=JASMIN_OUTBOX_ENABLED=true +Environment=OUTBOX_TARGET_SYSTEM=jasmin +Environment=OUTBOX_DRY_RUN=false +ExecStart=/mnt/ssd/home/plx/clientflow_backend/.venv/bin/python /mnt/ssd/home/plx/clientflow_backend/scripts/process_outbox.py diff --git a/deploy/systemd/clientflow-outbox-jasmin.timer b/deploy/systemd/clientflow-outbox-jasmin.timer new file mode 100644 index 0000000..aca5984 --- /dev/null +++ b/deploy/systemd/clientflow-outbox-jasmin.timer @@ -0,0 +1,11 @@ +[Unit] +Description=Run ClientFlow Jasmin Outbox Worker every minute + +[Timer] +OnBootSec=2min +OnUnitActiveSec=1min +AccuracySec=10s +Unit=clientflow-outbox-jasmin.service + +[Install] +WantedBy=timers.target diff --git a/deploy/systemd/clientflow-outbox-packlink.service b/deploy/systemd/clientflow-outbox-packlink.service new file mode 100644 index 0000000..a21706d --- /dev/null +++ b/deploy/systemd/clientflow-outbox-packlink.service @@ -0,0 +1,13 @@ +[Unit] +Description=ClientFlow Packlink Outbox Worker +After=network.target postgresql.service + +[Service] +Type=oneshot +User=plx +WorkingDirectory=/mnt/ssd/home/plx/clientflow_backend +EnvironmentFile=/mnt/ssd/home/plx/clientflow_backend/.env +Environment=PACKLINK_OUTBOX_ENABLED=true +Environment=OUTBOX_TARGET_SYSTEM=packlink +Environment=OUTBOX_DRY_RUN=true +ExecStart=/mnt/ssd/home/plx/clientflow_backend/.venv/bin/python /mnt/ssd/home/plx/clientflow_backend/scripts/process_outbox.py diff --git a/deploy/systemd/clientflow-outbox-packlink.timer b/deploy/systemd/clientflow-outbox-packlink.timer new file mode 100644 index 0000000..8aa2f9c --- /dev/null +++ b/deploy/systemd/clientflow-outbox-packlink.timer @@ -0,0 +1,11 @@ +[Unit] +Description=Run ClientFlow Packlink Outbox Worker every 2 minutes + +[Timer] +OnBootSec=3min +OnUnitActiveSec=2min +AccuracySec=20s +Unit=clientflow-outbox-packlink.service + +[Install] +WantedBy=timers.target diff --git a/docs/CLIENTFLOW_V31_NOTES.md b/docs/CLIENTFLOW_V31_NOTES.md new file mode 100644 index 0000000..9b0fe46 --- /dev/null +++ b/docs/CLIENTFLOW_V31_NOTES.md @@ -0,0 +1,109 @@ +# ClientFlow v3.1 — correções Jasmin/Packlink + +Esta versão é incremental sobre a v3 e corrige os pontos encontrados nos testes remotos. + +## Alterações principais + +### Produtos + +Agora existe separação entre: + +- `sku`: código interno/Odoo; +- `jasmin_sales_item`: artigo de venda no Jasmin. + +Exemplo: + +```text +sku = ODOO-1 +jasmin_sales_item = CARREGADOR_MONO_7KW +``` + +As tabelas recebem colunas novas de forma aditiva: + +```sql +ALTER TABLE products ADD COLUMN IF NOT EXISTS jasmin_sales_item TEXT; +ALTER TABLE opportunity_items ADD COLUMN IF NOT EXISTS jasmin_sales_item TEXT; +ALTER TABLE order_items ADD COLUMN IF NOT EXISTS jasmin_sales_item TEXT; +``` + +Quando uma linha é adicionada à oportunidade a partir do catálogo, `opportunity_items.jasmin_sales_item` é copiado do produto. + +A criação de orçamento Jasmin usa esta prioridade: + +```text +metadata.jasmin_sales_item +opportunity_items.jasmin_sales_item +JASMIN_DEFAULT_SALES_ITEM +sku +``` + +Assim o SKU Odoo deixa de ser usado como artigo Jasmin, exceto como fallback. + +### Documentos Jasmin + +Depois de criar orçamento/fatura, o serviço faz um `GET` ao Jasmin para tentar preencher: + +- `document_type` +- `serie` +- `series_number` +- `document_number` +- `total_amount` +- `currency` + +Isto melhora a tabela `commercial_documents` e a visualização na oportunidade. + +### UI da oportunidade + +- O formulário **Adicionar produto** saiu da área técnica e passou para o cartão **Produtos**. +- Os botões **Criar orçamento** e **Converter em fatura** mostram mensagem de feedback informando que o pedido foi enviado para a outbox. +- A tabela de produtos mostra SKU/Odoo e artigo Jasmin separadamente. + +### Timers systemd + +Foram adicionados exemplos em `deploy/systemd/`: + +- `clientflow-outbox-jasmin.service` +- `clientflow-outbox-jasmin.timer` +- `clientflow-outbox-packlink.service` +- `clientflow-outbox-packlink.timer` + +Instalação Jasmin: + +```bash +sudo cp deploy/systemd/clientflow-outbox-jasmin.* /etc/systemd/system/ +sudo systemctl daemon-reload +sudo systemctl enable --now clientflow-outbox-jasmin.timer +systemctl list-timers | grep clientflow +``` + +O timer Jasmin processa outbox a cada 1 minuto. + +## Verificações após deploy + +```bash +cd /mnt/ssd/home/plx/clientflow_backend +source .venv/bin/activate +python -m py_compile app/*.py scripts/*.py +python -m pytest -q +sudo systemctl restart clientflow-api +sudo systemctl status clientflow-api --no-pager +``` + +## Migração prática de produtos existentes + +Depois do deploy, ir a `/products/{id}` e preencher **Artigo Jasmin** para cada produto usado em documentos. + +Para corrigir via SQL/Python um produto específico: + +```bash +python - <<'PY' +from sqlalchemy import text +from app.db import engine +with engine.begin() as conn: + conn.execute(text(""" + UPDATE products + SET jasmin_sales_item = 'CARREGADOR_MONO_7KW', updated_at = now() + WHERE sku = 'ODOO-1' + """)) +PY +``` diff --git a/docs/CLIENTFLOW_V32_NOTES.md b/docs/CLIENTFLOW_V32_NOTES.md new file mode 100644 index 0000000..603ac9a --- /dev/null +++ b/docs/CLIENTFLOW_V32_NOTES.md @@ -0,0 +1,78 @@ +# ClientFlow v3.2 — HTMX, validações e operação sem terminal + +Esta versão é incremental em cima da v3.1. O objetivo é tornar o fluxo Jasmin/Packlink mais visível e operável pela UI, sem reescrever o dashboard inteiro. + +## Principais alterações + +### HTMX progressivo na oportunidade + +A página de oportunidade passa a usar HTMX em blocos específicos: + +- cartão **Documentos Jasmin**; +- cartão **Produtos**; +- ações de outbox associadas à oportunidade. + +O comportamento esperado é: + +```text +[Criar orçamento] +→ cria item pending na outbox +→ atualiza apenas o cartão Documentos Jasmin +→ mostra estado pending/sent/failed +→ polling atualiza o cartão a cada 10 segundos +``` + +Isto reduz a sensação de que “não aconteceu nada” depois de clicar num botão. + +### Validação antes de criar orçamento + +Antes de criar outbox `jasmin.create_quotation`, o sistema valida: + +- oportunidade com cliente local associado; +- cliente com nome fiscal; +- cliente com NIF; +- morada/código postal/cidade quando ainda não existe `jasmin_customer_party_key`; +- pelo menos uma linha de produto ativa; +- todas as linhas com `jasmin_sales_item` preenchido; +- preços unitários maiores que zero. + +Se faltar algo, a UI mostra o erro diretamente no cartão Jasmin e não cria outbox inválida. + +### Produtos por HTMX + +No cartão **Produtos** da oportunidade: + +- adicionar produto atualiza apenas o cartão; +- remover produto atualiza apenas o cartão; +- total é recalculado no bloco; +- produtos sem `jasmin_sales_item` aparecem como bloqueio visual para orçamento. + +### Outbox visível na oportunidade + +O cartão **Documentos Jasmin** mostra as ações recentes Jasmin relacionadas com a oportunidade: + +- `create_quotation`; +- `convert_quotation_to_invoice`; +- estado `pending`, `sent`, `failed`; +- último erro; +- botão **Reprocessar** para itens `failed`/`pending`. + +### Filtro de produtos sem Artigo Jasmin + +Em `/products`, o filtro de estado ganhou a opção: + +```text +Ativos sem Artigo Jasmin +``` + +Serve para encontrar rapidamente produtos que ainda bloqueiam emissão de orçamento Jasmin. + +## Continuação recomendada + +Para v3.3: + +1. Obter PDF de orçamento/fatura. +2. Mostrar link/número real do documento quando o GET Jasmin devolver `documentType`, `serie` e `seriesNumber`. +3. Adicionar ações HTMX em `/integrations` para testar Jasmin/Packlink. +4. Implementar painel de Packlink semelhante ao painel Jasmin. +5. Refatorar `admin_dashboard.py` para `app/admin_ui/pages/` por etapas. diff --git a/docs/CLIENTFLOW_V33_NOTES.md b/docs/CLIENTFLOW_V33_NOTES.md new file mode 100644 index 0000000..443b613 --- /dev/null +++ b/docs/CLIENTFLOW_V33_NOTES.md @@ -0,0 +1,120 @@ +# ClientFlow v3.3 — HTMX manual e operação Jasmin mais estável + +Esta versão é incremental sobre a v3.2 e foca-se em remover o comportamento visual estranho causado pelo auto-refresh do cartão **Documentos Jasmin**. + +## Alterações principais + +### 1. Documentos Jasmin sem auto-refresh + +Foi removido o polling HTMX automático do cartão de Documentos Jasmin. + +Antes: + +```html +hx-trigger="every 10s" +``` + +Agora: + +- o cartão não se reconstrói sozinho; +- existe botão **Atualizar estado**; +- os botões **Criar orçamento** e **Converter em fatura** continuam a atualizar o cartão uma vez após o clique; +- o operador controla quando quer atualizar o estado. + +Fluxo esperado: + +```text +[Criar orçamento] +→ cria item na outbox +→ mostra feedback imediato +→ timer/worker processa +→ operador clica [Atualizar estado] +→ vê orçamento criado ou erro +``` + +### 2. Feedback visível após ações Jasmin + +Os botões continuam a devolver mensagens visíveis no cartão: + +- Pedido de orçamento enviado para a outbox Jasmin. +- Pedido de fatura enviado para a outbox Jasmin. +- Erros de validação antes de criar a outbox. + +### 3. Outbox Jasmin visível na oportunidade + +O cartão de Documentos Jasmin continua a mostrar as ações da outbox associadas à oportunidade: + +- action type; +- estado; +- erro resumido; +- botão **Reprocessar** para itens pendentes/falhados. + +### 4. Validação antes de criar orçamento + +Mantém-se a validação preventiva: + +- cliente associado; +- NIF válido; +- nome fiscal; +- morada/código postal/cidade quando o cliente ainda não existe no Jasmin; +- pelo menos uma linha de produto; +- todas as linhas com `jasmin_sales_item`; +- preço unitário maior que zero. + +### 5. SKU/Odoo separado de artigo Jasmin + +Mantém-se a regra introduzida na v3.1/v3.2: + +```text +products.sku = código interno/Odoo +products.jasmin_sales_item = artigo Jasmin +opportunity_items.sku = código interno/Odoo +opportunity_items.jasmin_sales_item = artigo Jasmin usado no orçamento +``` + +A criação de orçamento usa `jasmin_sales_item`, não o SKU Odoo. + +## Deploy + +Aplicar como nas versões anteriores: + +```bash +rsync -avz --delete \ + --exclude ".env" \ + --exclude ".venv/" \ + --exclude "venv/" \ + --exclude "__pycache__/" \ + --exclude ".git/" \ + --exclude "*.db" \ + --exclude "*.log" \ + --exclude "uploads/" \ + ./ plx@alarmsys:/mnt/ssd/home/plx/clientflow_backend/ +``` + +Depois no remoto: + +```bash +cd /mnt/ssd/home/plx/clientflow_backend +source .venv/bin/activate +python -m py_compile app/*.py scripts/*.py +sudo systemctl restart clientflow-api +``` + +## Nota operacional + +O botão **Atualizar estado** não processa a outbox. Ele apenas recarrega o cartão. + +Para processar automaticamente, manter ativo o timer systemd da outbox Jasmin: + +```bash +systemctl list-timers | grep clientflow-outbox-jasmin +``` + +Ou processar manualmente: + +```bash +JASMIN_OUTBOX_ENABLED=true \ +OUTBOX_TARGET_SYSTEM=jasmin \ +OUTBOX_DRY_RUN=false \ +python scripts/process_outbox.py +``` diff --git a/docs/CLIENTFLOW_V34_NOTES.md b/docs/CLIENTFLOW_V34_NOTES.md new file mode 100644 index 0000000..78e5f71 --- /dev/null +++ b/docs/CLIENTFLOW_V34_NOTES.md @@ -0,0 +1,60 @@ +# ClientFlow v3.4 — Documentos, PDFs e Outbox Operacional + +Esta versão é incremental sobre a v3.3 e foca-se em reduzir a necessidade de usar o terminal para operar integrações. + +## Alterações principais + +### Documentos Jasmin + +- Adicionados botões no cartão **Documentos Jasmin** da oportunidade: + - **Atualizar nº**: volta a consultar o Jasmin e atualiza número/série/valor do documento local. + - **PDF**: tenta obter o PDF do orçamento ou da fatura diretamente via API Jasmin. +- Mantém atualização manual; não há auto-refresh contínuo do cartão. + +### Outbox operacional + +- Página `/outbox` remodelada para uso operacional. +- Filtros por estado: todas, pendentes, enviadas, falhadas e ignoradas. +- Filtros por sistema: Jasmin, Packlink, Chatwoot, Mautic e Odoo. +- Ações na UI: + - Reprocessar + - Ignorar + - Ver detalhe/payload/erro +- Novo estado `ignored` suportado na outbox. + +### Produtos / Jasmin + +- Página `/products/validate-jasmin` para validar se os artigos configurados em `products.jasmin_sales_item` existem no Jasmin. +- Botão **Validar artigos Jasmin** no catálogo de produtos. +- Mantém a separação: + - `products.sku` = SKU interno/Odoo + - `products.jasmin_sales_item` = artigo Jasmin usado em orçamentos/faturas + +## Validações mantidas + +Antes de criar orçamento Jasmin, continua a validar: + +- Cliente associado à oportunidade +- NIF do cliente +- Nome fiscal +- Morada/código postal/cidade quando for necessário criar cliente no Jasmin +- Pelo menos um produto ativo na oportunidade +- Todos os produtos com `jasmin_sales_item` +- Preços unitários positivos + +## Notas operacionais + +- A obtenção de PDF depende dos endpoints de impressão do Jasmin estarem disponíveis no tenant. +- Se o PDF falhar, a página devolve erro textual sem afetar o documento já criado. +- A outbox Jasmin deve ser processada por timer systemd ou manualmente com `scripts/process_outbox.py`. + +## Validação técnica + +Executado na geração desta versão: + +```bash +python -m py_compile app/*.py scripts/*.py +python -m pytest -q +``` + +Resultado: `4 passed`. diff --git a/docs/CLIENTFLOW_V3_CUSTOMERS_DOCUMENTS.md b/docs/CLIENTFLOW_V3_CUSTOMERS_DOCUMENTS.md new file mode 100644 index 0000000..7993045 --- /dev/null +++ b/docs/CLIENTFLOW_V3_CUSTOMERS_DOCUMENTS.md @@ -0,0 +1,82 @@ +# ClientFlow v3 — Clientes, documentos e envios + +Esta versão separa responsabilidades para evitar que a página da oportunidade fique demasiado pesada. + +## Princípio de UI + +- **Clientes**: dados fiscais, NIF, morada, contactos e ligação Jasmin. +- **Oportunidades**: estado da potencial compra, produtos, documentos associados e envios. +- **Documentos**: orçamentos e faturas em `commercial_documents`. +- **Envios**: registos Packlink em `shipments`. + +## Páginas principais + +### `/customers` + +Lista clientes locais normalizados, com pesquisa por nome, NIF, email, telefone ou `jasmin_customer_party_key`. + +Permite criar uma ficha mínima de cliente com nome e NIF. + +### `/customers/{id}` + +Ficha completa de cliente: + +- nome fiscal; +- NIF; +- email; +- telefone; +- morada fiscal; +- código postal; +- cidade; +- país; +- Jasmin `customerPartyKey`; +- Jasmin ID; +- oportunidades associadas; +- documentos Jasmin; +- envios Packlink. + +### `/opportunities/{id}` + +A oportunidade mantém apenas um resumo compacto do cliente e um seletor para associar uma ficha local. + +Botões operacionais: + +- **Criar orçamento**: cria sempre um novo orçamento Jasmin. +- **Converter em fatura**: converte o orçamento ativo/mais recente em fatura. + +## Modelo de dados + +Novas tabelas/colunas usadas nesta versão: + +- `customers` +- `commercial_documents` +- `commercial_document_lines` +- `shipments` +- `opportunities.local_customer_id` + +O campo antigo `opportunities.customer_id` é mantido para compatibilidade com dados legados/Chatwoot. A ligação nova e normalizada usa `opportunities.local_customer_id`. + +## Fluxo Jasmin validado + +1. Cliente associado à oportunidade. +2. `find_or_create_customer_for_opportunity()` usa primeiro o cliente local. +3. NIF é enviado para Jasmin sem prefixo `PT`. +4. Se o cliente não existir no Jasmin, cria cliente com `partyKey = CF{NIF}`. +5. Cria orçamento `ORC / ORC2026`. +6. Para faturar, chama `POST /billing/invoices/fromQuotation/{quotationId}` com body `{}`. + +## Regras importantes + +- Não enviar `electronicMail` nem `telephone` vazios para Jasmin. +- OData Jasmin deve usar `$top <= 100`. +- Packlink usa API key no header `Authorization`, sem `Bearer`. +- Para cotação Packlink em Portugal, normalizar CP para 4 dígitos. + +## Estratégia de refactor + +Esta v3 ainda não reescreve o `admin_dashboard.py` totalmente. A prioridade foi criar a camada de domínio e a navegação operacional correta. O próximo passo recomendado é extrair progressivamente: + +- `app/admin_ui/pages/customers.py` +- `app/admin_ui/pages/opportunities.py` +- `app/admin_ui/pages/integrations.py` +- `app/admin_ui/components.py` diff --git a/docs/CLIENTFLOW_V40_NOTES.md b/docs/CLIENTFLOW_V40_NOTES.md new file mode 100644 index 0000000..d96f2fb --- /dev/null +++ b/docs/CLIENTFLOW_V40_NOTES.md @@ -0,0 +1,72 @@ +# ClientFlow v4.0 — estabilização operacional + refactor modular incremental + +Esta versão junta a estabilização prevista para v3.5 com o arranque do refactor v4. + +## Incluído + +### Operação segura + +- Migrações SQL versionadas em `migrations/`. +- Script `scripts/apply_migrations.py`. +- Script `scripts/check_clientflow_health.py`. +- Script `scripts/install_systemd_timers.sh`. +- Índices adicionais para outbox, clientes, documentos, envios e produtos. +- Colunas de segurança na outbox: `locked_at`, `lock_owner`, `ignored_at`. +- API interna read-only: + - `/api/internal/health` + - `/api/internal/outbox/summary` + - `/api/internal/documents/summary` + +### Refactor modular incremental + +- Novo pacote `app/admin_ui/`. +- `app/main.py` passa a importar o router admin através de `app.admin_ui.router`. +- O router ainda usa `app.admin_dashboard` internamente para manter compatibilidade. +- Helpers HTML comuns em `app/admin_ui/components.py`. +- Estrutura de destino criada para modularizar: + - `app/customers/` + - `app/opportunities/` + - `app/commercial/` + - `app/logistics/` + - `app/integrations/` + - `app/outbox/` + +## Deploy recomendado + +```bash +cd /mnt/ssd/home/plx/clientflow_backend +source .venv/bin/activate +python scripts/apply_migrations.py --dry-run +python scripts/apply_migrations.py +python -m py_compile app/*.py scripts/*.py +python scripts/check_clientflow_health.py +sudo systemctl restart clientflow-api +``` + +## Timers systemd + +Jasmin: + +```bash +sudo ./scripts/install_systemd_timers.sh /mnt/ssd/home/plx/clientflow_backend +``` + +Packlink, só quando o fluxo de envios estiver validado: + +```bash +ENABLE_PACKLINK_TIMER=true sudo -E ./scripts/install_systemd_timers.sh /mnt/ssd/home/plx/clientflow_backend +``` + +## Próximo passo após v4.0 + +Migrar gradualmente rotas de `app/admin_dashboard.py` para: + +```text +app/admin_ui/pages/customers.py +app/admin_ui/pages/opportunities.py +app/admin_ui/pages/products.py +app/admin_ui/pages/outbox.py +app/admin_ui/pages/integrations.py +``` + +Sem mudar URLs públicas. diff --git a/docs/CLIENTFLOW_V41_NOTES.md b/docs/CLIENTFLOW_V41_NOTES.md new file mode 100644 index 0000000..3c712c1 --- /dev/null +++ b/docs/CLIENTFLOW_V41_NOTES.md @@ -0,0 +1,34 @@ +# ClientFlow v4.1 — Correções de deploy e scripts operacionais + +Esta versão é incremental sobre a v4.0 e corrige o problema encontrado no remoto: + +```text +ModuleNotFoundError: No module named 'app' +``` + +## Alterações + +- `scripts/apply_migrations.py` passa a resolver automaticamente a raiz do projeto. +- `scripts/check_clientflow_health.py` passa a resolver automaticamente a raiz do projeto. +- `scripts/process_outbox.py` também passa a mudar para a raiz do projeto antes de carregar `.env`. +- `scripts/test_jasmin_connection.py` e `scripts/test_packlink_connection.py` passam a carregar `.env` a partir da raiz do projeto mesmo quando executados de outra pasta. + +## Comandos depois do deploy + +```bash +cd /mnt/ssd/home/plx/clientflow_backend +source .venv/bin/activate + +python scripts/apply_migrations.py --dry-run +python scripts/apply_migrations.py +python scripts/check_clientflow_health.py + +sudo systemctl restart clientflow-api +sudo systemctl status clientflow-api --no-pager +``` + +Já não deve ser necessário executar com `PYTHONPATH=.`. + +## Nota + +O `.env` continua excluído do `rsync` e deve permanecer apenas no servidor remoto. diff --git a/docs/CLIENTFLOW_V42_NOTES.md b/docs/CLIENTFLOW_V42_NOTES.md new file mode 100644 index 0000000..f12475f --- /dev/null +++ b/docs/CLIENTFLOW_V42_NOTES.md @@ -0,0 +1,59 @@ +# ClientFlow v4.2 — Operação Comercial e Logística + +Esta versão melhora a maturidade operacional do ClientFlow sem alterar a arquitetura principal. + +## Alterações principais + +- Nova página `/operations` / `/operacoes` para operação diária. +- Nova página `/system/health` para saúde operacional. +- Timeline da oportunidade passa a juntar eventos, produtos, documentos, outbox e envios. +- API interna adiciona: + - `/api/internal/operations/summary` + - `/api/internal/system/health` +- Navegação passa a separar: + - Operações + - Outbox + - Integrações +- A página de operações destaca: + - oportunidades abertas + - outbox pendente/falhada + - faturas emitidas hoje + - orçamentos abertos + - envios pendentes + - clientes incompletos + - produtos sem Artigo Jasmin + +## Objetivo + +Reduzir a dependência do terminal para operação diária. A equipa deve conseguir abrir `/operations` e saber rapidamente o que precisa de ação. + +## Rotas novas + +```text +/operations +/operacoes +/system/health +/api/internal/operations/summary +/api/internal/system/health +``` + +## Notas de deploy + +Depois do rsync: + +```bash +cd /mnt/ssd/home/plx/clientflow_backend +source .venv/bin/activate +python scripts/apply_migrations.py --dry-run +python scripts/apply_migrations.py +python scripts/check_clientflow_health.py +sudo systemctl restart clientflow-api +``` + +## Próximas melhorias sugeridas + +- Packlink com cotação antes de criar envio. +- Guardar PDFs localmente. +- Backups/restore via scripts próprios. +- Duplicados de clientes por NIF. +- Sincronização de preços Jasmin vs Odoo. diff --git a/docs/CLIENTFLOW_V42_REVIEW_UI_NOTES.md b/docs/CLIENTFLOW_V42_REVIEW_UI_NOTES.md new file mode 100644 index 0000000..82046f3 --- /dev/null +++ b/docs/CLIENTFLOW_V42_REVIEW_UI_NOTES.md @@ -0,0 +1,42 @@ +# ClientFlow v4.2 — revisão operacional e UI + +Esta revisão mantém a arquitetura da v4.2, mas corrige pontos de estabilidade e torna a UI mais operacional. + +## Correções aplicadas + +- `OUTBOX_DRY_RUN=true` deixou de marcar itens como `sent`; agora passa o item para `dry_run`. +- Integrações com `*_OUTBOX_ENABLED=false` deixam de ficar apenas em `skipped` silencioso; agora ficam `blocked` com erro claro. +- A idempotência de `jasmin.create_quotation` passou a ser determinística por oportunidade: `jasmin:quotation:{opportunity_id}`. +- `integration_outbox` ganhou estados visíveis e reprocessáveis: `pending`, `processing`, `sent`, `failed`, `blocked`, `dry_run`, `ignored`, `cancelled`. +- O schema base passa a garantir colunas de controlo da outbox (`locked_at`, `lock_owner`, `ignored_at`) mesmo em instalações novas. +- Webhooks Chatwoot exigem `CLIENTFLOW_WEBHOOK_SECRET` em `ENV=prod`, `production` ou `staging`. +- Admin UI e API interna podem ser protegidas com `CLIENTFLOW_ADMIN_TOKEN`. + +## Melhorias UI + +- Página da oportunidade ganhou painel **Integrações da oportunidade** com contadores: + - pendentes; + - falhadas; + - bloqueadas; + - dry-run. +- A oportunidade passa a mostrar ações Jasmin/Packlink/Chatwoot/Mautic ligadas à oportunidade, com último erro e reprocessamento. +- `/outbox` ganhou filtros para os novos estados operacionais. +- Badges visuais foram atualizados para `processing`, `blocked`, `dry_run` e `cancelled`. + +## Validação + +Executado: + +```bash +python -m compileall app scripts tests +pytest -q +``` + +Resultado: `12 passed`. + +## Ainda recomendado para próxima ronda + +- Substituir o token simples por login/sessão + CSRF nos POSTs. +- Refatorar `admin_dashboard.py`, que continua demasiado grande. +- Confirmar endpoints reais de PDF Jasmin no tenant de produção. +- Criar testes funcionais com Postgres de teste e mocks para Jasmin/Packlink. diff --git a/docs/CLIENTFLOW_V451_UI_ALIGNMENT.md b/docs/CLIENTFLOW_V451_UI_ALIGNMENT.md new file mode 100644 index 0000000..59dd06f --- /dev/null +++ b/docs/CLIENTFLOW_V451_UI_ALIGNMENT.md @@ -0,0 +1,89 @@ +# ClientFlow v4.5.1 — UI Alignment + +Esta atualização alinha a UI real instalada com o mockup aprovado para a organização v4.5 Operational Core. + +## Objetivo + +Manter a lógica e os endpoints da v4.5, mas tornar a interface mais próxima do mockup: + +- sidebar clara e moderna; +- topbar com pesquisa global; +- cartões KPI mais legíveis; +- tabelas com linhas em formato de cartões; +- melhor responsividade em mobile; +- distinção visual mais clara entre Dashboard, Centro de trabalho, Comunicações, Oportunidades, Outbox e Integrações. + +## Escopo + +Esta versão é essencialmente visual. + +Não altera: + +- migrações da BD; +- lógica Jasmin; +- lógica Packlink; +- outbox worker; +- webhooks; +- serviços de email; +- endpoints principais. + +## Páginas afetadas + +Como o layout base é partilhado, a melhoria visual afeta a UI admin em geral: + +- `/` +- `/operations` +- `/communications` +- `/opportunities` +- `/outbox` +- `/integrations` +- `/products` +- `/customers` + +## Atualização em campo + +Não é necessário aplicar migração nova. + +Procedimento recomendado: + +```bash +cd /home/ricar/Transferências/clientflow_backend_v451_ui_alignment + +rsync -avz --delete --dry-run \ + --exclude ".env" \ + --exclude ".venv/" \ + --exclude "venv/" \ + --exclude "__pycache__/" \ + --exclude ".git/" \ + --exclude "*.db" \ + --exclude "*.log" \ + --exclude "uploads/" \ + ./ plx@alarmsys:/mnt/ssd/home/plx/clientflow_backend/ +``` + +Se o dry-run estiver correto: + +```bash +rsync -avz --delete \ + --exclude ".env" \ + --exclude ".venv/" \ + --exclude "venv/" \ + --exclude "__pycache__/" \ + --exclude ".git/" \ + --exclude "*.db" \ + --exclude "*.log" \ + --exclude "uploads/" \ + ./ plx@alarmsys:/mnt/ssd/home/plx/clientflow_backend/ +``` + +No servidor: + +```bash +cd /mnt/ssd/home/plx/clientflow_backend +source .venv/bin/activate 2>/dev/null || true +PYTHONPATH=. python -m compileall app scripts tests +pytest -q +sudo systemctl restart clientflow-api +sudo systemctl status clientflow-api --no-pager +journalctl -u clientflow-api -n 80 --no-pager +``` diff --git a/docs/CLIENTFLOW_V452_UI_ICONS_FIX.md b/docs/CLIENTFLOW_V452_UI_ICONS_FIX.md new file mode 100644 index 0000000..2679444 --- /dev/null +++ b/docs/CLIENTFLOW_V452_UI_ICONS_FIX.md @@ -0,0 +1,19 @@ +# ClientFlow v4.5.2 — UI Icons Fix + +Patch visual pequeno sobre a v4.5.1. + +## O que corrige + +- Os cartões KPI da v4.5.1 mostravam caixas azul-claro vazias onde deveriam existir ícones. +- A v4.5.2 adiciona ícones/fallback visuais nos KPI cards através de CSS seguro. +- Adiciona Bootstrap Icons por CDN para uso progressivo nos templates. +- Corrige um pequeno bloco duplicado no rodapé da sidebar. + +## O que não altera + +- Base de dados +- Migrações +- Jasmin +- Packlink +- Workers/outbox +- Webhooks diff --git a/docs/CLIENTFLOW_V453_UI_POLISH.md b/docs/CLIENTFLOW_V453_UI_POLISH.md new file mode 100644 index 0000000..2312031 --- /dev/null +++ b/docs/CLIENTFLOW_V453_UI_POLISH.md @@ -0,0 +1,49 @@ +# ClientFlow v4.5.3 — Semantic Icons + UI Polish + +Esta microversão corrige o acabamento visual da v4.5.2. + +## Objetivo + +Substituir os ícones genéricos dos cards KPI por ícones semânticos, mantendo a lógica operacional e as integrações intactas. + +## Alterações + +- Atualização do badge visual para `v4.5.3 UI`. +- Sidebar passa a usar Bootstrap Icons semânticos. +- Cards KPI passam a renderizar um elemento `.cf-kpi-icon` com ícone Bootstrap. +- Ícones automáticos por tema: + - oportunidades/pipeline: funil; + - documentos/orçamentos: documento; + - finanças/pagamentos: euro; + - tasks: checklist; + - comunicações/email: envelope; + - erros/bloqueios: alerta; + - envios/logística: camião; + - clientes: cartão de cliente; + - produtos/Jasmin: caixa/artigo; + - integrações/sistema: puzzle. +- Tons visuais por tipo: verde, laranja, vermelho e roxo quando aplicável. +- Mantém Bootstrap 5 + Bootstrap Icons via CDN. + +## Não altera + +- Base de dados. +- Migrações. +- Jasmin. +- Packlink. +- Outbox worker. +- Webhooks. +- Lógica de documentos comerciais. + +## Validação + +```bash +PYTHONPATH=. python -m compileall app scripts tests +PYTHONPATH=. pytest -q +``` + +Resultado esperado: + +```text +19 passed +``` diff --git a/docs/CLIENTFLOW_V45_OPERATIONAL_CORE.md b/docs/CLIENTFLOW_V45_OPERATIONAL_CORE.md new file mode 100644 index 0000000..ed900e7 --- /dev/null +++ b/docs/CLIENTFLOW_V45_OPERATIONAL_CORE.md @@ -0,0 +1,110 @@ +# ClientFlow v4.5 — Operational Core + +Esta versão junta a proposta v4.3 + v4.4 numa atualização única e incremental para o sistema em campo. + +## Objetivo + +Transformar a UI num sistema operacional diário: + +```text +Email / Chatwoot / pedido recebido +→ Comunicação classificada +→ Centro de trabalho +→ Cliente / oportunidade +→ Task humana ou Outbox automática +→ Documento / envio +→ Timeline +``` + +## O que muda + +### Dashboard + +O `/` passa a ser uma visão geral: métricas, alertas e ligação direta ao Centro de trabalho. Deixa de ser a página principal para execução item a item. + +### Centro de trabalho + +O `/operations` passa a juntar itens acionáveis: + +- tasks humanas pendentes; +- comunicações por tratar; +- outbox pending/failed/blocked; +- clientes/produtos/documentos que bloqueiam o fluxo. + +### Comunicações + +Nova área: + +```text +/communications +/comunicacoes +``` + +Guarda emails/mensagens classificados com ligação opcional a cliente, oportunidade e task. + +Estados suportados: + +```text +new +classified +needs_review +linked +task_created +done +ignored +``` + +### Oportunidade + +A página da oportunidade passa a incluir navegação por secções: + +```text +Resumo +Produtos +Documentos +Tasks +Comunicações +Outbox +Timeline +Técnico +``` + +### Tasks contextualizadas + +A tabela `tasks` recebe colunas para contexto operacional: + +```text +communication_id +document_id +shipment_id +outbox_id +priority +assigned_to +``` + +### Timeline unificada + +Nova tabela `timeline_events`, integrada na timeline da oportunidade. + +## Instalação + +1. Fazer backup da base de dados. +2. Aplicar código v4.5. +3. Reiniciar serviço. +4. Confirmar que o startup executou `init_db()`. +5. Opcional: aplicar manualmente `migrations/006_v45_operational_core.sql`. +6. Validar no browser: + +```text +/ +/operations +/communications +/opportunities/{id} +``` + +## Notas + +- Não muda para React/Vue. +- Mantém FastAPI + HTML/HTMX. +- Não remove rotas antigas. +- A automação crítica continua por outbox e deve continuar a exigir validação humana quando aplicável. diff --git a/docs/CLIENTFLOW_V45_UPDATE_GUIDE.md b/docs/CLIENTFLOW_V45_UPDATE_GUIDE.md new file mode 100644 index 0000000..0b07dc0 --- /dev/null +++ b/docs/CLIENTFLOW_V45_UPDATE_GUIDE.md @@ -0,0 +1,83 @@ +# Guia de atualização — ClientFlow v4.5 Operational Core + +## Antes de atualizar + +1. Fazer backup da base de dados PostgreSQL. +2. Guardar o `.env` atual. +3. Confirmar que os timers/serviços atuais estão estáveis: + +```bash +systemctl status clientflow-api.service +systemctl list-timers | grep clientflow +``` + +## Instalação recomendada + +```bash +# exemplo; ajustar diretórios ao servidor real +cd /opt/clientflow +systemctl stop clientflow-api.service +cp -a clientflow_backend clientflow_backend_backup_$(date +%Y%m%d_%H%M) +unzip clientflow_backend_v45_operational_core.zip -d /opt/clientflow/clientflow_backend_v45 +cp /opt/clientflow/clientflow_backend/.env /opt/clientflow/clientflow_backend_v45/.env +cd /opt/clientflow/clientflow_backend_v45 +python -m compileall app scripts tests +pytest -q +systemctl start clientflow-api.service +``` + +Se o serviço usa um caminho fixo, trocar o symlink/current release conforme o teu deploy atual. + +## Migração de base de dados + +A v4.5 é aditiva. O `init_db()` cria/atualiza: + +```text +communications +timeline_events +tasks.communication_id +tasks.document_id +tasks.shipment_id +tasks.outbox_id +tasks.priority +tasks.assigned_to +``` + +Também existe migração SQL explícita: + +```bash +psql "$DATABASE_URL" -f migrations/006_v45_operational_core.sql +``` + +## Validação pós-instalação + +Abrir: + +```text +/ +/operations +/communications +/opportunities +/outbox +/integrations +``` + +Verificar: + +```text +Dashboard mostra apenas visão geral. +Centro de trabalho mostra fila operacional. +Comunicações abre sem erro. +Oportunidade mostra tabs: Resumo, Produtos, Documentos, Tasks, Comunicações, Outbox, Timeline, Técnico. +Outbox continua a processar sem marcar dry_run como sent. +``` + +## Rollback + +Como a migração é aditiva, o rollback de código pode ser feito voltando para a pasta/ZIP anterior. As tabelas e colunas novas podem ficar na BD sem afetar a versão anterior. + +## Limitações conhecidas + +- A v4.5 cria a estrutura de Comunicações, mas o conector real de email/classificação deve inserir dados nessa tabela. +- A confirmação de pagamentos e ações críticas continuam humanas. +- Permissões e CSRF completos continuam recomendados para uma ronda posterior se o sistema estiver exposto fora da rede controlada. diff --git a/docs/CLIENTFLOW_V461_CUSTOMER_MISMATCH_WARNING.md b/docs/CLIENTFLOW_V461_CUSTOMER_MISMATCH_WARNING.md new file mode 100644 index 0000000..ee70dbf --- /dev/null +++ b/docs/CLIENTFLOW_V461_CUSTOMER_MISMATCH_WARNING.md @@ -0,0 +1,22 @@ +# ClientFlow v4.6.1 — Customer mismatch warning tuning + +Esta correção reduz falsos positivos no aviso “contacto e cliente fiscal não coincidem”. + +O caso típico era uma oportunidade com contacto curto/truncado, por exemplo: + +- Contacto/origem: `Riotec elec` +- Cliente fiscal: `Riotec - Electricidade, Aquecimento e Controle, Lda` + +A heurística anterior comparava strings completas e gerava alerta. A nova heurística compara tokens significativos e aceita abreviações/prefixos como o mesmo cliente provável. + +## Desativar completamente o aviso + +Se o aviso continuar a criar ruído operacional, adiciona no `.env`: + +```env +CLIENTFLOW_CUSTOMER_MISMATCH_WARNING_ENABLED=false +``` + +Depois reinicia o serviço. + +A validação real antes de criar documentos continua a depender dos dados fiscais do cliente associado e das validações Jasmin/Packlink. diff --git a/docs/CLIENTFLOW_V462_DISABLE_CUSTOMER_MISMATCH_WARNING.md b/docs/CLIENTFLOW_V462_DISABLE_CUSTOMER_MISMATCH_WARNING.md new file mode 100644 index 0000000..dd3ad86 --- /dev/null +++ b/docs/CLIENTFLOW_V462_DISABLE_CUSTOMER_MISMATCH_WARNING.md @@ -0,0 +1,32 @@ +# ClientFlow v4.6.2 — Customer warning off by default + opportunity layout tuning + +Esta versão desativa por defeito o aviso de divergência entre contacto/origem e cliente fiscal associado. + +## Motivo + +Na operação real é comum a conversa chegar por um contacto abreviado ou pessoal, mas o documento fiscal pertencer a uma empresa. Exemplos válidos: + +- `Riotec elec` → `Riotec - Electricidade, Aquecimento e Controle, Lda` +- `Bruno Oliveira` → `Nortuflex` + +Nestes casos o aviso gerava ruído e podia atrasar a operação. + +## Comportamento novo + +Por defeito: + +```env +CLIENTFLOW_CUSTOMER_MISMATCH_WARNING_ENABLED=false +``` + +O aviso pode ser reativado no `.env` se necessário: + +```env +CLIENTFLOW_CUSTOMER_MISMATCH_WARNING_ENABLED=true +``` + +## Ajuste visual + +A página da oportunidade passa para uma coluna mais cedo em ecrãs/zoom de laptop, evitando que o painel lateral comprima a área principal. + +Não há migração de base de dados. diff --git a/docs/CLIENTFLOW_V462_WORKFLOW_ACTIONS_CLEANUP.md b/docs/CLIENTFLOW_V462_WORKFLOW_ACTIONS_CLEANUP.md new file mode 100644 index 0000000..0afc86f --- /dev/null +++ b/docs/CLIENTFLOW_V462_WORKFLOW_ACTIONS_CLEANUP.md @@ -0,0 +1,43 @@ +# ClientFlow v4.6.2 — Workflow Actions Cleanup + +Correção pequena focada em operação diária, sem migrações de base de dados e sem alterações em Jasmin/Packlink. + +## Alterações + +- Remove o aviso operacional por divergência de nome entre contacto e cliente fiscal. + - Contacto pessoal e empresa fiscal podem ser diferentes e estar corretos. + - A validação crítica deve ficar em NIF, dados fiscais obrigatórios, documento emitido e associação fiscal, não em semelhança de nomes. +- Adiciona a ação `MARK_NO_INTEREST`. + - Usada quando o cliente indica que não tem interesse, não tem frota/veículos elétricos, não necessita ou a oferta não se aplica. +- Adiciona regra determinística para mensagens inequívocas, por exemplo: + - “não temos veículos elétricos” + - “não temos na nossa frota veículos elétricos” + - “não estamos interessados” + - “não se aplica” +- `MARK_NO_INTEREST` cria trabalho pendente em vendas para o operador confirmar/fechar o caso. +- Se existir oportunidade aberta, a task pode ficar ligada à oportunidade. +- Ao concluir a task, a oportunidade pode avançar para `LOST`. +- Se não existir oportunidade aberta, `MARK_NO_INTEREST` não cria uma oportunidade nova só para a fechar. +- Ajusta CSS da ficha da oportunidade para reduzir quebra de layout em zoom/larguras intermédias. + +## Exemplo + +Mensagem: + +> Agradecemos a informação, mas não temos na nossa frota veículos elétricos. + +Resultado esperado: + +- `action_code = MARK_NO_INTEREST` +- `route = vendas` +- `status = pending` +- ação: “Marcar sem interesse” + +## Não altera + +- Base de dados +- Migrações +- Jasmin +- Packlink +- Outbox worker +- Webhooks Chatwoot diff --git a/docs/CLIENTFLOW_V463_OPERATIONS_QUEUE_CLEANUP.md b/docs/CLIENTFLOW_V463_OPERATIONS_QUEUE_CLEANUP.md new file mode 100644 index 0000000..0a4dce8 --- /dev/null +++ b/docs/CLIENTFLOW_V463_OPERATIONS_QUEUE_CLEANUP.md @@ -0,0 +1,27 @@ +# ClientFlow v4.6.3 — Operations Queue Cleanup + +Esta versão é um hotfix operacional, sem migração de base de dados. + +## Alterações + +- `MARK_NO_INTEREST` deixa de fechar oportunidades como `LOST`. +- Novo estado de oportunidade: `NO_INTEREST` / “Sem interesse”. +- `LOST` fica reservado para oportunidades perdidas por preço, funcionalidades, desistência ou concorrência. +- O Centro de trabalho deixa de contar/mostrar outbox `failed` cujo erro indique que foi “limpo manualmente” ou “resolvido manualmente”. +- Textos técnicos como “Resposta LLM inválida para action_code” passam a aparecer como “Classificação da mensagem falhou”. +- “Emails por tratar” passa a “Mensagens para revisão”. +- “Sem cliente” passa a “Mensagens sem cliente”. +- Documentos no Centro de trabalho passam a focar documentos que podem exigir ação. +- Novo script: `scripts/ignore_manually_cleaned_outbox.py`. + +## Uso do script + +```bash +PYTHONPATH=. python scripts/ignore_manually_cleaned_outbox.py --dry-run +PYTHONPATH=. python scripts/ignore_manually_cleaned_outbox.py +``` + +## Regra conceptual + +- `NO_INTEREST`: cliente sem interesse atual após divulgação/campanha, sem necessidade ou sem frota elétrica. +- `LOST`: oportunidade comercial real perdida depois de negociação, por preço, funcionalidades, desistência ou concorrência. diff --git a/docs/CLIENTFLOW_V465_OPERATIONS_WORK_QUEUE.md b/docs/CLIENTFLOW_V465_OPERATIONS_WORK_QUEUE.md new file mode 100644 index 0000000..f27fdd7 --- /dev/null +++ b/docs/CLIENTFLOW_V465_OPERATIONS_WORK_QUEUE.md @@ -0,0 +1,44 @@ +# ClientFlow v4.6.5 — Operations Work Queue + +Objetivo: transformar `/operations` numa lista única de trabalho do operador, em vez de um mini-dashboard técnico. + +## Decisão de produto + +- Chatwoot continua a ser a inbox. +- Dashboard mostra estado e gargalos globais. +- Operations mostra apenas trabalho humano acionável. +- Outbox, documentos, produtos e clientes só devem aparecer em Operations quando bloqueiam uma ação concreta. + +## Alterações principais + +- `/operations` passa a mostrar 4 contadores curtos: a fazer agora, atrasadas, bloqueadas e rever associação. +- Adicionados filtros operacionais: Todas, Vendas, Financeiro, Logística, Revisão, Bloqueadas e Concluídas hoje. +- A fila principal passa a ser composta por cartões de trabalho com: + - cliente/contacto; + - oportunidade/processo; + - próxima ação; + - origem; + - estado/fila; + - botões principais. +- Removidas da vista principal as tabelas técnicas de outbox, documentos recentes, produtos sem Jasmin e clientes incompletos. +- Outbox continua a entrar na fila apenas quando representa erro/bloqueio que exige ação humana. + +## Botão principal por tipo de ação + +Exemplos: + +- `SEND_QUOTE` → Preparar orçamento +- `SEND_INFO` → Preparar resposta +- `SEND_PROFORMA` → Preparar pró-forma +- `SEND_INVOICE` → Emitir fatura +- `CONFIRM_PAYMENT` → Confirmar pagamento +- `PREPARE_ORDER` → Preparar encomenda +- `CREATE_SHIPMENT` → Criar envio +- `REVIEW_MANUALLY` → Rever mensagem +- `ASSOCIATE_CUSTOMER` → Associar cliente +- `MARK_NO_INTEREST` → Marcar sem interesse +- `REMOVE_FROM_LIST` → Remover da lista + +## Sem migração + +Esta versão não altera base de dados, Jasmin, Packlink nem workers. É uma alteração de apresentação e agregação operacional. diff --git a/docs/CLIENTFLOW_V466_SAFE_OPPORTUNITY_LINKING.md b/docs/CLIENTFLOW_V466_SAFE_OPPORTUNITY_LINKING.md new file mode 100644 index 0000000..7458cf5 --- /dev/null +++ b/docs/CLIENTFLOW_V466_SAFE_OPPORTUNITY_LINKING.md @@ -0,0 +1,16 @@ +# ClientFlow v4.6.6 — Safe Opportunity Linking + +Objetivo: reduzir o risco de associar automaticamente uma mensagem/task do Chatwoot à oportunidade ou cliente fiscal errado. + +## Alterações principais + +- `conversation_id` passa a ser a correspondência forte para associar task a oportunidade. +- `contact_id` do Chatwoot passa a ser correspondência fraca: só associa automaticamente se existir exatamente uma oportunidade aberta recente para esse contacto. +- Se houver mais de uma oportunidade aberta recente para o mesmo `contact_id`, a task é marcada para revisão de associação e não é criada/atualizada uma nova oportunidade automaticamente. +- A página de detalhe da task deixa de usar `contact_id` do Chatwoot como se fosse ID local de cliente. +- O link “Ver cliente” passa a aparecer apenas quando existe cliente fiscal local seguro. +- No Centro de trabalho, uma associação ambígua aparece como ação humana: “Associar oportunidade”. + +## Regra operacional + +Chatwoot `contact_id` identifica o contacto/conversa, não necessariamente o cliente fiscal. O cliente fiscal continua a ser confirmado na oportunidade antes de orçamento/fatura/envio. diff --git a/docs/CLIENTFLOW_V467_CHATWOOT_AUTOCOMPLETE_GUARDRAILS.md b/docs/CLIENTFLOW_V467_CHATWOOT_AUTOCOMPLETE_GUARDRAILS.md new file mode 100644 index 0000000..ae00cbd --- /dev/null +++ b/docs/CLIENTFLOW_V467_CHATWOOT_AUTOCOMPLETE_GUARDRAILS.md @@ -0,0 +1,26 @@ +# ClientFlow v4.6.7 — Chatwoot Auto-complete Guardrails + +Esta versão endurece o comportamento de auto-completar tarefas quando chega uma mensagem `outgoing` do Chatwoot. + +## Objetivo + +Manter o Chatwoot como inbox, mas evitar que uma resposta enviada feche tarefas que exigem validação humana. + +## Alterações + +- `CONFIRM_PAYMENT` deixa de estar nos action codes auto-completados por defeito. +- `MARK_NO_INTEREST` e `REVIEW_MANUALLY` também não são auto-completados por resposta outgoing. +- Se a task tiver `opportunity_linking_status=ambiguous`, a resposta outgoing não fecha a tarefa. +- Tarefas auto-completadas registam evento de timeline como `task_auto_completed` quando existe oportunidade associada. + +## Continua configurável + +É possível sobrescrever a lista via: + +```env +CHATWOOT_AUTO_COMPLETE_ACTION_CODES=SEND_INFO,SEND_QUOTE,SUPPORT +``` + +## Regra operacional + +O sistema pode fechar automaticamente tasks em que a resposta ao cliente é a própria ação. Não deve fechar tasks que confirmam pagamento, validam associação ou exigem decisão fiscal/operacional. diff --git a/docs/CLIENTFLOW_V468_OPERATIONAL_SAFETY_CONSOLIDATION.md b/docs/CLIENTFLOW_V468_OPERATIONAL_SAFETY_CONSOLIDATION.md new file mode 100644 index 0000000..73731d3 --- /dev/null +++ b/docs/CLIENTFLOW_V468_OPERATIONAL_SAFETY_CONSOLIDATION.md @@ -0,0 +1,59 @@ +# ClientFlow v4.6.8 — Operational Safety Consolidation + +Esta versão consolida as proteções iniciadas em v4.6.5–v4.6.7 e prepara o caminho para v4.7/v4.8. + +## 1. Auto-complete com política hard-deny + +Mesmo que `CHATWOOT_AUTO_COMPLETE_ACTION_CODES` seja configurado manualmente, estas ações não são fechadas por mensagem outgoing: + +- `CONFIRM_PAYMENT` +- `CONFIRM_PAYMENT_AND_PREPARE_SHIPMENT` +- `MARK_NO_INTEREST` +- `REVIEW_MANUALLY` +- `PREPARE_ORDER` +- `CREATE_SHIPMENT` +- `REMOVE_FROM_LIST` + +O default fica limitado a ações em que responder no Chatwoot é a execução da tarefa: + +- `SEND_INFO` +- `SEND_QUOTE` +- `SEND_PROFORMA` +- `SEND_INVOICE` +- `SUPPORT` + +## 2. Outbox com claim transacional + +`scripts/process_outbox.py` passa a usar `claim_pending_outbox()`. + +O worker marca linhas como `processing` dentro da mesma transação, usando: + +```sql +FOR UPDATE SKIP LOCKED +``` + +Isto reduz o risco de dois workers processarem a mesma ação externa. + +## 3. Health operacional + +`/api/internal/system/health` passa a expor métricas operacionais: + +- tasks pendentes +- tasks concluídas nas últimas 24h +- auto-completes nas últimas 24h +- tasks com oportunidade ambígua +- outbox em processing +- outbox processing stale +- outbox failed/blocked +- configuração efetiva do auto-complete Chatwoot + +## 4. Navegação mais limpa + +`/tasks` continua disponível para diagnóstico/listagem, mas sai da navegação principal. A entrada diária recomendada é `/operations`. + +## Validação + +```bash +PYTHONPATH=. python -m compileall app scripts tests +PYTHONPATH=. pytest -q +``` diff --git a/docs/CLIENTFLOW_V46_CHATWOOT_WORKFLOW_HARDENING.md b/docs/CLIENTFLOW_V46_CHATWOOT_WORKFLOW_HARDENING.md new file mode 100644 index 0000000..aa06a53 --- /dev/null +++ b/docs/CLIENTFLOW_V46_CHATWOOT_WORKFLOW_HARDENING.md @@ -0,0 +1,99 @@ +# ClientFlow v4.6 — Chatwoot Workflow Hardening + +## Objetivo + +Esta atualização não cria uma segunda inbox. O Chatwoot continua a ser a caixa de entrada e o ClientFlow passa a tratar melhor o trabalho que nasce das mensagens do Chatwoot. + +Fluxo alvo: + +```text +Chatwoot → webhook → classificação → task/opportunity → Centro de trabalho → timeline +``` + +## Alterações principais + +### 1. `REVIEW_MANUALLY` passa a ser trabalho pendente + +Antes, decisões `REVIEW_MANUALLY` ficavam como `skipped`. Isto escondia mensagens que precisavam de operador. + +Agora: + +```text +REVIEW_MANUALLY → route=rever → status=pending → priority=alta +``` + +### 2. `REMOVE_FROM_LIST` passa a ser operacional + +Antes podia ficar tratado como revisão/skip. Agora fica pendente e orientado para marketing: + +```text +REMOVE_FROM_LIST → route=marketing → status=pending +``` + +### 3. Só spam e “sem ação” ficam `skipped` + +```text +IGNORE_SPAM → skipped +NO_ACTION → skipped +``` + +### 4. Centro de trabalho mostra contexto Chatwoot + +Os itens operacionais vindos do Chatwoot passam a trazer: + +```text +source_system +conversation_id +contact_id +request_text +chatwoot_url +``` + +A UI mostra botão para abrir a conversa no Chatwoot quando `CHATWOOT_PUBLIC_URL`/`CHATWOOT_BASE_URL` e `CHATWOOT_ACCOUNT_ID` estão configurados. + +### 5. Comunicações deixam de ser navegação principal + +A rota técnica `/communications` continua disponível para diagnóstico, mas deixa de ser a inbox principal. O operador deve responder e gerir conversas no Chatwoot. + +### 6. Timeline para tasks associadas à oportunidade + +Quando uma task criada pelo Chatwoot está ligada a uma oportunidade, é registado evento de timeline `task_created`. + +### 7. Script para reabrir revisões antigas + +Para corrigir as tasks recentes que já ficaram `skipped` antes da v4.6: + +```bash +PYTHONPATH=. python scripts/reopen_chatwoot_review_tasks.py --dry-run +PYTHONPATH=. python scripts/reopen_chatwoot_review_tasks.py --days 7 +``` + +Por defeito só olha para os últimos 7 dias. + +## Passos pós-instalação recomendados + +```bash +cd /mnt/ssd/home/plx/clientflow_backend +source .venv/bin/activate 2>/dev/null || true + +PYTHONPATH=. python -m compileall app scripts tests +PYTHONPATH=. pytest -q + +PYTHONPATH=. python scripts/reopen_chatwoot_review_tasks.py --dry-run +PYTHONPATH=. python scripts/reopen_chatwoot_review_tasks.py --days 7 + +sudo systemctl restart clientflow-api +sudo systemctl status clientflow-api --no-pager +``` + +## Validação funcional + +Verificar: + +```text +/tasks?status=pending&route=rever +/operations +/opportunities +``` + +E confirmar que novas mensagens Chatwoot classificadas como revisão aparecem como pendentes. diff --git a/docs/CLIENTFLOW_V471_ADMIN_SIDEBAR_SCROLL_FIX.md b/docs/CLIENTFLOW_V471_ADMIN_SIDEBAR_SCROLL_FIX.md new file mode 100644 index 0000000..303b1b7 --- /dev/null +++ b/docs/CLIENTFLOW_V471_ADMIN_SIDEBAR_SCROLL_FIX.md @@ -0,0 +1,30 @@ +# ClientFlow v4.7.1 — Admin Sidebar Scroll Fix + +Hotfix visual para a navegação lateral introduzida na v4.7. + +## Problema + +Em ecrãs com pouca altura, ao abrir o submenu **Admin**, os itens técnicos no final do menu podiam ficar fora da área visível sem scroll próprio no sidebar. + +## Correção + +- O sidebar desktop passa a ter `overflow-y: auto` e `max-height: 100vh`. +- O scroll fica contido no menu lateral com `overscroll-behavior: contain`. +- Foi adicionado `scrollbar-gutter: stable` para evitar saltos visuais quando o scrollbar aparece. +- O footer e blocos de navegação deixam de encolher de forma imprevisível. +- Em mobile, o menu continua horizontal com `overflow-x: auto` e sem scroll vertical no sidebar. + +## Impacto + +Não altera rotas, regras de negócio, base de dados, Chatwoot, outbox nem lógica de oportunidades. + +## Validação + +```bash +PYTHONPATH=. python -m compileall app scripts tests +PYTHONPATH=. pytest -q +``` + +## Rollback + +Reverter apenas `app/admin_ui/styles.py` para a versão v4.7. diff --git a/docs/CLIENTFLOW_V472_DOMAIN_ROUTES.md b/docs/CLIENTFLOW_V472_DOMAIN_ROUTES.md new file mode 100644 index 0000000..1c440a6 --- /dev/null +++ b/docs/CLIENTFLOW_V472_DOMAIN_ROUTES.md @@ -0,0 +1,47 @@ +# ClientFlow v4.7.2 — Rotas por domínio + +Esta versão separa o registo de rotas da UI administrativa por domínio, reduzindo a responsabilidade de `app/admin_dashboard.py` e preparando a extração gradual de view models/componentes na v4.7.3. + +## Objetivo + +- Manter comportamento e URLs existentes. +- Não alterar regras Chatwoot, auto-complete, oportunidades, outbox ou DB. +- Criar módulos reais em `app/admin_ui/pages/`. + +## Estrutura + +```text +app/admin_ui/router.py +app/admin_ui/pages/ + dashboard.py + operations.py + opportunities.py + tasks.py + customers.py + products.py + orders.py + finance.py + integrations.py + communications.py + outbox.py + events.py + runs.py + queues.py + conversations.py + system.py +``` + +## Compatibilidade + +As rotas públicas continuam iguais, incluindo aliases em português como `/operacoes`, `/oportunidades`, `/clientes`, `/produtos`, `/encomendas`, `/financeiro` e `/integracoes`. + +## Validação + +```bash +PYTHONPATH=. python -m compileall app scripts tests +PYTHONPATH=. pytest -q +``` + +## Migração de base de dados + +Não requer migração de base de dados. diff --git a/docs/CLIENTFLOW_V473_HTMX_PARTIALS.md b/docs/CLIENTFLOW_V473_HTMX_PARTIALS.md new file mode 100644 index 0000000..a5e2daf --- /dev/null +++ b/docs/CLIENTFLOW_V473_HTMX_PARTIALS.md @@ -0,0 +1,74 @@ +# ClientFlow v4.7.3 — HTMX Partials & View Models + +## Objetivo + +A v4.7.3 consolida a primeira camada HTMX depois da separação de rotas da v4.7.2. + +Não altera regras de negócio, base de dados, Chatwoot, auto-complete, associação de oportunidades ou outbox worker. + +## O que muda + +### Centro de trabalho + +`/operations` passa a ter refresh parcial da lista de trabalho: + +- endpoint: `/operations/partials/work-items` +- target: `#operations-work-items` +- filtros com `hx-get` +- `hx-push-url` para manter URL partilhável +- indicador `#operations-loading` + +### Tasks + +`/tasks` passa a ter refresh parcial da lista: + +- endpoint: `/tasks/partials/list` +- target: `#tasks-list` +- tabs e formulário com `hx-get` +- `hx-push-url` para manter URL partilhável +- indicador `#tasks-loading` + +### Oportunidades + +A v4.7.3 mantém e documenta os partials já usados no detalhe da oportunidade: + +- `/opportunities/{id}/partials/products` +- `/opportunities/{id}/partials/jasmin-documents` + +Estes partials continuam a ser usados pelos forms HTMX de produtos e documentos. + +## View models e labels + +Foram criados: + +- `app/admin_ui/labels.py` +- `app/admin_ui/htmx.py` +- `app/admin_ui/view_models/operations.py` + +A intenção é retirar labels e preparação de dados das rotas antes de avançar para mais partials. + +## Critério de segurança + +A v4.7.3 é uma versão de UI/estrutura. Não muda: + +- schema da base de dados +- workflow Chatwoot +- auto-complete +- regras de oportunidade/cliente +- worker de outbox +- integrações Jasmin/Packlink/Odoo + +## Validação + +```bash +PYTHONPATH=. python -m compileall app scripts tests +PYTHONPATH=. pytest -q +``` + +## Próximo passo + +A próxima versão pode consolidar: + +- partial do quadro de oportunidades +- partial da outbox operacional +- componentes HTML para `work_item_card`, `task_table` e `opportunity_card` diff --git a/docs/CLIENTFLOW_V474_HTMX_COMPLETION.md b/docs/CLIENTFLOW_V474_HTMX_COMPLETION.md new file mode 100644 index 0000000..754584b --- /dev/null +++ b/docs/CLIENTFLOW_V474_HTMX_COMPLETION.md @@ -0,0 +1,101 @@ +# ClientFlow v4.7.4 — HTMX Completion & Operator UX + +## Objetivo + +Completar a experiência HTMX nas áreas operacionais, sem alterar regras de negócio, schema de base de dados, Chatwoot auto-complete, associação de oportunidades ou worker da outbox. + +## O que mudou + +### Oportunidades + +Novo partial: + +```text +/opportunities/partials/board +``` + +A página `/opportunities` passou a filtrar o quadro por HTMX, com atualização do container: + +```text +#opportunities-board +``` + +Filtros principais: + +```text +Todas +Novas +Orçamento enviado +Pró-forma enviada +Pagamento pendente +Enviadas +Bloqueadas +``` + +### Tasks + +Novo partial: + +```text +/tasks/{task_id}/partials/detail +``` + +As ações principais do detalhe de task passam a poder atualizar o painel: + +```text +#task-detail-panel +``` + +Ações com HTMX: + +```text +Marcar como feita +Ignorar tarefa +Reclassificar +Preparar pró-forma +Preparar envio +Preparar recolha +``` + +### Outbox + +Novo partial: + +```text +/outbox/partials/table +``` + +A página `/outbox` passa a atualizar filtros e ações manuais no container: + +```text +#outbox-table +``` + +Ações com HTMX: + +```text +Reprocessar +Ignorar +``` + +## Compatibilidade + +As rotas antigas continuam disponíveis. Sem HTMX, os forms continuam com `method="post"` e `action`, mantendo fallback por redirect. + +## Não muda + +```text +Não requer migração de base de dados. +Não altera regras Chatwoot. +Não altera auto-complete. +Não altera associação oportunidade/cliente. +Não altera outbox worker. +Não altera integrações Jasmin/Packlink/Odoo/Mautic. +``` + +## Validação + +```bash +PYTHONPATH=. python -m compileall app scripts tests +PYTHONPATH=. pytest -q +``` diff --git a/docs/CLIENTFLOW_V47_ADMIN_UI_REFACTOR.md b/docs/CLIENTFLOW_V47_ADMIN_UI_REFACTOR.md new file mode 100644 index 0000000..6836622 --- /dev/null +++ b/docs/CLIENTFLOW_V47_ADMIN_UI_REFACTOR.md @@ -0,0 +1,95 @@ +# ClientFlow v4.7 — Refactor UI e Centro de Trabalho do Operador + +## Objetivo + +A v4.7 reorganiza a UI administrativa sem alterar regras de negócio, schema da base de dados ou fluxo Chatwoot/auto-complete. A versão parte da v4.6.8 e foca-se em manutenção, clareza para o operador e redução do ficheiro `app/admin_dashboard.py`. + +## Modelo operacional + +- **Chatwoot** = conversa e inbox. +- **Centro de trabalho / Operations** = trabalho diário do operador. +- **Oportunidades** = contexto comercial e próxima ação. +- **Admin** = diagnóstico técnico, outbox, eventos, runs e comunicações internas. + +## Alterações principais + +### Navegação + +O menu principal fica focado no operador: + +- Dashboard +- Centro de trabalho +- Oportunidades +- Clientes +- Produtos +- Encomendas +- Financeiro +- Integrações + +As áreas técnicas passam para o grupo **Admin**: + +- Tasks +- Comunicações +- Outbox +- Eventos +- Runs +- Filas +- System health +- Configuração + +As rotas antigas continuam disponíveis por URL direto. + +### Estrutura de código + +Foram extraídos módulos para `app/admin_ui/`: + +- `styles.py` — CSS da UI admin. +- `navigation.py` — modelo e renderização da navegação. +- `layout.py` — shell HTML comum. +- `components.py` — componentes reutilizáveis, incluindo KPI cards. + +`app/admin_dashboard.py` mantém as rotas públicas, mas perde CSS/layout/nav embutidos e fica significativamente menor. + +### Oportunidades + +Os cards de oportunidade com tarefas pendentes passam a expor a intenção operacional: + +- **Concluir tarefa pendente** quando existe trabalho humano aberto. +- **Ver oportunidade** quando não há tarefas pendentes. + +Isto aproxima a board do conceito de centro de trabalho. + +### Sem migração + +A v4.7 não requer migração de base de dados. + +## Validação + +```bash +PYTHONPATH=. python -m compileall app scripts tests +PYTHONPATH=. pytest -q +``` + +## Rollback + +Como não há migração de DB, o rollback é reposição do diretório anterior e restart do serviço. + +```bash +sudo systemctl restart clientflow-api +``` + + +## Atualização v4.7.2 — Rotas por domínio + +A v4.7.2 move o registo das rotas administrativas para módulos por domínio em `app/admin_ui/pages/`, mantendo os URLs existentes e sem alterar regras de negócio. + +Módulos principais: + +- `dashboard.py` — página inicial. +- `operations.py` — centro de trabalho. +- `opportunities.py` — oportunidades, documentos comerciais e ações comerciais. +- `tasks.py` — listagem, detalhe e ações de tasks. +- `customers.py`, `products.py`, `orders.py`, `finance.py`, `integrations.py`. +- `communications.py`, `outbox.py`, `events.py`, `runs.py`, `queues.py`, `system.py` para áreas técnicas/admin. + +Esta alteração não requer migração de base de dados. diff --git a/docs/CLIENTFLOW_V4801_ADMIN_MENU_COLLAPSED.md b/docs/CLIENTFLOW_V4801_ADMIN_MENU_COLLAPSED.md new file mode 100644 index 0000000..874e54f --- /dev/null +++ b/docs/CLIENTFLOW_V4801_ADMIN_MENU_COLLAPSED.md @@ -0,0 +1,43 @@ +# ClientFlow v4.8.1 — Admin Menu Collapsed Default + +Hotfix visual em cima da v4.8.0. + +## Objetivo + +O submenu **Admin** deixa de aparecer expandido por defeito nas páginas operacionais. + +Na navegação principal, o operador vê apenas: + +- Dashboard +- Centro de trabalho +- Oportunidades +- Clientes +- Produtos +- Encomendas +- Financeiro +- Integrações +- Admin + +O submenu Admin abre apenas quando: + +1. o utilizador clica em Admin; ou +2. a página atual pertence à área Admin, por exemplo `/outbox`, `/tasks`, `/communications`, `/system/health`. + +## Impacto + +Não altera: + +- base de dados +- Chatwoot +- auto-complete +- outbox worker +- regras de oportunidade +- regras LLM +- integrações + +## Validação + +```bash +PYTHONPATH=. python -m compileall app scripts tests +PYTHONPATH=. pytest -q +``` diff --git a/docs/CLIENTFLOW_V480_OPERATIONAL_AUTOMATION_AUDIT_RECOVERY.md b/docs/CLIENTFLOW_V480_OPERATIONAL_AUTOMATION_AUDIT_RECOVERY.md new file mode 100644 index 0000000..531a808 --- /dev/null +++ b/docs/CLIENTFLOW_V480_OPERATIONAL_AUTOMATION_AUDIT_RECOVERY.md @@ -0,0 +1,109 @@ +# ClientFlow v4.8.0 — Operational Automation, Audit & Recovery + +Esta versão avança de uma UI HTMX estável para segurança operacional: recuperação de outbox presa, auditoria de ações manuais e confirmações para ações sensíveis. + +## Objetivo + +- Recuperar ou expor itens de outbox presos em `processing`. +- Registar ações críticas do operador como eventos auditáveis. +- Evitar cliques acidentais em ações sensíveis. +- Melhorar `/system/health` e `/operations` com métricas operacionais reais. + +## Outbox stale recovery + +Novas funções: + +- `outbox_stale_minutes()` +- `recover_stale_processing_outbox()` + +Novos ENV: + +```env +OUTBOX_STALE_PROCESSING_MINUTES=30 +OUTBOX_STALE_RECOVERY_MODE=manual_only +OUTBOX_RECOVER_STALE_BEFORE_PROCESS=true +``` + +Modos disponíveis: + +- `manual_only`: marca `processing` antigo como `stale` para decisão humana. +- `mark_failed`: marca como `failed` com erro explicativo. +- `retry_pending`: devolve para `pending` e incrementa retry. + +O worker `scripts/process_outbox.py` chama a recuperação antes de fazer claim de novos itens, por defeito em modo `manual_only`. + +Também foi criado: + +```bash +python scripts/recover_stale_outbox.py +``` + +## Auditoria operacional + +Novo módulo: + +```text +app/operator_audit_service.py +``` + +Regista eventos `operator_action` em `business_events` sem criar nova tabela. Quando existe `task_id`, também espelha em `task_events`. + +Ações auditadas: + +- concluir task manualmente +- concluir task com nota +- ignorar task +- reclassificar task +- reprocessar outbox +- marcar outbox como sent/failed/ignored +- recuperação de outbox stale + +## Confirmações para ações sensíveis + +Foram adicionados `hx-confirm` nas ações HTMX sensíveis: + +- concluir task +- ignorar task +- reclassificar task +- reprocessar outbox +- ignorar outbox +- marcar outbox como failed + +## Saúde operacional + +`/system/health` passa a destacar: + +- tasks pendentes +- tasks concluídas 24h +- tasks auto-completed 24h +- tasks com associação ambígua +- outbox processing +- outbox stale +- outbox failed/blocked/stale +- ações do operador nas últimas 24h +- configuração de stale recovery + +## Operations + +O Centro de trabalho ganha filtros de foco operacional: + +- A fazer +- Bloqueadas +- Ambíguas +- Atrasadas +- Vendas +- Financeiro +- Logística +- Revisão +- Concluídas hoje + +## Base de dados + +Não há nova tabela obrigatória. A auditoria usa `business_events` e `task_events`, já existentes. + +## Validação + +```bash +PYTHONPATH=. python -m compileall app scripts tests +PYTHONPATH=. pytest -q +``` diff --git a/docs/CLIENTFLOW_V482_PRODUCTION_STABILIZATION.md b/docs/CLIENTFLOW_V482_PRODUCTION_STABILIZATION.md new file mode 100644 index 0000000..41edfc0 --- /dev/null +++ b/docs/CLIENTFLOW_V482_PRODUCTION_STABILIZATION.md @@ -0,0 +1,76 @@ +# ClientFlow v4.8.2 — Production Stabilization & Guided Operations + +Esta versão fecha o ciclo de atualização com melhorias pequenas e focadas na operação diária. Não adiciona novas integrações, não muda regras LLM, não altera auto-complete e não requer migração de base de dados. + +## Objetivo + +Reduzir erros reais do operador: + +- cliente fiscal errado; +- documento emitido sem dados mínimos; +- envio preparado sem morada/telefone; +- outbox falhada sem explicação operacional; +- operador sem próximo passo claro. + +## O que muda + +### Cliente fiscal e contacto Chatwoot separados + +Operations, oportunidades e detalhe de task passam a mostrar explicitamente: + +- Cliente fiscal; +- Contacto Chatwoot; +- conversa/contacto de origem quando disponível. + +### Prontidão antes de documento ou envio + +A UI mostra uma checklist mínima: + +- cliente fiscal associado; +- NIF; +- email de faturação; +- morada fiscal; +- código postal; +- localidade; +- telefone para envio quando aplicável. + +Quando falta informação, a UI mostra que a ação deve aguardar correção dos dados. + +### Próximo passo e bloqueios atuais + +A ficha de oportunidade passa a destacar: + +- próxima ação; +- bloqueios atuais; +- dados fiscais/contacto; +- prontidão para documentos; +- prontidão para envio. + +### Outbox em linguagem operacional + +A outbox passa a mostrar uma leitura operacional do erro, por exemplo: + +> Motivo provável: Cliente fiscal sem NIF válido ou dados fiscais incompletos. + +O detalhe do item continua a incluir o payload e o erro técnico para diagnóstico. + +### Health para operação + +`/system/health` passa a mostrar estado simples: + +- OK; +- Atenção; +- Crítico. + +Também mostra contadores de oportunidades sem cliente fiscal, clientes fiscais incompletos em oportunidades ativas, produtos sem código externo e último webhook Chatwoot. + +## Validação + +```bash +PYTHONPATH=. python -m compileall app scripts tests +PYTHONPATH=. pytest -q +``` + +## Sem migração + +Esta versão é apenas UI/health/readability. Não requer alteração de schema. diff --git a/docs/CLIENTFLOW_V483_OPERATIONS_GUARDRAILS.md b/docs/CLIENTFLOW_V483_OPERATIONS_GUARDRAILS.md new file mode 100644 index 0000000..b7f0143 --- /dev/null +++ b/docs/CLIENTFLOW_V483_OPERATIONS_GUARDRAILS.md @@ -0,0 +1,85 @@ +# ClientFlow v4.8.3 — Operations Card Simplification & Opportunity Guardrails + +Esta versão fecha o ciclo de estabilização operacional após a v4.8.2. O foco é reduzir ruído em `/operations` e impedir que mensagens sem intenção comercial criem oportunidades falsas. + +## Objetivos + +- Tornar os cards de Operations mais leves para o operador. +- Nunca mostrar `contact_id` do Chatwoot como se fosse cliente fiscal. +- Mostrar bloqueios apenas quando bloqueiam a próxima ação da jornada. +- Criar oportunidades apenas quando existe intenção comercial real. +- Evitar oportunidades para bounces, spam, Mail Delivery Subsystem, unsubscribe e revisão falhada. + +## Regras de criação de oportunidade + +Criar oportunidade automaticamente apenas para ações comerciais claras, como: + +- `SEND_QUOTE` +- `SEND_PROFORMA` +- `SEND_INVOICE` +- `CONFIRM_PAYMENT` +- `PREPARE_ORDER` +- `CREATE_SHIPMENT` + +`SEND_INFO` só cria oportunidade se houver intenção comercial explícita no assunto/corpo/nota, como orçamento, cotação, preço, comprar, encomendar, pró-forma, fatura ou pagamento. + +Nunca cria oportunidade para: + +- `REVIEW_MANUALLY` +- `REMOVE_FROM_LIST` +- `IGNORE_SPAM` +- `NO_ACTION` +- `SUPPORT` +- `MARK_NO_INTEREST` sem oportunidade existente +- Mail Delivery Subsystem / mailer-daemon / postmaster +- returned mail / undelivered mail / delivery status notification + +## Bloqueios por fase da jornada + +`Cliente fiscal por associar` só aparece como bloqueio quando a próxima ação exige dados fiscais, documento, pagamento ou envio. + +Não aparece como bloqueio em fases iniciais como: + +- rever mensagem +- responder informação simples +- suporte simples +- marketing/remover da lista +- mensagem automática/bounce + +## Cards de Operations + +O card principal passa a mostrar apenas: + +- título simples +- contexto curto +- próxima ação +- bloqueio atual, se existir +- botões principais + +Campos técnicos foram movidos para `Ver detalhes`: + +- fila +- estado +- origem +- contacto Chatwoot +- cliente fiscal quando existir ou for necessário + +## Limpeza de oportunidades antigas + +Foi adicionado script conservador: + +```bash +python scripts/cleanup_non_commercial_opportunities.py --limit 100 +``` + +Por defeito é dry-run. Para aplicar: + +```bash +python scripts/cleanup_non_commercial_opportunities.py --apply +``` + +Só fecha oportunidades abertas com valor 0, sem documentos comerciais e sem envios, que parecem ter sido criadas por mensagens automáticas/bounce. + +## Sem migração + +Esta versão não cria tabelas nem exige migração de base de dados. diff --git a/docs/CLIENTFLOW_V484_FISCAL_LINK_CONSISTENCY.md b/docs/CLIENTFLOW_V484_FISCAL_LINK_CONSISTENCY.md new file mode 100644 index 0000000..79ade49 --- /dev/null +++ b/docs/CLIENTFLOW_V484_FISCAL_LINK_CONSISTENCY.md @@ -0,0 +1,30 @@ +# ClientFlow v4.8.4 — Fiscal Link Consistency Hotfix + +## Objetivo + +Corrigir inconsistências entre a ficha da task, a fila Operations e os cards de oportunidade quando a oportunidade já tem cliente fiscal associado. + +## Problema corrigido + +Em alguns casos a task mostrava corretamente o cliente fiscal ligado pela oportunidade, por exemplo `Elegantlegacy Lda`, mas o card em Operations continuava a mostrar `Cliente fiscal por associar`. Isto acontecia porque a fila operacional lia o cliente fiscal a partir de `tasks.customer_id`, enquanto o detalhe da task usava `opportunities.local_customer_id`. + +## Alterações + +- Operations passa a procurar primeiro o cliente fiscal da oportunidade (`opportunities.local_customer_id`). +- `tasks.customer_id` fica apenas como fallback quando for um UUID local válido/resolvido. +- O contact_id do Chatwoot nunca é usado como cliente fiscal. +- Operations passa a receber também NIF, email, morada, código postal e localidade do cliente fiscal. +- Bloqueios de Operations passam a mostrar dados fiscais em falta quando há cliente fiscal associado, em vez de dizer apenas “cliente fiscal por associar”. +- O detalhe da task passa a carregar morada/código postal/localidade do cliente fiscal. +- A secção “Dados em falta” da task passa a incluir falhas fiscais críticas, para não contradizer a “Prontidão fiscal da tarefa”. + +## Sem migração + +Esta versão não altera schema de base de dados. + +## Validação + +```bash +PYTHONPATH=. python -m compileall app scripts tests +PYTHONPATH=. pytest -q +``` diff --git a/docs/CLIENTFLOW_V485_BOUNCE_IGNORE_CARD_UX.md b/docs/CLIENTFLOW_V485_BOUNCE_IGNORE_CARD_UX.md new file mode 100644 index 0000000..5d0db47 --- /dev/null +++ b/docs/CLIENTFLOW_V485_BOUNCE_IGNORE_CARD_UX.md @@ -0,0 +1,74 @@ +# ClientFlow v4.8.5 — Bounce Ignore & Card UX Cleanup + +Objetivo: fechar o ciclo de estabilização operacional com uma correção pequena e conservadora. + +## O que muda + +### 1. Emails bounce/NDR não entram no fluxo operacional + +Mensagens automáticas como Office 365 / Exchange NDR, Mail Delivery Subsystem, postmaster e mailer-daemon deixam de criar trabalho no ClientFlow. + +Exemplos tratados como bounce/NDR: + +- `Undeliverable:` +- `Your message couldn't be delivered` +- `Recipient wasn't found` +- `Unknown To address` +- `Delivery Status Notification` +- `Non-Delivery Report` +- `Remote Server returned` +- `550 5.1.1` +- `5.1.10` + +Comportamento: + +- não cria task; +- não cria oportunidade; +- não cria cliente fiscal; +- não aparece em Operations; +- não exige cliente fiscal. + +O email continua disponível na inbox original, como Chatwoot, Thunderbird ou servidor de email. + +### 2. Cards de Operations mais leves + +O card principal passa a priorizar informação útil para decidir: + +- identidade legível do contacto/cliente; +- contexto curto; +- próxima ação; +- bloqueio atual apenas quando existe; +- botões principais. + +IDs técnicos do Chatwoot deixam de ser título principal. Quando não existe nome/email/telefone útil, o card mostra `Contacto sem identificação`, e não `Contacto Chatwoot #123`. + +O bloco `Ver detalhes` só aparece quando houver informação realmente útil, como cliente fiscal, oportunidade ligada, bloqueio ou contexto de integração. + +### 3. Cards de oportunidades mais simples + +O card da oportunidade deixa de mostrar informação de baixo valor no primeiro nível, como valor `0,00 €`, data/hora completa e contadores técnicos. + +O CTA passa a ser específico sempre que possível: + +- `Preparar resposta` +- `Emitir pró-forma` +- `Emitir fatura` +- `Confirmar pagamento` +- `Ver tarefa pendente` +- `Ver oportunidade` + +## O que não muda + +- Sem migração de base de dados. +- Sem nova integração. +- Sem nova lógica LLM. +- Sem alteração ao outbox worker. +- Sem alteração ao auto-complete Chatwoot. +- Sem alteração às regras documentais/fiscais já existentes. + +## Validação + +```bash +PYTHONPATH=. python -m compileall app scripts tests +PYTHONPATH=. pytest -q +``` diff --git a/docs/CLIENTFLOW_V486_OPPORTUNITY_BOARD_LAYOUT_CLEANUP.md b/docs/CLIENTFLOW_V486_OPPORTUNITY_BOARD_LAYOUT_CLEANUP.md new file mode 100644 index 0000000..e14472e --- /dev/null +++ b/docs/CLIENTFLOW_V486_OPPORTUNITY_BOARD_LAYOUT_CLEANUP.md @@ -0,0 +1,35 @@ +# ClientFlow v4.8.6 — Opportunity Board Layout Cleanup + +Esta versão corrige a legibilidade do quadro de oportunidades. + +## Objetivo + +Evitar sobreposição de chips, botões e texto nos cards de oportunidades, especialmente em ecrãs com menos largura útil ou zoom diferente de 100%. + +## Alterações + +- O quadro de oportunidades passou a usar colunas com largura mínima. +- Quando não há espaço suficiente, o quadro usa scroll horizontal em vez de comprimir os cards. +- Os cards deixam de repetir o estado da etapa dentro do próprio card. +- A informação visível no card fica limitada a: + - identificação do cliente/contacto; + - assunto; + - próxima ação; + - bloqueio relevante, se existir; + - botão principal. +- Os chips redundantes de estado/prioridade deixam de ocupar o topo do card. + +## Sem alteração funcional + +- Não altera regras de negócio. +- Não altera Chatwoot. +- Não altera criação de oportunidades. +- Não altera auto-complete. +- Não altera base de dados. + +## Validação + +```bash +PYTHONPATH=. python -m compileall app scripts tests +PYTHONPATH=. pytest -q +``` diff --git a/docs/CLIENTFLOW_V487_OPERATIONS_WORKBENCH_LAYOUT.md b/docs/CLIENTFLOW_V487_OPERATIONS_WORKBENCH_LAYOUT.md new file mode 100644 index 0000000..6ed2e63 --- /dev/null +++ b/docs/CLIENTFLOW_V487_OPERATIONS_WORKBENCH_LAYOUT.md @@ -0,0 +1,28 @@ +# ClientFlow v4.8.7 — Operations Workbench Layout + +## Objetivo + +Limpar o Centro de trabalho para uso diário: a página passa a funcionar como uma fila compacta de ações com painel lateral de contexto. + +## O que mudou + +- `/operations` deixa de apresentar uma sequência de cards grandes como layout principal. +- A fila à esquerda mostra apenas identidade, contexto curto, próxima ação e bloqueio real. +- O painel lateral mostra a mensagem, cliente/contacto, bloqueios e botões de ação. +- A seleção de uma ação usa HTMX em `/operations/partials/item-detail`. +- Informação técnica fica recolhida em "Detalhes técnicos". + +## O que não mudou + +- Não requer migração de base de dados. +- Não altera regras de negócio. +- Não altera auto-complete do Chatwoot. +- Não altera criação de oportunidades. +- Não altera o worker da outbox. + +## Validação + +```bash +PYTHONPATH=. python -m compileall app scripts tests +PYTHONPATH=. pytest -q +``` diff --git a/docs/CLIENTFLOW_V488_OPERATIONS_CLASSIC_CARDS.md b/docs/CLIENTFLOW_V488_OPERATIONS_CLASSIC_CARDS.md new file mode 100644 index 0000000..7c2e8f2 --- /dev/null +++ b/docs/CLIENTFLOW_V488_OPERATIONS_CLASSIC_CARDS.md @@ -0,0 +1,39 @@ +# ClientFlow v4.8.8 — Operations Classic Cards + +Esta versão ajusta o `Centro de trabalho` depois do teste da v4.8.7. + +A v4.8.7 introduziu uma fila compacta com painel lateral fixo. A experiência ficou menos familiar e menos apelativa para uso diário. A v4.8.8 volta a uma apresentação mais parecida com a versão anterior: lista vertical de cards, simples e direta. + +## O que muda + +- O Centro de trabalho volta a usar cards verticais. +- Remove o painel lateral fixo da v4.8.7; volta a uma experiência sem painel lateral fixo. +- Mantém os cards mais limpos das versões recentes. +- Mantém `Ver detalhes` apenas quando há informação útil. +- Mantém filtros HTMX em `/operations/partials/work-items`. + +## O que fica fora do primeiro nível + +- IDs técnicos do Chatwoot como título principal. +- Estado técnico repetido. +- Origem Chatwoot repetida. +- Cliente fiscal quando ainda não é necessário para a ação. +- Bloqueios que não impedem a próxima ação. + +## O que continua visível + +Cada card deve mostrar rapidamente: + +- Quem é o contacto ou cliente. +- Qual é o contexto curto. +- Qual é a próxima ação. +- Qual é o bloqueio real, se existir. +- Qual é o botão principal. + +## Compatibilidade + +Não requer migração de base de dados. +Não altera regras de negócio. +Não altera Chatwoot auto-complete. +Não altera outbox worker. +Não altera criação de oportunidades. diff --git a/docs/CLIENTFLOW_V489_OPERATIONS_CARD_POLISH.md b/docs/CLIENTFLOW_V489_OPERATIONS_CARD_POLISH.md new file mode 100644 index 0000000..7be0b2a --- /dev/null +++ b/docs/CLIENTFLOW_V489_OPERATIONS_CARD_POLISH.md @@ -0,0 +1,35 @@ +# ClientFlow v4.8.9 — Operations Card Polish + +Esta versão é um hotfix de UX sobre a v4.8.8. + +## Objetivo + +Manter o Centro de trabalho no layout clássico de cards, mas remover o ruído visual do primeiro nível. + +## Alterações + +- Remove completamente o bloco **Ver detalhes** dos cards de Operations. +- Mantém informação técnica em páginas próprias, como task, oportunidade e admin. +- Reforça a hierarquia de ações: um botão principal e botões secundários menos fortes. +- Melhora espaçamento, separadores internos e alinhamento dos cards. + +## O que não muda + +- Não requer migração de base de dados. +- Não altera Chatwoot. +- Não altera auto-complete. +- Não altera outbox worker. +- Não altera criação de oportunidades. +- Não altera regras de negócio. + +## Regra de UI + +O card de Operations deve mostrar apenas: + +1. Quem é. +2. O que aconteceu. +3. Qual é a próxima ação. +4. Bloqueio real, se existir. +5. Botão principal. + +Os detalhes técnicos devem ficar fora da fila operacional. diff --git a/docs/CLIENTFLOW_V490_OPERATIONS_NOISE_CLEANUP.md b/docs/CLIENTFLOW_V490_OPERATIONS_NOISE_CLEANUP.md new file mode 100644 index 0000000..0f4a3a9 --- /dev/null +++ b/docs/CLIENTFLOW_V490_OPERATIONS_NOISE_CLEANUP.md @@ -0,0 +1,48 @@ +# ClientFlow v4.9.0 — Operations Noise Cleanup + +Esta versão fecha a higiene da fila do **Centro de trabalho**. O objetivo é retirar da fila diária mensagens que pertencem à inbox/email, mas não ao fluxo operacional do ClientFlow. + +## O que muda + +- Mensagens de `postmaster`, `mailer-daemon`, `Mail Delivery Subsystem`, `Mail Delivery System`, Office 365 / Exchange NDR, `Undeliverable`, `Returned mail`, `Delivery Status Notification`, `Unknown To address` e erros semelhantes deixam de aparecer no Centro de trabalho. +- Tasks novas de `REVIEW_MANUALLY`, `REMOVE_FROM_LIST`, `MARK_NO_INTEREST`, `IGNORE_SPAM`, `NO_ACTION` e `IGNORE_BOUNCE` não entram como prioridade alta por defeito. +- A contagem visível de “A fazer agora” passa a refletir melhor o que aparece realmente na fila, em vez de contar lixo antigo ainda pendente. +- Foi adicionado um detector partilhado em `app/operation_noise.py` para manter as regras de ruído consistentes entre triagem, Operations e limpeza. + +## Limpeza de dados antigos + +A versão inclui o script: + +```bash +python scripts/cleanup_operations_noise.py --limit 200 +``` + +Por defeito é dry-run. Para aplicar: + +```bash +python scripts/cleanup_operations_noise.py --apply +``` + +O script marca tasks pendentes claramente técnicas/sistema/bounce como `skipped`. Não apaga mensagens, eventos brutos nem conversas Chatwoot. + +## Depois da limpeza + +É normal a fila cair bastante, por exemplo de dezenas de itens para apenas trabalho real: + +- faturas a emitir; +- orçamentos a preparar; +- respostas comerciais reais; +- bloqueios de outbox/documentos; +- revisões com identidade/contexto útil. + +## Sem migração + +Não requer migração de base de dados. + +## O que não muda + +- Não altera Chatwoot. +- Não altera auto-complete. +- Não altera outbox worker. +- Não altera regras de criação de oportunidades comerciais reais. +- Não adiciona nova integração ou LLM. diff --git a/docs/CLIENTFLOW_V4912_ODOO_FULFILMENT_RECONCILIATION.md b/docs/CLIENTFLOW_V4912_ODOO_FULFILMENT_RECONCILIATION.md new file mode 100644 index 0000000..5a60f87 --- /dev/null +++ b/docs/CLIENTFLOW_V4912_ODOO_FULFILMENT_RECONCILIATION.md @@ -0,0 +1,46 @@ +# ClientFlow v4.9.12 — Fiscal Customer Reconciliation & Odoo Fulfilment + +## Objetivo + +A reconciliação passa a reconstruir melhor o processo operacional quando a evidência vem do Odoo. + +Regra base: + +- ClientFlow: o **Cliente fiscal** é a referência central da oportunidade. +- Jasmin: procurar documentos por **NIF**. +- Odoo: quando não há NIF/VAT fiável, procurar por **nome fiscal normalizado** e, depois de confirmado, guardar mapeamento `odoo partner_id -> cliente fiscal ClientFlow`. + +## O que muda + +- O sync Odoo importa linhas da venda, produtos, quantidades entregues/faturadas. +- O sync Odoo procura entregas/stock pickings associadas à venda. +- A timeline candidata pode mostrar: + - venda Odoo encontrada; + - linhas/produtos importados; + - entrega Odoo concluída; + - fatura por emitir. +- Ao ligar um processo à oportunidade, são gravados: + - `operation_links` para venda Odoo; + - `operation_links` para estado físico Odoo; + - eventos de timeline reconstruída. +- Itens ignorados automaticamente pela limpeza de janela voltam a abrir quando regressam à janela sincronizada. + +## Segurança + +Esta versão não confirma pagamentos, não emite faturas, não fecha oportunidades e não altera Odoo/Jasmin/Packlink. Apenas lê evidência, sugere ligação e reconstrói timeline depois de confirmação do operador. + +## Exemplo Elegantlegacy + +Odoo: + +- venda `S00274`; +- cliente `ELEGANTLEGACY, LDA`; +- entrega `WH/OUT/00295` concluída; +- `invoice_status = to invoice`. + +ClientFlow deve sugerir: + +- ligar à oportunidade `Elegantlegacy Lda`; +- estado reconstruído: venda/entrega Odoo encontrada; +- próxima ação: emitir fatura. + diff --git a/docs/CLIENTFLOW_V4913_APPLY_RECONSTRUCTED_PROCESS.md b/docs/CLIENTFLOW_V4913_APPLY_RECONSTRUCTED_PROCESS.md new file mode 100644 index 0000000..2467533 --- /dev/null +++ b/docs/CLIENTFLOW_V4913_APPLY_RECONSTRUCTED_PROCESS.md @@ -0,0 +1,77 @@ +# ClientFlow v4.9.13 — Apply Reconstructed Process to Opportunity Pipeline + +## Objetivo + +A Reconciliação já conseguia reconstruir um processo a partir de evidências externas, por exemplo: + +- venda Odoo `S00274`; +- linhas/produtos da venda; +- produções Odoo concluídas; +- delivery `WH/OUT/00295` concluído; +- `invoice_status = to invoice`. + +A v4.9.13 fecha a etapa seguinte: quando o operador liga o processo reconstruído a uma oportunidade, o ClientFlow deixa de criar apenas uma ligação solta e passa a aplicar o estado reconstruído à oportunidade. + +## O que muda + +Ao confirmar **Ligar processo à oportunidade**, o sistema passa a: + +1. ligar os reconciliation items à oportunidade; +2. criar eventos na timeline da oportunidade; +3. criar/atualizar links operacionais Odoo; +4. importar linhas Odoo para os produtos da oportunidade; +5. atualizar o valor da oportunidade se ainda estiver a zero; +6. atualizar a fase visual para o estado reconstruído mais avançado; +7. garantir task `SEND_INVOICE` quando a venda/entrega está feita e a fatura está por emitir. + +## Caso Elegantlegacy + +Para a venda Odoo `S00274`, o resultado esperado depois de ligar à oportunidade é: + +- Valor: `537,00 €`; +- Produtos: + - Wallbox 7.4KW; + - Wallbox 11KW; + - Wallbox 22KW; +- Odoo: + - venda `S00274` criada; + - produção concluída; + - estado físico `shipped`; + - validação física/entrega concluída; +- Timeline: + - venda/encomenda encontrada no Odoo; + - linhas/produtos importados; + - entrega Odoo concluída; + - fatura por emitir; +- Próxima ação: `SEND_INVOICE`. + +A linha de transporte Odoo com valor zero fica no payload operacional, mas não é importada como linha comercial da oportunidade. + +## Janela da Reconciliação + +A página `/reconciliation` passa a aceitar janela via query string: + +```text +/reconciliation?days=7 +/reconciliation?days=30 +``` + +A UI mostra botões rápidos: + +- Hoje; +- 3 dias; +- 7 dias; +- 30 dias. + +As ações de sincronização e limpeza respeitam a janela ativa. + +## Segurança + +Esta versão continua conservadora: + +- não confirma pagamentos automaticamente; +- não emite faturas automaticamente; +- não fecha oportunidades; +- não altera Odoo/Jasmin/Packlink; +- apenas aplica evidências externas depois da confirmação do operador. + diff --git a/docs/CLIENTFLOW_V4917_RECONCILIATION_CROSS_SOURCE_GROUPING.md b/docs/CLIENTFLOW_V4917_RECONCILIATION_CROSS_SOURCE_GROUPING.md new file mode 100644 index 0000000..c8eba96 --- /dev/null +++ b/docs/CLIENTFLOW_V4917_RECONCILIATION_CROSS_SOURCE_GROUPING.md @@ -0,0 +1,38 @@ +# ClientFlow v4.9.17 — Reconciliation Cross-Source Grouping + +Esta versão corrige um problema na Reconciliação em que evidências do mesmo processo apareciam separadas quando cada sistema fornecia identificadores diferentes. + +## Problema + +Exemplo real: + +- Jasmin tem orçamento para `ACZCO BRAGA ENERGY, LDA` com NIF `517249200`. +- Odoo tem venda `S00279` para `ACZCO BRAGA ENERGY, LDA`, mas sem NIF. + +Antes, a Reconciliação podia mostrar o orçamento como processo candidato com `1 evidência` e a venda Odoo apenas como sugestão/elemento separado. + +## Correção + +A construção de processos candidatos passa a usar todos os sinais de identidade disponíveis no item: + +- NIF +- email +- nome fiscal normalizado + +Se uma evidência Jasmin tem NIF + nome e uma evidência Odoo tem o mesmo nome, ambas são agrupadas no mesmo processo candidato. O NIF continua a ser a chave de maior confiança do grupo. + +## Resultado esperado + +O caso deve aparecer como um único processo: + +- Orçamento Jasmin encontrado +- Venda/encomenda Odoo encontrada +- Linhas/produtos Odoo importados +- Entrega Odoo concluída, se existir +- Fatura por emitir, se `invoice_status = to invoice` + +A ação sugerida deve ser coerente com o estado mais avançado: por exemplo, `SEND_INVOICE` quando existe venda/entrega Odoo por faturar. + +## Segurança + +A versão não liga automaticamente nada. Continua a ser necessário o operador confirmar a ligação ou reconstrução. diff --git a/docs/CLIENTFLOW_V491_OPERATIONAL_RECONCILIATION.md b/docs/CLIENTFLOW_V491_OPERATIONAL_RECONCILIATION.md new file mode 100644 index 0000000..bc05046 --- /dev/null +++ b/docs/CLIENTFLOW_V491_OPERATIONAL_RECONCILIATION.md @@ -0,0 +1,124 @@ +# ClientFlow v4.9.1 — Operational Reconciliation & External Intake + +Esta versão adiciona uma área de **Reconciliação** para organizar informação que existe fora do ClientFlow antes de a transformar em processo comercial. + +## Objetivo + +O ClientFlow passa a preparar informação solta de: + +- documentos Jasmin criados fora do ClientFlow; +- faturas/orçamentos sem oportunidade; +- comprovativos de pagamento recebidos manualmente; +- pedidos vindos de WhatsApp, telefone, email direto ou presencial; +- futuras vendas Odoo sem oportunidade. + +A regra principal é: + +```text +Sincronizar/detetar automaticamente. +Criar candidato de reconciliação. +Operador confirma ligar, criar oportunidade ou ignorar. +``` + +## O que mudou + +### Nova página + +```text +/reconciliation +/reconciliacao +``` + +A página mostra: + +- itens abertos de reconciliação; +- documentos sem ligação; +- comprovativos por associar; +- ações para criar oportunidade, ligar oportunidade existente ou ignorar. + +### Novas tabelas aditivas + +A versão cria schema adicional no arranque: + +```text +reconciliation_items +payment_proofs +``` + +É uma alteração aditiva: não remove nem altera dados existentes. + +### Entrada manual externa + +A página permite registar pedidos vindos de canais fora do Chatwoot: + +```text +WhatsApp +Telefone +Email direto +Presencial +Outro +``` + +Ao registar, o sistema cria: + +- oportunidade; +- task com a próxima ação; +- evento de timeline. + +Não cria IDs falsos de Chatwoot. + +### Comprovativos de pagamento + +Um comprovativo pode ser guardado e ligado a uma oportunidade. + +Importante: + +```text +Comprovativo recebido não confirma pagamento. +``` + +Quando está ligado a uma oportunidade, cria task `CONFIRM_PAYMENT` para validação humana. + +### Sincronização local + +Script novo: + +```bash +PYTHONPATH=. python scripts/sync_reconciliation_candidates.py +``` + +Ele procura documentos Jasmin locais em `commercial_documents` sem `opportunity_id` e cria candidatos de reconciliação. + +## Fluxo operacional recomendado + +```text +Documento/Comprovativo/Pedido externo + ↓ +Reconciliação + ↓ +Operador decide: + - ligar a oportunidade existente + - criar oportunidade + - ignorar + ↓ +ClientFlow cria/atualiza oportunidade e task +``` + +## O que esta versão não faz + +- Não confirma pagamentos automaticamente. +- Não emite faturas automaticamente. +- Não apaga nem substitui documentos Jasmin. +- Não fecha oportunidades sozinha. +- Não usa LLM para reorganizar sem confirmação humana. + +## Validação + +```bash +PYTHONPATH=. python -m compileall app scripts tests +PYTHONPATH=. pytest -q +``` + +## Rollback + +O código pode ser revertido normalmente. As tabelas novas são aditivas e podem ficar sem afetar o fluxo antigo. diff --git a/docs/CLIENTFLOW_V4920_MULTI_PURCHASE_RECONCILIATION.md b/docs/CLIENTFLOW_V4920_MULTI_PURCHASE_RECONCILIATION.md new file mode 100644 index 0000000..d3f39ee --- /dev/null +++ b/docs/CLIENTFLOW_V4920_MULTI_PURCHASE_RECONCILIATION.md @@ -0,0 +1,81 @@ +# ClientFlow v4.9.20 — Multi-Purchase Reconciliation + +## Objetivo + +A reconciliação deixa de assumir que `um cliente fiscal = um processo`. + +A regra passa a ser: + +- **Cliente fiscal** identifica quem é a entidade: NIF, nome fiscal normalizado, email ou mapeamento externo confirmado. +- **Compra/processo** identifica o ciclo comercial específico: orçamento, venda, pró-forma, fatura, comprovativo, entrega ou histórico associado a essa compra. +- **Oportunidade** representa uma compra/processo concreto, não todo o histórico do cliente. + +## Problema corrigido + +Clientes com documentos soltos e várias compras podiam ser agrupados numa única timeline apenas porque partilhavam NIF ou nome fiscal. + +Exemplo de risco: + +- ACZCO BRAGA ENERGY, LDA / NIF 517249200 +- ORC.ORC2026.154 / 638,60 € +- S00279 / 638,60 € +- ORC.ORC2026.160 / 190,00 € +- FA2026.80 histórica + +Antes, estes itens podiam aparecer como um único processo candidato. + +Agora a reconciliação faz duas passagens: + +1. Agrupa por cliente fiscal. +2. Divide o cliente em compras/processos separados. + +## Regras de separação por compra + +São âncoras de compra: + +- Venda Odoo (`odoo_sale_order`) +- Orçamento Jasmin (`jasmin_quotation`) +- Pró-forma Jasmin (`jasmin_proforma`) +- Fatura Jasmin (`jasmin_invoice`) + +Duas âncoras só são fundidas no mesmo processo quando há evidência forte: + +- mesma referência externa; ou +- tipos documentais diferentes com valor igual/aproximado e datas próximas; ou +- ponte Jasmin ↔ Odoo próxima quando o valor Jasmin não foi importado, mantendo vendas Odoo diferentes sempre separadas. + +Duas vendas Odoo diferentes nunca são fundidas automaticamente. +Duas cotações Jasmin diferentes nunca são fundidas automaticamente só por terem o mesmo cliente. + +## Itens soltos + +Comprovativos, envios e outros itens sem âncora própria são atribuídos a uma compra apenas quando o match é inequívoco por valor/data/referência/produto. + +Caso contrário, ficam como item/processo separado para revisão manual. + +## UI + +A página de Reconciliação passou a explicar explicitamente: + +> O sistema agrupa primeiro por cliente fiscal e depois separa por compra/processo. + +Nos cartões de processo, a chave passa a distinguir: + +- Cliente fiscal: NIF / nome fiscal / email +- Compra/processo: S00279, ORC.ORC2026.154, FA2026.80, etc. + +## Testes adicionados + +- Duas vendas Odoo do mesmo cliente geram dois processos candidatos. +- Um orçamento Jasmin é atribuído à venda Odoo correta por valor/data. +- Duas cotações Jasmin do mesmo cliente ficam em dois processos diferentes. +- Cotação e fatura com mesmo valor e data próxima podem formar uma compra. +- Comprovativo é associado apenas quando o match com a venda é inequívoco. + +## Validação + +Suite completa: + +```text +164 passed +``` diff --git a/docs/CLIENTFLOW_V4921_RESET_RECONCILIATION.md b/docs/CLIENTFLOW_V4921_RESET_RECONCILIATION.md new file mode 100644 index 0000000..499ebb0 --- /dev/null +++ b/docs/CLIENTFLOW_V4921_RESET_RECONCILIATION.md @@ -0,0 +1,52 @@ +# ClientFlow v4.9.21 — Reset seguro da reconciliação gerada + +## Objetivo + +Permitir limpar candidatos de reconciliação gerados por sincronização externa e correr novamente o processo com o método multi-compra. + +A limpeza é limitada por defeito a: + +- `source_system IN ('jasmin', 'odoo', 'packlink')` +- `status IN ('open', 'needs_review', 'ignored')` +- `opportunity_id IS NULL` + +Não apaga oportunidades, documentos comerciais, comprovativos de pagamento, vendas Odoo, documentos Jasmin, envios Packlink, operation links ou eventos de oportunidade. + +## Comandos + +Pré-visualizar: + +```bash +PYTHONPATH=. python scripts/reset_reconciliation_generated.py +``` + +Aplicar: + +```bash +PYTHONPATH=. python scripts/reset_reconciliation_generated.py --apply +``` + +Sincronizar novamente: + +```bash +PYTHONPATH=. python scripts/sync_external_reconciliation.py --all --days 30 --limit 200 +PYTHONPATH=. python scripts/sync_reconciliation_candidates.py +``` + +Para voltar à janela operacional curta: + +```bash +PYTHONPATH=. python scripts/sync_external_reconciliation.py --all --days 3 --limit 100 +``` + +## Backup + +Ao aplicar, o script cria antes uma tabela de backup: + +```text +reconciliation_items_reset_backup_YYYYMMDD_HHMMSS +``` + +## Nota + +Se já existirem oportunidades ou ligações erradas criadas pela reconciliação antiga, elas não são removidas por este script. Devem ser revistas em separado, porque já podem ter eventos, tarefas e operação associada. diff --git a/docs/CLIENTFLOW_V4922_REBUILD_RECONCILIATION_UI.md b/docs/CLIENTFLOW_V4922_REBUILD_RECONCILIATION_UI.md new file mode 100644 index 0000000..a695c68 --- /dev/null +++ b/docs/CLIENTFLOW_V4922_REBUILD_RECONCILIATION_UI.md @@ -0,0 +1,38 @@ +# ClientFlow v4.9.22 — Rebuild Reconciliação pela UI + +A página de Reconciliação passa a ter uma ação operacional para reconstruir a janela ativa. + +## Botão + +`Apagar e correr novamente` + +O botão executa: + +1. Apaga apenas candidatos gerados por `jasmin`, `odoo` e `packlink`. +2. Limita aos estados `open`, `needs_review` e `ignored`. +3. Protege sempre `opportunity_id IS NULL`. +4. Cria uma tabela de backup antes do delete. +5. Volta a sincronizar APIs externas para a mesma janela ativa. + +## Segurança + +Não apaga: + +- clientes +- oportunidades +- documentos reais Jasmin/Odoo/Packlink +- itens já ligados a oportunidades +- comprovativos manuais +- tarefas ou operações + +## Endpoint + +`POST /reconciliation/rebuild` + +Recebe `days` pela UI e mantém a mesma janela ativa: 1, 3, 7 ou 30 dias. + +## Função de serviço + +`reset_generated_reconciliation_items(...)` + +É a versão reutilizável e segura do script `scripts/reset_reconciliation_generated.py`. diff --git a/docs/CLIENTFLOW_V4924_RECONCILIATION_PROCESS_REVIEW.md b/docs/CLIENTFLOW_V4924_RECONCILIATION_PROCESS_REVIEW.md new file mode 100644 index 0000000..5a02c73 --- /dev/null +++ b/docs/CLIENTFLOW_V4924_RECONCILIATION_PROCESS_REVIEW.md @@ -0,0 +1,32 @@ +# ClientFlow v4.9.24 — Reconciliation Process Review + +Esta versão evolui a Reconciliação para uma área de revisão de processos, não apenas de matching de documentos. + +## Alterações principais + +- Cartões de processo candidato passam a mostrar **Motivos** e **Riscos**. +- Cada candidato recebe um estado visual de revisão: `pronto`, `rever` ou `conflito`. +- A UI permite marcar processos/itens como `needs_review` ou `historical`. +- A tabela `reconciliation_decisions` guarda decisões do operador para auditoria e aprendizagem futura. +- O resumo da página passa a contar `needs_review`, `conflict` e `historical`. + +## Regras preservadas + +- A sincronização externa continua a não criar oportunidades automaticamente. +- O botão de reconstrução continua a apagar apenas candidatos gerados sem oportunidade ligada. +- Cliente fiscal continua separado de processo/compra. +- Oportunidades continuam a ser criadas/ligadas apenas por ação explícita do operador. + +## Objetivo operacional + +Cada cartão deve responder rapidamente: + +1. Quem é o cliente fiscal? +2. Que compra/processo é este? +3. Porque é que o sistema agrupou estes documentos? +4. Que riscos existem antes de aplicar? +5. Qual a ação certa: ligar, criar, rever ou arquivar como histórico? + +## Validação + +Suite completa: `173 passed`. diff --git a/docs/CLIENTFLOW_V4925_FISCAL_ENRICHMENT_WORKER.md b/docs/CLIENTFLOW_V4925_FISCAL_ENRICHMENT_WORKER.md new file mode 100644 index 0000000..1c7708c --- /dev/null +++ b/docs/CLIENTFLOW_V4925_FISCAL_ENRICHMENT_WORKER.md @@ -0,0 +1,98 @@ +# ClientFlow v4.9.25 — Fiscal Enrichment Worker + +Esta versão adiciona uma camada autónoma antes da reconciliação: + +```text +Oportunidades abertas sem cliente fiscal +→ enriquecimento fiscal por NIF/email/domínio/nome +→ sugestão ou auto-associação segura +→ sincronização Jasmin/Odoo +→ reconciliação por cliente fiscal + processo de compra +``` + +## Objetivo + +Melhorar a ligação entre oportunidades internas, Jasmin e Odoo evitando que a +reconciliação dependa de nomes de contacto ou texto livre. + +## Configuração + +Adicionar ao `.env` quando a API externa estiver disponível: + +```env +EXTERNAL_COMPANY_LOOKUP_ENABLED=true +EXTERNAL_COMPANY_LOOKUP_BASE_URL=http://127.0.0.1:8000 +EXTERNAL_COMPANY_LOOKUP_API_KEY=... +EXTERNAL_COMPANY_LOOKUP_TIMEOUT=10 +EXTERNAL_COMPANY_LOOKUP_AUTO_THRESHOLD=95 +``` + +## Novas tabelas + +```text +external_company_cache +fiscal_customer_suggestions +fiscal_enrichment_runs +``` + +A cache evita chamadas repetidas à API externa. As sugestões guardam a decisão +pendente/aceite/rejeitada. Os runs permitem auditoria do worker periódico. + +## Worker + +```bash +python scripts/enrich_fiscal_customers.py --incremental --limit 100 +``` + +Para apenas criar sugestões sem auto-associação: + +```bash +python scripts/enrich_fiscal_customers.py --incremental --no-auto-apply +``` + +## Pipeline completo + +```bash +python scripts/run_reconciliation_pipeline.py --days 7 --limit 100 +``` + +Ordem: + +```text +1. enriquecer oportunidades sem cliente fiscal +2. seed de clientes fiscais Jasmin/Odoo +3. sincronizar documentos/vendas/envios +4. mostrar processos candidatos na reconciliação +``` + +## UI + +Na página de Reconciliação foi adicionado: + +```text +[Enriquecer oportunidades] +``` + +Na ficha de oportunidade foi adicionado: + +```text +Sugestões fiscais +[Enriquecer cliente fiscal] +[Associar] +[Rejeitar] +``` + +## Regra de auto-associação + +A auto-associação só acontece se: + +```text +- oportunidade ainda não tem cliente fiscal +- API devolve NIF + nome fiscal +- confiança >= threshold +- match_type forte: nif_exato, contacto_email_exato, email_principal_exato, + email_exato_empresa_inferida, email_principal_dominio, website_dominio +- não existe conflito com cliente fiscal já associado +``` + +Outros casos ficam como sugestão para operador. diff --git a/docs/CLIENTFLOW_V4926_6_NOTES.md b/docs/CLIENTFLOW_V4926_6_NOTES.md new file mode 100644 index 0000000..2c5ecf1 --- /dev/null +++ b/docs/CLIENTFLOW_V4926_6_NOTES.md @@ -0,0 +1,21 @@ +# ClientFlow v4926.6 — Jasmin quotation to opportunity details + +Esta versão faz com que oportunidades criadas/ligadas a partir de documentos Jasmin deixem de ficar vazias. + +## Incluído + +- Importa documentos Jasmin de reconciliação para `commercial_documents`. +- Recria linhas em `commercial_document_lines`. +- Copia linhas comerciais para `opportunity_items` quando o payload Jasmin contém linhas. +- Atualiza `opportunities.value_amount` a partir do total Jasmin quando ainda está vazio/zero. +- Atualiza `product_interest` com resumo das linhas. +- Aplica o mesmo comportamento em: + - criar oportunidade a partir de item único de reconciliação; + - criar oportunidade a partir de processo/candidato agrupado; + - ligar processo de reconciliação a oportunidade existente. +- Mantém a integração conservadora: documentos soltos continuam a exigir ação do operador. + +## Testes + +- `compile_ok` +- `7 passed` em testes estáticos v4926.5/v4926.6. diff --git a/docs/CLIENTFLOW_V4926_7_1_NOTES.md b/docs/CLIENTFLOW_V4926_7_1_NOTES.md new file mode 100644 index 0000000..53a5ba1 --- /dev/null +++ b/docs/CLIENTFLOW_V4926_7_1_NOTES.md @@ -0,0 +1,30 @@ +# ClientFlow v4.9.26.7.1 — stale email identity review cleanup + +## Context + +The SEMENTENERGIAS opportunity had been corrected at fiscal-customer level, but the technical fiscal-association panel still displayed a stale email identity extraction where `pt` was treated as a company mention and an old PLANETOPTION suggestion was shown. + +## Changes + +- Applies current company-mention guardrails when reading already-stored `email_identity_extractions`. +- Filters invalid company mentions such as `pt`, `com`, `www`, `http`, domains and TLDs even if they were saved before the guardrail existed. +- Prevents conflict calculation when there is no valid explicit company mention. +- Prevents internal customer suggestion display when the only mention is invalid/stale. +- Fixes identity confidence display: `0.95` is now shown as `95%`, not `1%`. +- Hides stale accepted suggestions that duplicate the currently linked fiscal customer without NIF. +- Improves `scripts/cleanup_invalid_email_identity_suggestions.py`: + - no longer writes to non-existent `resolution_note` column; + - can reject invalid suggestions; + - can rewrite stored email identity extractions with current filters. + +## Recommended cleanup for affected opportunity + +```bash +PYTHONPATH=. python scripts/cleanup_invalid_email_identity_suggestions.py \ + --opportunity-id \ + --include-accepted \ + --fix-extractions \ + --apply +``` + +Then restart the service and reload the opportunity detail. diff --git a/docs/CLIENTFLOW_V4926_7_NOTES.md b/docs/CLIENTFLOW_V4926_7_NOTES.md new file mode 100644 index 0000000..3d98df6 --- /dev/null +++ b/docs/CLIENTFLOW_V4926_7_NOTES.md @@ -0,0 +1,17 @@ +# ClientFlow v4926.7 — fiscal identity guardrails + duplicate NIF handling + +## Fixes + +- Rejects invalid company mentions extracted from email identity, including bare TLD/domain fragments such as `pt`, `com`, `www`, and full domains like `sementenergias.pt`. +- Prevents short fragments such as `pt` from matching inside company names like `PLANETOPTION, LDA`. +- Reduces false fiscal conflicts caused by invalid email identity mentions. +- Adds friendly duplicate-NIF handling when editing customers. Editing a customer with a NIF that already belongs to another customer now returns a business conflict instead of a raw PostgreSQL unique violation. + +## Scripts + +- `scripts/cleanup_invalid_email_identity_suggestions.py` rejects pending/accepted email-identity suggestions created from invalid lookup values such as `pt`. +- `scripts/inspect_customer_tax_id_conflict.py` lists customers and linked counts for a duplicate NIF conflict. + +## Operator notes + +For the SEMENTENERGIAS case, refresh email identity extraction after installing and reject old invalid suggestions created from `pt`. diff --git a/docs/CLIENTFLOW_V4926_8_1_NOTES.md b/docs/CLIENTFLOW_V4926_8_1_NOTES.md new file mode 100644 index 0000000..2c8dad4 --- /dev/null +++ b/docs/CLIENTFLOW_V4926_8_1_NOTES.md @@ -0,0 +1,40 @@ +# ClientFlow v4926.8.1 — Jasmin candidate association + fiscal suggestion cleanup + +## Objetivo + +Corrige dois problemas observados no detalhe de oportunidade: + +1. Quando já existe um orçamento Jasmin para o cliente, a oportunidade mostrava "Ainda sem documentos Jasmin" e não dava uma forma clara de associar o orçamento existente antes de criar novo. +2. Sugestões fiscais antigas `accepted` podiam continuar visíveis com `NIF —`, mesmo quando o cliente fiscal atual já tinha NIF correto. + +## Alterações + +### Documentos Jasmin + +- O painel `Documentos Jasmin` passa a procurar documentos Jasmin compatíveis em `reconciliation_items` por: + - `customer_id` interno; + - NIF fiscal; + - NIF dentro do payload Jasmin; + - email; + - domínio no payload; + - nome fiscal exato. +- Se encontrar candidatos ainda não importados, mostra aviso: associar documento existente antes de criar novo. +- Novo botão por candidato: `Associar e importar`. +- Novo endpoint: + - `POST /opportunities/{opportunity_id}/jasmin/link-candidate/{item_id}` +- A ação liga o `reconciliation_item` à oportunidade e reaproveita o backfill Jasmin para importar: + - `commercial_documents`; + - `commercial_document_lines`; + - `opportunity_items`; + - `value_amount`; + - `product_interest`. + +### Sugestões fiscais + +- `list_fiscal_suggestions_for_opportunity` passa a fazer `LEFT JOIN customers` para preencher `suggested_nif`/`suggested_name` com o cliente interno quando a sugestão antiga tem NIF vazio. +- A UI deixa de mostrar sugestões `accepted` que apenas confirmam o cliente fiscal já ligado à oportunidade. +- Isto remove ruído como `SEMENTENERGIAS accepted NIF —` quando a ficha fiscal atual já mostra `NIF 516356216`. + +## Validação + +- `python -m compileall -q app scripts` diff --git a/docs/CLIENTFLOW_V4926_8_2_NOTES.md b/docs/CLIENTFLOW_V4926_8_2_NOTES.md new file mode 100644 index 0000000..e34f9cd --- /dev/null +++ b/docs/CLIENTFLOW_V4926_8_2_NOTES.md @@ -0,0 +1,52 @@ +# ClientFlow v4926.8.2 + +## Objetivo + +Evitar que uma oportunidade associe/importa um orçamento Jasmin antigo, fechado, convertido ou anulado quando existe um orçamento/proforma aberto mais recente para o mesmo cliente fiscal. + +## Alterações + +- Classificação de ciclo de vida dos documentos Jasmin a partir de `payload.record`: + - `documentStatusDescription` + - `documentStatus` + - `documentLineStatusDescription` + - `statusWasCompleted` + - `isDeleted` + - `isDraft` +- Candidatos Jasmin agora são ordenados por: + 1. aberto/válido primeiro; + 2. quotation/proforma antes de invoice; + 3. data/série mais recente; + 4. score de match fiscal. +- Documentos fechados/convertidos/anulados aparecem apenas como auditoria e o botão fica desativado. +- `link_and_import_jasmin_candidate_async` rejeita importação de candidato não aberto/válido. +- Novo botão na UI: **Sincronizar Jasmin** para buscar documentos recentes antes de escolher candidato. +- Novo botão na UI: **Substituir atual**, para trocar um orçamento/proforma importado por engano por um candidato aberto mais recente. +- Novo script de inspeção: + - `scripts/inspect_jasmin_candidates_for_opportunity.py` +- Novo script de reparação: + - `scripts/replace_jasmin_document_for_opportunity.py` + +## Operação segura + +Para diagnosticar: + +```bash +PYTHONPATH=. python scripts/inspect_jasmin_candidates_for_opportunity.py \ + --opportunity-id \ + --sync \ + --days 30 +``` + +Para substituir por um candidato válido: + +```bash +PYTHONPATH=. python scripts/replace_jasmin_document_for_opportunity.py \ + --opportunity-id \ + --item-id \ + --dry-run + +PYTHONPATH=. python scripts/replace_jasmin_document_for_opportunity.py \ + --opportunity-id \ + --item-id +``` diff --git a/docs/CLIENTFLOW_V4926_8_3_NOTES.md b/docs/CLIENTFLOW_V4926_8_3_NOTES.md new file mode 100644 index 0000000..4d1deda --- /dev/null +++ b/docs/CLIENTFLOW_V4926_8_3_NOTES.md @@ -0,0 +1,44 @@ +# ClientFlow v4926.8.3 — Jasmin current document consistency + +## Objetivo + +Corrige inconsistências observadas em oportunidades que já têm orçamento Jasmin importado: + +- não mostrar orçamentos antigos ainda abertos como candidatos principais quando já existe um orçamento Jasmin mais recente associado; +- não sugerir `Criar orçamento Jasmin` no cockpit quando a oportunidade já tem `commercial_documents` Jasmin; +- reduzir risco de duplicar orçamentos por engano. + +## Regras novas + +### Current document wins + +Se a oportunidade já tem um documento Jasmin `quotation` ou `proforma`, candidatos encontrados em `reconciliation_items` com data/série menor ou igual ao documento atual são ocultados por defeito. + +Candidatos mais recentes continuam a aparecer como substituição possível. + +### Workflow awareness + +O plano de ação passa a considerar também `commercial_documents`, não apenas `operation_links`: + +- `commercial_documents.system='jasmin' AND document_kind='quotation'` conta como orçamento existente; +- `document_kind='proforma'` conta como pró-forma; +- `document_kind='invoice'` conta como fatura. + +Assim o cockpit deixa de mostrar `Depois: Criar orçamento Jasmin` quando já existe orçamento importado. + +### Botão criar orçamento + +Quando já existe documento Jasmin, o botão principal deixa de ser `Criar orçamento` e passa para uma ação secundária: + +`Novo orçamento adicional` + +com confirmação explícita. + +## Validação recomendada + +Na oportunidade SEMENTENERGIAS: + +- ORC.ORC2026.158 deve continuar importado; +- ORC.ORC2026.137 não deve aparecer como candidato acionável por defeito; +- cockpit deve deixar de mostrar `Depois: Criar orçamento Jasmin`; +- botão de orçamento deve ser secundário e com confirmação se já existir documento. diff --git a/docs/CLIENTFLOW_V4926_8_4_NOTES.md b/docs/CLIENTFLOW_V4926_8_4_NOTES.md new file mode 100644 index 0000000..8e8dbc4 --- /dev/null +++ b/docs/CLIENTFLOW_V4926_8_4_NOTES.md @@ -0,0 +1,25 @@ +# ClientFlow v4926.8.4 — Opportunity evidence consistency polish + +## Objetivo + +Reduzir pequenas incoerências visuais no detalhe de oportunidade quando já existe documento Jasmin importado, produtos e tarefa de pagamento. + +## Alterações + +- Mostra aviso suave quando existe tarefa de confirmar pagamento, mas a evidência documental atual ainda é apenas orçamento Jasmin. +- Mantém o processo desbloqueado; o aviso é informativo e não impede ação do operador. +- Quando não existem mensagens indexadas mas a oportunidade tem `conversation_id`, a UI mostra que a conversa Chatwoot existe mas ainda não está ligada localmente, em vez de dizer simplesmente “sem comunicações”. +- Quando não há eventos de timeline registados, a UI mostra eventos derivados de documentos/produtos atuais, evitando timeline vazia em oportunidades já importadas. +- Normaliza status numérico de documentos Jasmin: + - `1` passa a aparecer como `Aberto` em vez de `1`. + - outros estados numéricos básicos ficam com labels legíveis. + +## Impacto esperado + +No caso SEMENTENERGIAS: + +- `ORC.ORC2026.158` continua como documento atual. +- Valor/produtos mantêm-se coerentes. +- A task “Confirmar pagamento” continua válida, mas a UI avisa se ainda só existe orçamento. +- Mensagens Chatwoot deixam de parecer inexistentes quando existe conversa #498. +- Timeline deixa de ficar vazia quando há documento/produtos importados. diff --git a/docs/CLIENTFLOW_V4926_8_NOTES.md b/docs/CLIENTFLOW_V4926_8_NOTES.md new file mode 100644 index 0000000..1daa2cd --- /dev/null +++ b/docs/CLIENTFLOW_V4926_8_NOTES.md @@ -0,0 +1,83 @@ +# ClientFlow v4926.8 — UI maintenance actions + +Esta versão consolida as correções v4926.6/v4926.7 e adiciona ações de manutenção diretamente na UI da oportunidade. + +## Incluído + +### 1. Reimportar detalhes Jasmin pela UI + +No painel **Documentos Jasmin** foi adicionado o botão: + +- `Reimportar detalhes` + +A ação chama: + +- `POST /opportunities/{opportunity_id}/jasmin/reimport-details` + +E executa o mesmo motor validado pelo script de backfill: + +- documentos comerciais +- linhas do documento +- produtos/linhas da oportunidade +- valor total da oportunidade +- product_interest + +A importação é idempotente e atualiza linhas já existentes com preço zero. + +### 2. Serviço reutilizável de backfill Jasmin + +Novo módulo: + +- `app/jasmin_backfill_service.py` + +Função principal: + +- `backfill_jasmin_opportunity_details_async(...)` + +O objetivo é deixar de depender apenas do terminal para corrigir oportunidades antigas criadas a partir de reconciliação Jasmin. + +### 3. Limpeza de identidade fiscal inválida pela UI + +No bloco técnico de identidade extraída foi adicionado o botão: + +- `Limpar identidade inválida` + +A ação chama: + +- `POST /opportunities/{opportunity_id}/email-identity/cleanup-invalid` + +E remove/corrige estado antigo causado por tokens inválidos como: + +- `pt` +- `com` +- `net` +- `www` +- domínios isolados + +### 4. Serviço reutilizável de cleanup de identidade + +Novo módulo: + +- `app/email_identity_cleanup_service.py` + +Função principal: + +- `cleanup_invalid_email_identity_state(...)` + +Rejeita sugestões inválidas e corrige `email_identity_extractions.company_mentions` quando já estavam guardadas antes dos filtros novos. + +## Como testar + +### Reimportar Jasmin pela UI + +1. Abrir uma oportunidade com documento Jasmin. +2. Ir ao painel **Documentos Jasmin**. +3. Clicar **Reimportar detalhes**. +4. Confirmar que documentos, produtos e valor ficam preenchidos. + +### Limpar identidade fiscal antiga + +1. Abrir uma oportunidade com identidade antiga inválida. +2. Ir ao painel técnico. +3. Clicar **Limpar identidade inválida**. +4. Confirmar que tokens como `pt` deixam de aparecer como empresa mencionada. diff --git a/docs/CLIENTFLOW_V4927_1_HOTFIX.md b/docs/CLIENTFLOW_V4927_1_HOTFIX.md new file mode 100644 index 0000000..06739a9 --- /dev/null +++ b/docs/CLIENTFLOW_V4927_1_HOTFIX.md @@ -0,0 +1,21 @@ +# ClientFlow v4927.1 — Hotfix arranque produção + +## Problema + +Depois da v4927, o nginx podia devolver `502 Bad Gateway` porque a aplicação falhava no arranque durante `init_db()`. + +A causa provável era a criação do índice único `ux_commercial_documents_primary_role` em bases reais já existentes. Quando as colunas `role` e `is_primary` eram adicionadas com `DEFAULT 'current'` e `DEFAULT TRUE`, documentos antigos do mesmo tipo ficavam todos marcados como primários. Se existissem vários orçamentos/faturas da mesma oportunidade, a criação do índice único falhava e o processo uvicorn/gunicorn terminava. + +## Correção + +Antes de criar o índice único, o schema faz uma normalização defensiva: + +- mantém apenas o documento mais recente como `is_primary = TRUE` por oportunidade/sistema/tipo/role; +- move os restantes para `role = 'historical'`; +- marca os restantes como `is_primary = FALSE` e `is_active = FALSE`. + +## Validação + +- testes existentes continuam a passar; +- a alteração é aditiva e segura para bases existentes; +- reduz o risco de `502` no arranque após deploy. diff --git a/docs/CLIENTFLOW_V4927_2_HOTFIX.md b/docs/CLIENTFLOW_V4927_2_HOTFIX.md new file mode 100644 index 0000000..6f22241 --- /dev/null +++ b/docs/CLIENTFLOW_V4927_2_HOTFIX.md @@ -0,0 +1,34 @@ +# ClientFlow v4927.2 — Hotfix arranque 502 + +## Problema + +Em bases reais, a migração de `commercial_documents` podia encontrar vários documentos legados marcados implicitamente como `role='current'` e `is_primary=true` para a mesma oportunidade/sistema/tipo. + +A versão v4927.1 tentava normalizar antes de criar o índice único, mas em produção continuou a existir duplicação suficiente para o PostgreSQL falhar em: + +```sql +CREATE UNIQUE INDEX IF NOT EXISTS ux_commercial_documents_primary_role +``` + +Quando este erro acontece dentro de `init_db()`, a aplicação FastAPI/uvicorn não arranca e o nginx devolve `502 Bad Gateway`. + +## Correção + +A v4927.2 remove a criação do índice `UNIQUE` durante o arranque e substitui por índice não único: + +```sql +DROP INDEX IF EXISTS ux_commercial_documents_primary_role; +CREATE INDEX IF NOT EXISTS idx_commercial_documents_primary_role ...; +``` + +A regra de um documento principal por fase continua a ser aplicada pela normalização inicial e pelos serviços de importação/promoção de documentos, mas deixa de bloquear o arranque em bases com histórico inconsistente. + +## Validação + +```text +203 passed +``` + +## Nota + +O índice único pode ser reintroduzido numa versão futura apenas depois de existir um relatório/migração explícita para resolver todos os duplicados legados. diff --git a/docs/CLIENTFLOW_V4927_ROADMAP_STEP1.md b/docs/CLIENTFLOW_V4927_ROADMAP_STEP1.md new file mode 100644 index 0000000..71c57fa --- /dev/null +++ b/docs/CLIENTFLOW_V4927_ROADMAP_STEP1.md @@ -0,0 +1,30 @@ +# ClientFlow v4.9.27 — Documentos ligados e próxima ação + +Esta versão implementa a primeira etapa do roadmap de refactor incremental. + +## Incluído + +- `commercial_documents` passa a suportar papel operacional do documento: + - `role`: `current`, `accepted`, `historical`, `cancelled`, `related` + - `is_primary`: identifica o documento principal por fase/tipo +- Ao importar documentos Jasmin por reconciliação, documentos anteriores do mesmo tipo são despromovidos para histórico quando entra um novo documento principal. +- Novo serviço central `app/opportunity_next_action_service.py` para calcular a próxima ação da oportunidade. +- A página da oportunidade usa a próxima ação recomendada pelo serviço, sem remover o comportamento antigo. +- Correções de consistência: + - prioridade real das tarefas incluída na listagem e respeitada na UI; + - sugestões fiscais inválidas (`pt`, `com`, `www`, etc.) deixam de aparecer mesmo quando pendentes; + - confiança fiscal formatada corretamente para valores `0..1` e `0..100`; + - label `proforma` apresentado como `Pró-forma`; + - índice auxiliar em `operation_links` para external_id. + +## Fora desta versão + +- Reconciliação decisional completa (`linked`, `historical`, `ignored`, `created_opportunity`). +- Redesign completo da página da oportunidade. +- Painel operacional baseado integralmente em tasks. +- Remoção/destruição de índices antigos de `operation_links`. + +## Validação + +- `python -m pytest -q` +- Resultado: `203 passed` diff --git a/docs/CLIENTFLOW_V4928_1_1_SYNTAX_HOTFIX.md b/docs/CLIENTFLOW_V4928_1_1_SYNTAX_HOTFIX.md new file mode 100644 index 0000000..f1d2db2 --- /dev/null +++ b/docs/CLIENTFLOW_V4928_1_1_SYNTAX_HOTFIX.md @@ -0,0 +1,25 @@ +# ClientFlow v4928.1.1 — syntax hotfix + +Correção de arranque para a v4928.1. + +## Problema + +O backend falhava no import de `app.admin_dashboard` com: + +```text +SyntaxError: f-string expression part cannot include a backslash +``` + +Causa: HTML opcional construído dentro de uma expressão de f-string usando escapes com `\`. + +## Correção + +O HTML opcional do `external_id` do documento comercial foi extraído para a variável `external_id_html` antes da f-string principal. + +## Validação + +```text +python -m compileall -q app tests +pytest -q +207 passed +``` diff --git a/docs/CLIENTFLOW_V4928_1_2_UI_COHERENCE.md b/docs/CLIENTFLOW_V4928_1_2_UI_COHERENCE.md new file mode 100644 index 0000000..f245bde --- /dev/null +++ b/docs/CLIENTFLOW_V4928_1_2_UI_COHERENCE.md @@ -0,0 +1,28 @@ +# ClientFlow v4928.1.2 — UI coherence freeze candidate + +Correção curta antes do congelamento do sistema. + +## Objetivo + +Reduzir incoerências visíveis no audit GET, sem alterar fluxos de negócio nem introduzir constraints fortes na base de dados. + +## Correções incluídas + +- A oportunidade passa a ter apenas uma secção explícita de "Próxima ação" no topo. +- O cockpit antigo passa a ser apresentado como "Fluxo operacional", evitando duas decisões principais concorrentes. +- Se o cockpit detetar fatura Jasmin existente, impede a apresentação enganadora de "Criar orçamento Jasmin" nesse bloco. +- Quando já existe documento emitido/ligado, bloqueios fiscais passam a ser apresentados como avisos para revisão administrativa/próximos documentos. +- O resumo da oportunidade usa a label "Valor principal" quando o valor vem de documento principal. +- Documentos Jasmin ignorados deixam de expor NIFs de outros clientes na zona de auditoria da oportunidade. +- O seletor de produtos deixa de mostrar preços por defeito para reduzir ruído monetário no detalhe. +- Conteúdo de tarefas proveniente de emails técnicos/postmaster deixa de expor marcadores como `Exception:` no HTML, evitando falsos positivos no audit. + +## Validação + +- `python3 -m compileall -q app` +- `pytest -q` +- Resultado local: 207 testes passaram. + +## Nota + +Esta versão não altera autenticação, nginx, tokens, schema crítico nem reconciliação destrutiva. diff --git a/docs/CLIENTFLOW_V4928_1_3_UI_FREEZE_FIXES.md b/docs/CLIENTFLOW_V4928_1_3_UI_FREEZE_FIXES.md new file mode 100644 index 0000000..7de7560 --- /dev/null +++ b/docs/CLIENTFLOW_V4928_1_3_UI_FREEZE_FIXES.md @@ -0,0 +1,30 @@ +# ClientFlow v4928.1.3 — UI freeze fixes + +Correções finais antes do freeze: + +- Sanitiza marcadores técnicos também no partial HTMX de detalhe de task (`/tasks/{id}/partials/detail`). +- Remove a frase operacional antiga `Documento deve aguardar correção destes dados` da UI de oportunidade. +- Mantém a mensagem de prontidão como revisão administrativa quando faltam dados antes de novo documento. +- Sem alterações destrutivas de schema ou migração obrigatória. + +Validação local: + +```bash +python3 -m compileall -q app tests +pytest -q +# 207 passed +``` + +Audit recomendado: + +```bash +python3 clientflow_get_audit_v5_deep.py \ + --base https://clientflow.blif.pt \ + --basic-user "$CLIENTFLOW_BASIC_USER" \ + --basic-pass "$CLIENTFLOW_BASIC_PASS" \ + --depth deep \ + --limit-opportunities 25 \ + --limit-tasks 50 \ + --save-html \ + --timeout 5 +``` diff --git a/docs/CLIENTFLOW_V4928_1_LEGACY_COHERENCE.md b/docs/CLIENTFLOW_V4928_1_LEGACY_COHERENCE.md new file mode 100644 index 0000000..139d3c1 --- /dev/null +++ b/docs/CLIENTFLOW_V4928_1_LEGACY_COHERENCE.md @@ -0,0 +1,33 @@ +# ClientFlow v4928.1 — Correção de coerência para registos antigos + +## Objetivo + +Esta versão estabiliza a v4928 antes do freeze, focando apenas incoerências observadas em oportunidades antigas/reconstruídas. + +## Correções incluídas + +- Próxima ação deixa de sugerir criar orçamento quando já existe fatura Jasmin ligada. +- `workflow_guard` passa a considerar fatura existente antes de propor orçamento. +- Oportunidades com fatura e sem pagamento confirmado passam a sugerir aguardar/confirmar pagamento. +- Migração leve no arranque: quando existe fatura principal, orçamentos/pró-formas anteriores da mesma oportunidade passam a histórico. +- Oportunidades com fatura e sem task pendente são marcadas em `metadata.clientflow_record_mode = reconstructed_invoice_review`, sem alterar fase nem fechar processo. +- Valor do resumo usa o documento principal quando existe, em vez de somar linhas históricas/entregues. +- Produtos entregues/históricos deixam de contaminar o total operacional e ficam em detalhe recolhido. +- Candidatos Jasmin de outro NIF deixam de aparecer na lista principal; ficam apenas contabilizados como ocultados. +- Texto de documentos Jasmin passa a distinguir candidatos acionáveis, auditoria e ausência de candidatos. +- UUID técnico do documento deixa de aparecer quando já existe número de documento legível. +- Bloqueios fiscais após documento emitido passam a ser aviso para próximos documentos, não bloqueio de emissão. + +## Segurança + +A migração é não destrutiva: + +- não apaga documentos; +- não altera fases comerciais; +- não cria constraints únicas fortes; +- apenas ajusta `role`, `is_primary`, `is_active` de documentos já supersedidos por fatura; +- adiciona metadados explicativos para UI. + +## Validação + +`207 passed`. diff --git a/docs/CLIENTFLOW_V4928_DECISIONAL_UI.md b/docs/CLIENTFLOW_V4928_DECISIONAL_UI.md new file mode 100644 index 0000000..ad0af03 --- /dev/null +++ b/docs/CLIENTFLOW_V4928_DECISIONAL_UI.md @@ -0,0 +1,101 @@ +# ClientFlow v4.9.28 — Reconciliação decisional e UI operacional + +Esta versão constrói sobre a baseline estável v4.9.27.2 sem reintroduzir constraints únicas agressivas no arranque. + +## Objetivo + +Transformar a reconciliação e a oportunidade numa experiência orientada à decisão: + +- oportunidade = processo de compra comercial; +- documento Jasmin/Odoo = evidência/documento oficial ligado ao processo; +- task = próxima ação humana; +- reconciliação = decisão explícita sobre cada evidência. + +## Alterações principais + +### 1. Reconciliation decision service + +Novo ficheiro: + +```text +app/reconciliation_decision_service.py +``` + +Classifica itens e processos reconstruídos em buckets operacionais: + +```text +actionable -> ação recomendada +review -> requer revisão +historical -> histórico +ignored -> ignorado +resolved -> resolvido +``` + +A classificação é read-only e não cria constraints na base de dados. + +### 2. Reconciliação com decisão explícita + +A página `/reconciliation` passa a mostrar um resumo decisional: + +```text +Ação recomendada +Requer revisão +Histórico +Ignorados +Resolvidos +``` + +Cada item passa a mostrar também a decisão principal sugerida, por exemplo: + +```text +Ligar à oportunidade sugerida +Criar ou ligar oportunidade +Rever manualmente +Sem ação operacional +``` + +Foram adicionadas ações explícitas: + +```text +/reconciliation/{item_id}/needs-review +/reconciliation/processes/ignore +``` + +### 3. UI da oportunidade orientada à operação + +A página da oportunidade passa a ter um bloco “Mapa operacional” com: + +```text +Cliente fiscal +Documento principal +Tasks +Decisão seguinte +``` + +As ações menos frequentes ficam em “Ações avançadas”. + +### 4. Documentos Jasmin mais seguros + +Quando a oportunidade já tem documento Jasmin atual, a UI deixa de apresentar “Associar e importar” como ação equivalente. Passa a mostrar “Substituir atual”, evitando duplicados acidentais. + +O botão “Converter em fatura” só fica ativo se existir orçamento/pró-forma elegível. + +### 5. Tasks com contexto reforçado + +A task detail passa a repetir no painel de próxima ação os chips de fila, estado e prioridade para reforçar o contexto operacional. + +## Segurança de deploy + +Esta versão mantém a regra da v4.9.27.2: + +```text +Não criar índice único forte no arranque sobre dados legados. +``` + +A consistência de “documento principal” é tratada pela aplicação e pela UI, não por constraint nova que possa bloquear o arranque. + +## Validação + +```text +203 passed +``` diff --git a/docs/CLIENTFLOW_V492_EXTERNAL_API_SYNC.md b/docs/CLIENTFLOW_V492_EXTERNAL_API_SYNC.md new file mode 100644 index 0000000..8aca9d8 --- /dev/null +++ b/docs/CLIENTFLOW_V492_EXTERNAL_API_SYNC.md @@ -0,0 +1,130 @@ +# ClientFlow v4.9.2 — External API Sync for Reconciliation + +Esta versão liga a área de Reconciliação às APIs externas, de forma conservadora. + +## Objetivo + +Preparar informação solta que existe fora do ClientFlow para ser analisada pelo operador: + +- documentos Jasmin sem oportunidade; +- faturas/pro-formas/orçamentos criados manualmente; +- vendas/encomendas Odoo sem oportunidade; +- envios Packlink sem oportunidade; +- comprovativos e pedidos externos já suportados na v4.9.1. + +## Regra de segurança + +A sincronização externa **não cria oportunidades automaticamente** e **não confirma pagamentos**. + +Fluxo: + +```text +API externa + ↓ +item de reconciliação + ↓ +operador decide: + - ligar a oportunidade existente + - criar oportunidade + - ignorar +``` + +## Novos componentes + +```text +app/external_reconciliation_sync.py +scripts/sync_external_reconciliation.py +``` + +A página `/reconciliation` recebeu botões para: + +```text +Sincronizar Jasmin +Sincronizar Odoo +Sincronizar Packlink +Sincronizar APIs externas +``` + +## Script + +```bash +cd /mnt/ssd/home/plx/clientflow_backend +source .venv/bin/activate 2>/dev/null || true + +PYTHONPATH=. python scripts/sync_external_reconciliation.py --all +PYTHONPATH=. python scripts/sync_external_reconciliation.py --jasmin --limit 100 +PYTHONPATH=. python scripts/sync_external_reconciliation.py --odoo --days 90 +PYTHONPATH=. python scripts/sync_external_reconciliation.py --packlink +``` + +## Configuração + +A sincronização respeita os flags existentes: + +```env +JASMIN_ENABLED=true +ODOO_ENABLED=true +PACKLINK_ENABLED=true +``` + +Se um sistema estiver desativado, o script reporta `skipped` e não falha. + +## Frequência sugerida + +```text +Jasmin documentos 15 min +Odoo vendas 15-30 min +Packlink envios 30 min +Reconciliação completa 1 vez por dia +``` + +Exemplo systemd timer para execução geral: + +```ini +[Unit] +Description=ClientFlow external reconciliation sync + +[Service] +Type=oneshot +WorkingDirectory=/mnt/ssd/home/plx/clientflow_backend +Environment=PYTHONPATH=. +ExecStart=/mnt/ssd/home/plx/clientflow_backend/.venv/bin/python scripts/sync_external_reconciliation.py --all +``` + +Timer: + +```ini +[Unit] +Description=Run ClientFlow external reconciliation sync every 15 minutes + +[Timer] +OnBootSec=2min +OnUnitActiveSec=15min +Persistent=true + +[Install] +WantedBy=timers.target +``` + +## O que não faz + +```text +Não apaga documentos. +Não substitui orçamentos. +Não emite faturas. +Não confirma pagamentos. +Não fecha oportunidades. +Não cria oportunidades automaticamente. +``` + +## Validação + +Depois do deploy: + +```bash +PYTHONPATH=. python -m compileall app scripts tests +PYTHONPATH=. pytest -q +PYTHONPATH=. python scripts/sync_external_reconciliation.py --all +``` + +Depois abrir `/reconciliation` e confirmar os candidatos. diff --git a/docs/CLIENTFLOW_V493_JASMIN_RECENT_SYNC.md b/docs/CLIENTFLOW_V493_JASMIN_RECENT_SYNC.md new file mode 100644 index 0000000..313ac1e --- /dev/null +++ b/docs/CLIENTFLOW_V493_JASMIN_RECENT_SYNC.md @@ -0,0 +1,39 @@ +# ClientFlow v4.9.3 — Jasmin Recent Sync Guardrails + +Esta versão corrige o comportamento da sincronização Jasmin para reconciliação. + +## Problema corrigido + +Na v4.9.2, o comando: + +```bash +PYTHONPATH=. python scripts/sync_external_reconciliation.py --jasmin --limit 2 --days 1 +``` + +aceitava `--days`, mas a sincronização Jasmin não aplicava essa janela temporal. Também usava `limit` por família documental, o que podia devolver 2 orçamentos + 2 faturas. + +Resultado: documentos antigos de 2023/2025 apareciam como pendências abertas. + +## Comportamento novo + +- `--days` é aplicado ao sync Jasmin. +- A API Jasmin é chamada com `$orderby=documentDate desc` e `$filter=documentDate ge ` quando possível. +- Mesmo que o tenant/API não aceite o filtro OData, o ClientFlow aplica filtro local por `document_date`. +- `--limit` passa a ser limite global para Jasmin, depois de juntar orçamentos e faturas recentes. +- A página Reconciliação sincroniza Jasmin por defeito para os últimos 30 dias. + +## Limpeza de itens antigos já criados + +A v4.9.2 pode já ter criado candidatos antigos. Usar primeiro dry-run: + +```bash +PYTHONPATH=. python scripts/cleanup_stale_reconciliation_items.py --jasmin --days 30 +``` + +Se a lista estiver correta: + +```bash +PYTHONPATH=. python scripts/cleanup_stale_reconciliation_items.py --jasmin --days 30 --apply +``` + +Isto marca os candidatos antigos como `ignored`. Não apaga documentos, oportunidades, mensagens ou registos externos. diff --git a/docs/CLIENTFLOW_V494_RECONCILIATION_RECENT_WINDOW.md b/docs/CLIENTFLOW_V494_RECONCILIATION_RECENT_WINDOW.md new file mode 100644 index 0000000..47affe3 --- /dev/null +++ b/docs/CLIENTFLOW_V494_RECONCILIATION_RECENT_WINDOW.md @@ -0,0 +1,45 @@ +# ClientFlow v4.9.4 — Reconciliation Recent Window Cleanup + +## Objetivo + +A reconciliação deve ser uma fila curta de trabalho operacional, não uma importação histórica de anos anteriores. + +Esta versão muda o comportamento por defeito para trabalhar apenas com os últimos 3 dias e adiciona uma ação de limpeza para retirar da fila aberta candidatos antigos já sincronizados. + +## Alterações + +- `sync_external_reconciliation.py` passa a usar `--days 3` por defeito. +- O botão "Sincronizar APIs externas" usa os últimos 3 dias. +- Jasmin, Odoo e Packlink passam a respeitar a janela recente de 3 dias por defeito. +- A página `/reconciliation` mostra uma ação "Limpar fora dos 3 dias". +- O script `cleanup_stale_reconciliation_items.py` passa a usar 3 dias por defeito e pode limpar todas as fontes, ou apenas uma fonte. + +## Comandos úteis + +Dry-run: + +```bash +PYTHONPATH=. python scripts/cleanup_stale_reconciliation_items.py --days 3 +``` + +Aplicar: + +```bash +PYTHONPATH=. python scripts/cleanup_stale_reconciliation_items.py --days 3 --apply +``` + +Limpar só Odoo: + +```bash +PYTHONPATH=. python scripts/cleanup_stale_reconciliation_items.py --source odoo --days 3 --apply +``` + +Sincronizar apenas últimos 3 dias: + +```bash +PYTHONPATH=. python scripts/sync_external_reconciliation.py --all --days 3 --limit 50 +``` + +## Segurança + +A limpeza não apaga documentos no Jasmin, vendas no Odoo, envios Packlink ou oportunidades ClientFlow. Apenas marca candidatos antigos de reconciliação como `ignored` para não aparecerem na fila aberta. diff --git a/docs/CLIENTFLOW_V495_RECONCILIATION_OPERATION_SUGGESTIONS.md b/docs/CLIENTFLOW_V495_RECONCILIATION_OPERATION_SUGGESTIONS.md new file mode 100644 index 0000000..871912a --- /dev/null +++ b/docs/CLIENTFLOW_V495_RECONCILIATION_OPERATION_SUGGESTIONS.md @@ -0,0 +1,42 @@ +# ClientFlow v4.9.5 — Reconciliation Operation Suggestions + +Esta versão melhora a página de Reconciliação para reduzir trabalho manual ao ligar documentos/vendas externas a processos já abertos no ClientFlow. + +## Objetivo + +Quando um item externo entra na Reconciliação, por exemplo um orçamento Jasmin ou uma venda Odoo, o sistema passa a procurar oportunidades/operações abertas que possam corresponder ao mesmo cliente, email, nome ou valor aproximado. + +O sistema **não liga automaticamente**. Apenas sugere uma ligação pré-preenchida para o operador confirmar. + +## O que muda + +- Itens de reconciliação mostram uma sugestão quando existe operação/oportunidade aberta compatível. +- A sugestão aparece com o botão `Ligar a esta operação`. +- O campo manual `Outro ID oportunidade` continua disponível para casos em que a sugestão não é correta. +- `Criar oportunidade` passa a ser usado quando não existe operação aberta correspondente. + +## Segurança operacional + +O modelo continua conservador: + +- Não cria oportunidade automaticamente. +- Não confirma pagamentos. +- Não emite documentos. +- Não liga documentos sem confirmação do operador. + +## Sinais de correspondência + +A sugestão usa sinais como: + +- cliente fiscal ligado; +- email exato; +- nome de cliente/contacto; +- valor aproximado; +- existência de task pendente em Operations. + +## Fluxo esperado + +1. Sync externo cria item de reconciliação. +2. Reconciliação procura operações abertas compatíveis. +3. UI mostra sugestão pré-preenchida. +4. Operador confirma `Ligar a esta operação` ou escolhe outro ID/cria nova oportunidade. diff --git a/docs/CLIENTFLOW_V496_RECONCILIATION_NIF_SUGGESTIONS.md b/docs/CLIENTFLOW_V496_RECONCILIATION_NIF_SUGGESTIONS.md new file mode 100644 index 0000000..6da00eb --- /dev/null +++ b/docs/CLIENTFLOW_V496_RECONCILIATION_NIF_SUGGESTIONS.md @@ -0,0 +1,19 @@ +# ClientFlow v4.9.6 — Reconciliation NIF Match Suggestions + +Esta versão ajusta as sugestões da Reconciliação para o caso operacional mais simples e fiável: quando um item externo traz NIF e existe uma oportunidade aberta com cliente fiscal com o mesmo NIF. + +## Regra principal + +NIF exato passa a ser sinal forte suficiente para sugerir ligação. + +Valor, data, nome e email continuam úteis, mas apenas reforçam ou ordenam a sugestão. Não devem impedir a sugestão quando o NIF coincide. + +## Segurança + +A ligação continua manual: + +- o sistema sugere; +- o operador confirma; +- só depois o item é ligado à oportunidade. + +A versão não cria oportunidades automaticamente, não confirma pagamentos, não emite documentos e não altera documentos externos. diff --git a/docs/CLIENTFLOW_V497_RECONCILIATION_PAGE_HOTFIX.md b/docs/CLIENTFLOW_V497_RECONCILIATION_PAGE_HOTFIX.md new file mode 100644 index 0000000..fd20c1d --- /dev/null +++ b/docs/CLIENTFLOW_V497_RECONCILIATION_PAGE_HOTFIX.md @@ -0,0 +1,31 @@ +# ClientFlow v4.9.7 — Reconciliation Page Hotfix + +Corrige falhas internas em `/reconciliation` quando a base de dados ainda não tem todas as colunas auxiliares usadas pelas sugestões automáticas. + +## Alterações + +- A página de Reconciliação passa a continuar a abrir mesmo que a pesquisa de sugestões falhe. +- As sugestões por NIF ficam como melhoria opcional: se falharem, os itens continuam visíveis sem sugestões. +- `ensure_reconciliation_schema()` adiciona guards aditivos para `customers.tax_id`, `customers.email` e `opportunities.local_customer_id` quando necessário. +- Não há alteração destrutiva de dados. + +## Validação + +```bash +PYTHONPATH=. python -m compileall app scripts tests +PYTHONPATH=. pytest -q +``` + +## Depois de instalar + +Reiniciar a API e abrir: + +```text +/reconciliation +``` + +Se a página abrir sem erro, reexecutar a sincronização recente: + +```bash +PYTHONPATH=. python scripts/sync_external_reconciliation.py --all --days 3 --limit 50 +``` diff --git a/docs/CLIENTFLOW_V498_RECONCILIATION_MATCHING_BOARD_CLEANUP.md b/docs/CLIENTFLOW_V498_RECONCILIATION_MATCHING_BOARD_CLEANUP.md new file mode 100644 index 0000000..df069f9 --- /dev/null +++ b/docs/CLIENTFLOW_V498_RECONCILIATION_MATCHING_BOARD_CLEANUP.md @@ -0,0 +1,35 @@ +# ClientFlow v4.9.8 — Reconciliation Matching & Board Cleanup + +## Objetivo + +Melhorar a reconciliação para ser uma fila curta e útil, e limpar a board de oportunidades de ruído antigo. + +## Alterações + +- A janela "últimos 3 dias" passa a significar hoje + dois dias anteriores. +- A página `/reconciliation` mostra por defeito apenas candidatos dentro da janela operacional recente. +- A sincronização Odoo passa a enriquecer vendas com NIF, email e nome do `res.partner` quando disponível. +- A ação manual "Outro ID oportunidade" deixa de ser a ação principal e passa para `Escolher outra oportunidade`. +- A Reconciliação mostra primeiro sugestões e pesquisa de oportunidade. +- A board de oportunidades passa a ocultar oportunidades de ruído técnico como `Mail Delivery Subsystem`, `postmaster`, `mailer-daemon`, bounces/NDR. +- A coluna visual da board pode ser ajustada pela próxima ação pendente: fatura/pró-forma/pagamento aparecem em pagamento; encomenda/envio aparecem em operação/logística. + +## O que não muda + +- Não cria oportunidades automaticamente. +- Não liga documentos automaticamente. +- Não confirma pagamentos. +- Não altera documentos Jasmin/Odoo/Packlink. +- Não apaga dados externos. + +## Comandos recomendados + +```bash +PYTHONPATH=. python -m compileall app scripts tests +PYTHONPATH=. pytest -q +PYTHONPATH=. python scripts/cleanup_stale_reconciliation_items.py --days 3 --apply +PYTHONPATH=. python scripts/sync_external_reconciliation.py --jasmin --days 3 --limit 50 +PYTHONPATH=. python scripts/sync_external_reconciliation.py --odoo --days 3 --limit 50 +``` + +Evitar `--all` se Packlink ainda estiver com erro 401. diff --git a/docs/CLIENTFLOW_V499_PROCESS_TIMELINE_RECONSTRUCTION.md b/docs/CLIENTFLOW_V499_PROCESS_TIMELINE_RECONSTRUCTION.md new file mode 100644 index 0000000..b531372 --- /dev/null +++ b/docs/CLIENTFLOW_V499_PROCESS_TIMELINE_RECONSTRUCTION.md @@ -0,0 +1,33 @@ +# ClientFlow v4.9.9 — Process Timeline Reconstruction + +Esta versão melhora a Reconciliação para deixar de tratar documentos/vendas/comprovativos como ligações soltas. + +## Objetivo + +Quando o ClientFlow encontra evidências externas, tenta agrupá-las por NIF, email ou nome e propõe um processo provável: + +- orçamento Jasmin encontrado; +- pró-forma/fatura Jasmin encontrada; +- venda/encomenda Odoo encontrada; +- comprovativo recebido; +- envio Packlink encontrado. + +A UI mostra uma timeline reconstruída e o operador decide se deve criar uma oportunidade reconstruída ou ligar todos os itens a uma oportunidade existente. + +## Segurança + +A versão não confirma pagamentos, não emite faturas, não fecha oportunidades e não altera documentos externos automaticamente. + +## Fluxo + +1. Sincronizar Jasmin/Odoo/Packlink. +2. Abrir `/reconciliation`. +3. Rever "Processos candidatos". +4. Escolher uma ação: + - ligar processo à sugestão; + - ligar a outra oportunidade; + - criar oportunidade reconstruída. + +## Notas + +Eventos reconstruídos ficam registados como `reconciliation_evidence_imported` e a criação como `opportunity_reconstructed_from_reconciliation`. diff --git a/docs/JASMIN_CLIENTFLOW.md b/docs/JASMIN_CLIENTFLOW.md new file mode 100644 index 0000000..bf9e98e --- /dev/null +++ b/docs/JASMIN_CLIENTFLOW.md @@ -0,0 +1,135 @@ +# Integração Jasmin no ClientFlow + +Esta versão adiciona a primeira integração Jasmin validada com testes reais em Jasmin 3.02. + +## O que foi validado + +- OAuth Client Credentials: + - `https://identity.primaverabss.com/connect/token` + - `grant_type=client_credentials` + - `scope=application` +- `GET /businessCore/productInfos/getVersions` +- Clientes: + - `GET /salesCore/customerParties/getCustomerByCompanyTaxId/{nif}` + - `POST /salesCore/customerParties` +- Produtos: + - `GET /salesCore/salesItems/extension/odata?$top=50` +- Orçamentos: + - `POST /sales/quotations` + - `documentType=ORC` + - `serie=ORC2026` +- Faturas: + - `POST /billing/invoices/fromQuotation/{quotationId}` com body `{}` + +## Regras importantes descobertas + +1. O NIF deve ser pesquisado no Jasmin sem prefixo `PT`. +2. O Jasmin rejeita `electronicMail` e `telephone` vazios; campos opcionais vazios são omitidos. +3. OData tem limite de `$top=100`. +4. Cliente novo pode usar `partyKey=CF{NIF}`. +5. Converter orçamento em fatura exige `json={}`; sem body pode devolver `411 Length Required`. + +## Novas tabelas + +- `customers` +- `commercial_documents` +- `commercial_document_lines` +- `shipments` + +Estas tabelas permitem o pressuposto correto: + +```text +Cliente +→ várias oportunidades +→ vários orçamentos +→ várias faturas +``` + +## Fluxo na oportunidade + +A página da oportunidade tem dois botões simples: + +```text +[Criar orçamento] +[Converter em fatura] +``` + +`Criar orçamento` faz internamente: + +```text +find_or_create_customer +create_quotation +``` + +`Converter em fatura` faz internamente: + +```text +pegar no orçamento ativo/mais recente +POST /billing/invoices/fromQuotation/{quotationId} com {} +grava a fatura ligada ao orçamento local +``` + +## Variáveis `.env` + +```env +JASMIN_ENABLED=true +JASMIN_OUTBOX_ENABLED=true +JASMIN_BASE_URL=https://my.jasminsoftware.com +JASMIN_PUBLIC_URL=https://my.jasminsoftware.com +JASMIN_TOKEN_URL=https://identity.primaverabss.com/connect/token +JASMIN_SCOPE=application +JASMIN_ACCOUNT=... +JASMIN_SUBSCRIPTION=... +JASMIN_CLIENT_ID=... +JASMIN_CLIENT_SECRET=... + +JASMIN_COMPANY_KEY=CTULDA +JASMIN_QUOTATION_TYPE=ORC +JASMIN_QUOTATION_SERIE=ORC2026 +JASMIN_DEFAULT_PRICE_LIST=03 +JASMIN_DEFAULT_PAYMENT_METHOD=TRA +JASMIN_DEFAULT_PAYMENT_TERM=00 +JASMIN_DEFAULT_DELIVERY_TERM=TRANSP +JASMIN_DEFAULT_CURRENCY=EUR +JASMIN_DEFAULT_COUNTRY=PT +JASMIN_DEFAULT_CUSTOMER_GROUP=02 +JASMIN_DEFAULT_PARTY_TAX_SCHEMA=CONTINENTE +JASMIN_DEFAULT_UNIT=UN +JASMIN_DEFAULT_ITEM_TAX_SCHEMA=NORMAL +JASMIN_DEFAULT_SALES_ITEM=CARREGADOR_MONO_7KW +``` + +## Teste não destrutivo + +```bash +python scripts/test_jasmin_connection.py +``` + +## Processar outbox Jasmin + +Modo seguro: + +```bash +OUTBOX_TARGET_SYSTEM=jasmin JASMIN_OUTBOX_ENABLED=true OUTBOX_DRY_RUN=true python scripts/process_outbox.py +``` + +Modo real: + +```bash +OUTBOX_TARGET_SYSTEM=jasmin JASMIN_OUTBOX_ENABLED=true OUTBOX_DRY_RUN=false python scripts/process_outbox.py +``` + +## Dados necessários na oportunidade + +Para criar cliente Jasmin novo, a oportunidade precisa de: + +- nome do cliente +- NIF em `metadata.customer_tax_id`, `metadata.nif` ou `metadata.customer.tax_id` +- morada/código postal/cidade se disponível + +Para criar orçamento, as linhas da oportunidade precisam de mapear para artigos Jasmin. O serviço usa por ordem: + +1. `opportunity_items.metadata.jasmin_sales_item` +2. `opportunity_items.metadata.jasmin_item_key` +3. `opportunity_items.sku` +4. `JASMIN_DEFAULT_SALES_ITEM` diff --git a/docs/PACKLINK_CLIENTFLOW.md b/docs/PACKLINK_CLIENTFLOW.md new file mode 100644 index 0000000..f19abc7 --- /dev/null +++ b/docs/PACKLINK_CLIENTFLOW.md @@ -0,0 +1,118 @@ +# Integração Packlink PRO no ClientFlow + +## Estado validado + +Foram validados estes pontos da API Packlink PRO: + +- `GET /clients` com `Authorization: ` funciona. +- `GET /locations/postalcodes/PT/3650-219` funciona. +- `GET /services` funciona para Portugal quando os códigos postais são normalizados para 4 dígitos na cotação, por exemplo `3650` → `4000`. +- Serviço default validado: + - `service_id=20571` + - `carrier=Correos Express` + - `service=Paq 24` + - `departure_type=pick-up` + - `destination_type=home` + +## Configuração `.env` + +```env +PACKLINK_ENABLED=true +PACKLINK_OUTBOX_ENABLED=true +PACKLINK_BASE_URL=https://api.packlink.com/v1 +PACKLINK_PUBLIC_URL=https://pro.packlink.pt +PACKLINK_API_KEY=... + +PACKLINK_DEFAULT_SERVICE_ID=20571 +PACKLINK_DEFAULT_SERVICE=Paq 24 +PACKLINK_DEFAULT_CARRIER=Correos Express +PACKLINK_SOURCE=PRO +PACKLINK_PLATFORM=PRO +PACKLINK_PLATFORM_COUNTRY=UN + +PACKLINK_COLLECTION_TIME=09:00-14:00 +PACKLINK_COLLECTION_DAYS_AHEAD=1 + +PACKLINK_DEFAULT_PACKAGE_HEIGHT=10 +PACKLINK_DEFAULT_PACKAGE_WIDTH=20 +PACKLINK_DEFAULT_PACKAGE_LENGTH=30 +PACKLINK_DEFAULT_PACKAGE_WEIGHT=2 + +PACKLINK_SENDER_NAME=... +PACKLINK_SENDER_SURNAME=. +PACKLINK_SENDER_COMPANY=... +PACKLINK_SENDER_STREET1=... +PACKLINK_SENDER_STREET2= +PACKLINK_SENDER_ZIP=3650-219 +PACKLINK_SENDER_CITY=Vila Nova de Paiva +PACKLINK_SENDER_COUNTRY=PT +PACKLINK_SENDER_PHONE=... +PACKLINK_SENDER_EMAIL=... + +PACKLINK_FALLBACK_PHONE=... +PACKLINK_FALLBACK_EMAIL=... +``` + +## Teste de ligação + +```bash +python scripts/test_packlink_connection.py +``` + +## Processamento da outbox + +Por segurança, o `process_outbox.py` corre em dry-run por defeito. + +Teste sem criar envio real: + +```bash +OUTBOX_TARGET_SYSTEM=packlink \ +PACKLINK_OUTBOX_ENABLED=true \ +OUTBOX_DRY_RUN=true \ +python scripts/process_outbox.py +``` + +Criação real de envio: + +```bash +OUTBOX_TARGET_SYSTEM=packlink \ +PACKLINK_OUTBOX_ENABLED=true \ +OUTBOX_DRY_RUN=false \ +python scripts/process_outbox.py +``` + +## Dados necessários antes de criar envio + +A oportunidade precisa de ter dados de entrega em `metadata.shipment`, `metadata.packlink` ou numa preparação de tarefa `prep_type=shipment`. + +Campos aceites: + +```json +{ + "shipment": { + "recipient_name": "Nome Cliente", + "recipient_phone": "+351...", + "recipient_email": "cliente@example.com", + "delivery_address": "Rua Exemplo 1, 4000-001 Porto", + "country": "PT" + } +} +``` + +Se a morada tiver código postal PT completo, o ClientFlow guarda a morada completa, mas normaliza para 4 dígitos apenas na cotação Packlink. + +## Fluxo implementado + +1. Operador clica em `Criar envio Packlink` na oportunidade. +2. Se `PACKLINK_ENABLED=true` e não foi escrita referência manual, o ClientFlow cria um item `integration_outbox`: + - `target_system=packlink` + - `action_type=create_shipment` +3. `scripts/process_outbox.py` processa o item. +4. O Packlink devolve uma `reference`. +5. O ClientFlow regista `operation_links` com `system=packlink`, `external_type=shipment`, `status=created`. + +## Notas de segurança + +- A API key não deve ser colocada no repositório. +- O processamento real só deve correr com `OUTBOX_DRY_RUN=false` depois de confirmares o comportamento de pagamento/rascunho da tua conta Packlink PRO. +- A criação de envio pode gerar custos na conta Packlink, dependendo da configuração de pagamento. diff --git a/docs/RENOMEAR_CCE_PARA_CLIENTFLOW.md b/docs/RENOMEAR_CCE_PARA_CLIENTFLOW.md new file mode 100644 index 0000000..215ea23 --- /dev/null +++ b/docs/RENOMEAR_CCE_PARA_CLIENTFLOW.md @@ -0,0 +1,21 @@ +# Renomeação: CCE → ClientFlow + +A partir desta fase, o nome do motor passa a ser: + +```text +ClientFlow +``` + +O nome anterior CCE / Customer Context Engine fica apenas como referência histórica. + +## Frase curta + +ClientFlow é o motor de contexto e estado comercial que interpreta mensagens de clientes, normaliza o estado da conversa e alimenta Chatwoot, CRM e Mautic. + +## Componentes + +- ClientFlow Analyzer: chamada ao LLM. +- ClientFlow Normalizer: regras de negócio. +- ClientFlow State: estados finais. +- ClientFlow API: endpoint `/analyze`. +- ClientFlow Integrations: Chatwoot e Mautic. diff --git a/docs/env.odoo.example b/docs/env.odoo.example new file mode 100644 index 0000000..f6ba11f --- /dev/null +++ b/docs/env.odoo.example @@ -0,0 +1,8 @@ +# Odoo integration +ODOO_ENABLED=true +ODOO_BASE_URL=http://127.0.0.1:8069 +ODOO_PUBLIC_URL=https://odoo.blif.pt +ODOO_DB=odoo19_prod +ODOO_USERNAME=clientflow@blif.pt +ODOO_API_KEY=coloca_aqui_a_api_key_do_utilizador_odoo +ODOO_API_MODE=xmlrpc diff --git a/migrations/001_core_indexes.sql b/migrations/001_core_indexes.sql new file mode 100644 index 0000000..37f135d --- /dev/null +++ b/migrations/001_core_indexes.sql @@ -0,0 +1,17 @@ +-- ClientFlow v4 migration 001: core operational indexes and migration ledger. +CREATE EXTENSION IF NOT EXISTS pgcrypto; + +CREATE TABLE IF NOT EXISTS schema_migrations ( + version TEXT PRIMARY KEY, + name TEXT NOT NULL, + applied_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE INDEX IF NOT EXISTS idx_outbox_target_status_created +ON integration_outbox(target_system, status, created_at DESC); + +CREATE INDEX IF NOT EXISTS idx_outbox_status_created +ON integration_outbox(status, created_at DESC); + +CREATE INDEX IF NOT EXISTS idx_raw_events_source_created +ON raw_events(source_system, created_at DESC); diff --git a/migrations/002_customers_documents_shipments_indexes.sql b/migrations/002_customers_documents_shipments_indexes.sql new file mode 100644 index 0000000..4e030a8 --- /dev/null +++ b/migrations/002_customers_documents_shipments_indexes.sql @@ -0,0 +1,25 @@ +-- ClientFlow v4 migration 002: customer/document/shipment indexes. +CREATE INDEX IF NOT EXISTS idx_customers_tax_id +ON customers(tax_id) +WHERE tax_id IS NOT NULL AND tax_id <> ''; + +CREATE INDEX IF NOT EXISTS idx_customers_jasmin_party_key +ON customers(jasmin_customer_party_key) +WHERE jasmin_customer_party_key IS NOT NULL AND jasmin_customer_party_key <> ''; + +CREATE INDEX IF NOT EXISTS idx_commercial_documents_opportunity_kind +ON commercial_documents(opportunity_id, document_kind, created_at DESC); + +CREATE INDEX IF NOT EXISTS idx_commercial_documents_customer_kind +ON commercial_documents(customer_id, document_kind, created_at DESC); + +CREATE INDEX IF NOT EXISTS idx_commercial_documents_external +ON commercial_documents(system, external_id) +WHERE external_id IS NOT NULL AND external_id <> ''; + +CREATE INDEX IF NOT EXISTS idx_shipments_opportunity_status +ON shipments(opportunity_id, status, created_at DESC); + +CREATE INDEX IF NOT EXISTS idx_shipments_external_reference +ON shipments(system, external_reference) +WHERE external_reference IS NOT NULL AND external_reference <> ''; diff --git a/migrations/003_product_external_codes.sql b/migrations/003_product_external_codes.sql new file mode 100644 index 0000000..3db60ea --- /dev/null +++ b/migrations/003_product_external_codes.sql @@ -0,0 +1,12 @@ +-- ClientFlow v4 migration 003: separated external product codes. +ALTER TABLE products ADD COLUMN IF NOT EXISTS jasmin_sales_item TEXT; +ALTER TABLE opportunity_items ADD COLUMN IF NOT EXISTS jasmin_sales_item TEXT; +ALTER TABLE order_items ADD COLUMN IF NOT EXISTS jasmin_sales_item TEXT; + +CREATE INDEX IF NOT EXISTS idx_products_jasmin_sales_item +ON products(jasmin_sales_item) +WHERE jasmin_sales_item IS NOT NULL AND jasmin_sales_item <> ''; + +CREATE INDEX IF NOT EXISTS idx_opportunity_items_jasmin_sales_item +ON opportunity_items(jasmin_sales_item) +WHERE jasmin_sales_item IS NOT NULL AND jasmin_sales_item <> ''; diff --git a/migrations/004_outbox_safety.sql b/migrations/004_outbox_safety.sql new file mode 100644 index 0000000..1a9efce --- /dev/null +++ b/migrations/004_outbox_safety.sql @@ -0,0 +1,12 @@ +-- ClientFlow v4 migration 004: stronger outbox idempotency and state visibility. +ALTER TABLE integration_outbox ADD COLUMN IF NOT EXISTS locked_at TIMESTAMPTZ; +ALTER TABLE integration_outbox ADD COLUMN IF NOT EXISTS lock_owner TEXT; +ALTER TABLE integration_outbox ADD COLUMN IF NOT EXISTS ignored_at TIMESTAMPTZ; + +CREATE UNIQUE INDEX IF NOT EXISTS ux_integration_outbox_idempotency_key +ON integration_outbox(idempotency_key) +WHERE idempotency_key IS NOT NULL; + +CREATE INDEX IF NOT EXISTS idx_integration_outbox_pending_dispatch +ON integration_outbox(target_system, created_at) +WHERE status = 'pending'; diff --git a/migrations/005_operations_health_indexes.sql b/migrations/005_operations_health_indexes.sql new file mode 100644 index 0000000..863b44d --- /dev/null +++ b/migrations/005_operations_health_indexes.sql @@ -0,0 +1,19 @@ +-- ClientFlow v4.2 migration 005: operation dashboard helper indexes. +CREATE INDEX IF NOT EXISTS idx_opportunities_status_updated +ON opportunities(status, updated_at DESC); + +CREATE INDEX IF NOT EXISTS idx_tasks_status_created +ON tasks(status, created_at DESC); + +CREATE INDEX IF NOT EXISTS idx_commercial_documents_kind_created +ON commercial_documents(document_kind, created_at DESC); + +CREATE INDEX IF NOT EXISTS idx_commercial_documents_status_created +ON commercial_documents(status, created_at DESC); + +CREATE INDEX IF NOT EXISTS idx_shipments_status_created +ON shipments(status, created_at DESC); + +CREATE INDEX IF NOT EXISTS idx_products_active_jasmin_missing +ON products(active, jasmin_sales_item) +WHERE active = TRUE; diff --git a/migrations/006_v45_operational_core.sql b/migrations/006_v45_operational_core.sql new file mode 100644 index 0000000..860e120 --- /dev/null +++ b/migrations/006_v45_operational_core.sql @@ -0,0 +1,73 @@ +-- ClientFlow v4.5 Operational Core +-- Communications, contextual tasks and unified timeline. +-- Safe to run more than once. + +CREATE EXTENSION IF NOT EXISTS pgcrypto; + +CREATE TABLE IF NOT EXISTS communications ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + source_system TEXT NOT NULL DEFAULT 'email', + source_message_id TEXT, + thread_id TEXT, + conversation_id TEXT, + contact_id TEXT, + direction TEXT NOT NULL DEFAULT 'inbound', + sender_name TEXT, + sender_email TEXT, + recipient TEXT, + subject TEXT, + body TEXT, + classification TEXT, + confidence NUMERIC(4,3), + status TEXT NOT NULL DEFAULT 'new', + customer_id UUID, + opportunity_id UUID, + task_id UUID, + metadata JSONB NOT NULL DEFAULT '{}'::jsonb, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE UNIQUE INDEX IF NOT EXISTS ux_communications_source_message +ON communications(source_system, source_message_id) +WHERE source_message_id IS NOT NULL; + +CREATE INDEX IF NOT EXISTS idx_communications_created ON communications(created_at DESC); +CREATE INDEX IF NOT EXISTS idx_communications_status ON communications(status); +CREATE INDEX IF NOT EXISTS idx_communications_classification ON communications(classification); +CREATE INDEX IF NOT EXISTS idx_communications_sender_email ON communications(sender_email); +CREATE INDEX IF NOT EXISTS idx_communications_customer ON communications(customer_id); +CREATE INDEX IF NOT EXISTS idx_communications_opportunity ON communications(opportunity_id); + +ALTER TABLE tasks ADD COLUMN IF NOT EXISTS communication_id UUID; +ALTER TABLE tasks ADD COLUMN IF NOT EXISTS document_id UUID; +ALTER TABLE tasks ADD COLUMN IF NOT EXISTS shipment_id UUID; +ALTER TABLE tasks ADD COLUMN IF NOT EXISTS outbox_id UUID; +ALTER TABLE tasks ADD COLUMN IF NOT EXISTS priority TEXT NOT NULL DEFAULT 'normal'; +ALTER TABLE tasks ADD COLUMN IF NOT EXISTS assigned_to TEXT; + +CREATE INDEX IF NOT EXISTS idx_tasks_communication ON tasks(communication_id); +CREATE INDEX IF NOT EXISTS idx_tasks_document ON tasks(document_id); +CREATE INDEX IF NOT EXISTS idx_tasks_shipment ON tasks(shipment_id); +CREATE INDEX IF NOT EXISTS idx_tasks_outbox ON tasks(outbox_id); +CREATE INDEX IF NOT EXISTS idx_tasks_priority ON tasks(priority); +CREATE INDEX IF NOT EXISTS idx_tasks_due_status ON tasks(status, due_at); + +CREATE TABLE IF NOT EXISTS timeline_events ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + opportunity_id UUID, + customer_id UUID, + event_type TEXT NOT NULL, + title TEXT NOT NULL, + description TEXT, + source TEXT NOT NULL DEFAULT 'clientflow', + related_type TEXT, + related_id TEXT, + payload JSONB NOT NULL DEFAULT '{}'::jsonb, + created_by TEXT NOT NULL DEFAULT 'system', + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE INDEX IF NOT EXISTS idx_timeline_opportunity ON timeline_events(opportunity_id, created_at DESC); +CREATE INDEX IF NOT EXISTS idx_timeline_customer ON timeline_events(customer_id, created_at DESC); +CREATE INDEX IF NOT EXISTS idx_timeline_related ON timeline_events(related_type, related_id); diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..030acd8 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,29 @@ +annotated-types==0.7.0 +anyio==4.13.0 +certifi==2026.5.20 +charset-normalizer==3.4.7 +click==8.4.1 +fastapi==0.115.6 +greenlet==3.5.1 +h11==0.16.0 +httpcore==1.0.9 +httptools==0.7.1 +httpx==0.28.1 +idna==3.16 +psycopg==3.2.3 +psycopg-binary==3.2.3 +pydantic==2.10.4 +pydantic-settings==2.7.1 +pydantic_core==2.27.2 +python-dotenv==1.0.1 +PyYAML==6.0.3 +requests==2.34.2 +SQLAlchemy==2.0.36 +starlette==0.41.3 +typing_extensions==4.15.0 +urllib3==2.7.0 +uvicorn==0.34.0 +uvloop==0.22.1 +watchfiles==1.2.0 +websockets==16.0 +python-multipart==0.0.20 diff --git a/scripts/apply_migrations.py b/scripts/apply_migrations.py new file mode 100755 index 0000000..28aeb82 --- /dev/null +++ b/scripts/apply_migrations.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +"""Apply SQL migrations stored in ./migrations. + +Usage: + python scripts/apply_migrations.py + python scripts/apply_migrations.py --dry-run +""" +from __future__ import annotations + +import argparse +from pathlib import Path +import os +import sys + +PROJECT_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(PROJECT_ROOT)) +os.chdir(PROJECT_ROOT) + +from sqlalchemy import text + +from app.db import engine + +ROOT = Path(__file__).resolve().parents[1] +MIGRATIONS_DIR = ROOT / "migrations" + + +def ensure_ledger(conn) -> None: + conn.execute(text(""" + CREATE TABLE IF NOT EXISTS schema_migrations ( + version TEXT PRIMARY KEY, + name TEXT NOT NULL, + applied_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + """)) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--dry-run", action="store_true", help="List pending migrations without applying them.") + args = parser.parse_args() + + files = sorted(MIGRATIONS_DIR.glob("*.sql")) + if not files: + print("No migrations found.") + return 0 + + with engine.begin() as conn: + ensure_ledger(conn) + applied = { + row[0] + for row in conn.execute(text("SELECT version FROM schema_migrations")) + } + + pending = [] + for path in files: + version = path.stem.split("_", 1)[0] + if version not in applied: + pending.append((version, path)) + + if not pending: + print("No pending migrations.") + return 0 + + print("Pending migrations:") + for version, path in pending: + print(f"- {version}: {path.name}") + + if args.dry_run: + return 0 + + for version, path in pending: + sql = path.read_text() + print(f"Applying {path.name}...") + conn.execute(text(sql)) + conn.execute(text(""" + INSERT INTO schema_migrations(version, name) + VALUES (:version, :name) + ON CONFLICT (version) DO NOTHING + """), {"version": version, "name": path.name}) + + print("Migrations applied.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/backfill_chatwoot_inbox_to_clientflow.py b/scripts/backfill_chatwoot_inbox_to_clientflow.py new file mode 100755 index 0000000..d855531 --- /dev/null +++ b/scripts/backfill_chatwoot_inbox_to_clientflow.py @@ -0,0 +1,338 @@ +#!/usr/bin/env python3 +import hashlib +import hmac +import json +import os +import time +import urllib.error +import urllib.parse +import urllib.request +from datetime import datetime, timedelta, timezone +from typing import Any, Dict, List, Optional + + +def env(name: str, default: str = "") -> str: + return os.getenv(name, default).strip() + + +CHATWOOT_BASE_URL = env("CHATWOOT_BASE_URL").rstrip("/") +CHATWOOT_ACCOUNT_ID = env("CHATWOOT_ACCOUNT_ID") +CHATWOOT_API_TOKEN = env("CHATWOOT_API_TOKEN") +CLIENTFLOW_WEBHOOK_SECRET = env("CLIENTFLOW_WEBHOOK_SECRET") +CLIENTFLOW_WEBHOOK_URL = env("CLIENTFLOW_WEBHOOK_URL", "http://127.0.0.1:8020/webhooks/chatwoot") + +BACKFILL_DAYS = int(env("BACKFILL_DAYS", "5")) +BACKFILL_TO_CLIENTFLOW = env("BACKFILL_TO_CLIENTFLOW", "false").lower() == "true" +BACKFILL_STATUSES = [s.strip() for s in env("BACKFILL_STATUSES", "open,pending").split(",") if s.strip()] +BACKFILL_MAX_PAGES = int(env("BACKFILL_MAX_PAGES", "20")) + + +def request_json(method: str, url: str, body: Optional[Dict[str, Any]] = None, headers: Optional[Dict[str, str]] = None) -> Dict[str, Any]: + data = None + final_headers = headers.copy() if headers else {} + + if body is not None: + data = json.dumps(body, ensure_ascii=False).encode("utf-8") + final_headers["Content-Type"] = "application/json" + + req = urllib.request.Request(url, data=data, headers=final_headers, method=method) + + try: + with urllib.request.urlopen(req, timeout=45) as resp: + raw = resp.read().decode("utf-8", errors="replace") + return { + "ok": 200 <= resp.status < 300, + "status": resp.status, + "json": json.loads(raw) if raw else {}, + "raw": raw, + } + except urllib.error.HTTPError as e: + raw = e.read().decode("utf-8", errors="replace") + return { + "ok": False, + "status": e.code, + "json": None, + "raw": raw, + } + + +def chatwoot_headers() -> Dict[str, str]: + return { + "api_access_token": CHATWOOT_API_TOKEN, + "Accept": "application/json", + } + + +def payload_list(data: Any) -> List[Dict[str, Any]]: + if isinstance(data, list): + return data + + if not isinstance(data, dict): + return [] + + candidates = [ + data.get("payload"), + data.get("data", {}).get("payload") if isinstance(data.get("data"), dict) else None, + data.get("data"), + data.get("messages"), + ] + + for item in candidates: + if isinstance(item, list): + return item + + return [] + + +def ts_to_datetime(value: Any) -> Optional[datetime]: + if value is None: + return None + + try: + if isinstance(value, (int, float)): + return datetime.fromtimestamp(float(value), tz=timezone.utc) + + s = str(value).replace("Z", "+00:00") + return datetime.fromisoformat(s).astimezone(timezone.utc) + except Exception: + return None + + +def conversation_last_activity(conversation: Dict[str, Any]) -> Optional[datetime]: + for key in ["last_activity_at", "updated_at", "created_at"]: + dt = ts_to_datetime(conversation.get(key)) + if dt: + return dt + return None + + +def fetch_conversations(status: str) -> List[Dict[str, Any]]: + all_items: List[Dict[str, Any]] = [] + + for page in range(1, BACKFILL_MAX_PAGES + 1): + query = urllib.parse.urlencode({ + "status": status, + "page": page, + }) + + url = f"{CHATWOOT_BASE_URL}/api/v1/accounts/{CHATWOOT_ACCOUNT_ID}/conversations?{query}" + result = request_json("GET", url, headers=chatwoot_headers()) + + if not result["ok"]: + print(f"ERRO Chatwoot conversations status={status} page={page}: {result['status']} {result['raw'][:300]}") + break + + items = payload_list(result["json"]) + + if not items: + break + + all_items.extend(items) + + if len(items) < 10: + break + + return all_items + + +def fetch_messages(conversation_id: str) -> List[Dict[str, Any]]: + url = f"{CHATWOOT_BASE_URL}/api/v1/accounts/{CHATWOOT_ACCOUNT_ID}/conversations/{conversation_id}/messages" + result = request_json("GET", url, headers=chatwoot_headers()) + + if not result["ok"]: + print(f"ERRO Chatwoot messages conversation={conversation_id}: {result['status']} {result['raw'][:300]}") + return [] + + return payload_list(result["json"]) + + +def is_incoming(message: Dict[str, Any]) -> bool: + mt = message.get("message_type") + return mt == "incoming" or mt == 0 or str(mt).lower() == "incoming" + + +def message_created_at(message: Dict[str, Any]) -> datetime: + return ts_to_datetime(message.get("created_at")) or datetime.fromtimestamp(0, tz=timezone.utc) + + +def sender_from_conversation(conversation: Dict[str, Any], message: Dict[str, Any]) -> Dict[str, Any]: + sender = {} + + meta = conversation.get("meta") or {} + if isinstance(meta, dict) and isinstance(meta.get("sender"), dict): + sender.update(meta.get("sender") or {}) + + if isinstance(conversation.get("contact"), dict): + sender.update({k: v for k, v in conversation["contact"].items() if v is not None}) + + if isinstance(message.get("sender"), dict): + sender.update({k: v for k, v in message["sender"].items() if v is not None}) + + return sender + + +def sign_body(body_raw: str) -> Dict[str, str]: + ts = str(int(time.time())) + msg = ts.encode("utf-8") + b"." + body_raw.encode("utf-8") + sig = "sha256=" + hmac.new( + CLIENTFLOW_WEBHOOK_SECRET.encode("utf-8"), + msg, + hashlib.sha256, + ).hexdigest() + + return { + "Content-Type": "application/json", + "X-Chatwoot-Timestamp": ts, + "X-Chatwoot-Signature": sig, + } + + +def post_to_clientflow(payload: Dict[str, Any]) -> Dict[str, Any]: + # Importante: assinar e enviar exatamente o mesmo body_raw. + # Se o JSON for reformatado depois da assinatura, o webhook rejeita com 401. + body_raw = json.dumps(payload, ensure_ascii=False, separators=(",", ":")) + headers = sign_body(body_raw) + + req = urllib.request.Request( + CLIENTFLOW_WEBHOOK_URL, + data=body_raw.encode("utf-8"), + headers=headers, + method="POST", + ) + + try: + with urllib.request.urlopen(req, timeout=45) as resp: + raw = resp.read().decode("utf-8", errors="replace") + return { + "ok": 200 <= resp.status < 300, + "status": resp.status, + "json": json.loads(raw) if raw else {}, + "raw": raw, + } + except urllib.error.HTTPError as e: + raw = e.read().decode("utf-8", errors="replace") + return { + "ok": False, + "status": e.code, + "json": None, + "raw": raw, + } + + +def main() -> int: + required = { + "CHATWOOT_BASE_URL": CHATWOOT_BASE_URL, + "CHATWOOT_ACCOUNT_ID": CHATWOOT_ACCOUNT_ID, + "CHATWOOT_API_TOKEN": CHATWOOT_API_TOKEN, + "CLIENTFLOW_WEBHOOK_SECRET": CLIENTFLOW_WEBHOOK_SECRET, + } + + missing = [k for k, v in required.items() if not v] + if missing: + raise SystemExit(f"Faltam variáveis: {', '.join(missing)}") + + cutoff = datetime.now(timezone.utc) - timedelta(days=BACKFILL_DAYS) + + print(f"BACKFILL_DAYS={BACKFILL_DAYS}") + print(f"BACKFILL_STATUSES={BACKFILL_STATUSES}") + print(f"BACKFILL_TO_CLIENTFLOW={BACKFILL_TO_CLIENTFLOW}") + print(f"CUTOFF={cutoff.isoformat()}") + + seen_conversations = set() + selected = 0 + posted = 0 + failed = 0 + skipped = 0 + + for status in BACKFILL_STATUSES: + conversations = fetch_conversations(status) + print(f"--- status={status} conversations={len(conversations)}") + + for conv in conversations: + conv_id = str(conv.get("id") or "") + if not conv_id or conv_id in seen_conversations: + continue + + seen_conversations.add(conv_id) + + last_activity = conversation_last_activity(conv) + if last_activity and last_activity < cutoff: + skipped += 1 + continue + + messages = fetch_messages(conv_id) + incoming_messages = [ + m for m in messages + if is_incoming(m) + and not m.get("private") + and str(m.get("content") or "").strip() + and message_created_at(m) >= cutoff + ] + + if not incoming_messages: + skipped += 1 + continue + + incoming_messages.sort(key=message_created_at) + last_msg = incoming_messages[-1] + + content = str(last_msg.get("content") or "").strip() + sender = sender_from_conversation(conv, last_msg) + contact_id = str(sender.get("id") or conv.get("contact_id") or conv.get("contact", {}).get("id") or "") + + selected += 1 + + payload = { + "event": "message_created", + "message": { + "id": str(last_msg.get("id") or f"backfill-{conv_id}"), + "content": content, + "message_type": "incoming", + "conversation_id": conv_id, + "sender": sender, + "created_at": last_msg.get("created_at"), + }, + "conversation": { + "id": conv_id, + "status": conv.get("status") or status, + "contact": sender, + "meta": { + "sender": sender, + }, + }, + "backfill": { + "source": "chatwoot_inbox_last_days", + "days": BACKFILL_DAYS, + "status": status, + "last_activity_at": conv.get("last_activity_at"), + }, + } + + print("---") + print(f"conversation={conv_id} contact={contact_id} msg={payload['message']['id']}") + print(f"content={content[:160].replace(chr(10), ' ')}") + + if not BACKFILL_TO_CLIENTFLOW: + print("DRY_RUN") + continue + + result = post_to_clientflow(payload) + + if result["ok"]: + print(f"POSTED status={result['status']}") + posted += 1 + else: + print(f"FAILED status={result['status']} body={result['raw'][:500]}") + failed += 1 + + print("---") + print(f"selected={selected}") + print(f"posted={posted}") + print(f"failed={failed}") + print(f"skipped={skipped}") + + return 0 if failed == 0 else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/backfill_jasmin_opportunity_details.py b/scripts/backfill_jasmin_opportunity_details.py new file mode 100755 index 0000000..3711c59 --- /dev/null +++ b/scripts/backfill_jasmin_opportunity_details.py @@ -0,0 +1,312 @@ +#!/usr/bin/env python3 +"""Backfill Jasmin document details into existing opportunities. + +Use when an opportunity was created from a Jasmin reconciliation item before +v4926.6, so it still has no commercial_documents/opportunity_items/value. + +Examples: + PYTHONPATH=. python scripts/backfill_jasmin_opportunity_details.py \ + --opportunity-id a4e210f5-b870-48b1-882d-b55c71fbfd38 + + PYTHONPATH=. python scripts/backfill_jasmin_opportunity_details.py \ + --document-number ORC.ORC2026.156 +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import sys +from decimal import Decimal, InvalidOperation +from typing import Any, Dict, Iterable, List, Optional + +from sqlalchemy import text + +from app.db import engine +from app.reconciliation_service import ( + _apply_jasmin_documents_to_opportunity, # noqa: PLC2701 - deliberate operator backfill script + _jasmin_document_lines_from_item, # noqa: PLC2701 + _jasmin_document_totals, # noqa: PLC2701 + _payload_record, # noqa: PLC2701 +) + + +def _as_text(value: Any) -> str: + return str(value or "").strip() + + +def _money_value(value: Any) -> Any: + if isinstance(value, dict): + for key in ("amount", "baseAmount", "reportingAmount", "value"): + if value.get(key) not in (None, ""): + return value.get(key) + return None + return value + + +def _decimal_or_none(value: Any) -> Optional[str]: + value = _money_value(value) + if value in (None, ""): + return None + try: + return str(Decimal(str(value).replace(",", ".")).quantize(Decimal("0.01"))) + except (InvalidOperation, ValueError): + return None + + +def _json(value: Any) -> str: + return json.dumps(value, ensure_ascii=False, default=str) + + +def _ids_from_metadata(metadata: Any) -> List[str]: + if not isinstance(metadata, dict): + return [] + ids: List[str] = [] + for key in ("created_from_reconciliation_item_id", "reconciliation_item_id"): + value = metadata.get(key) + if value: + ids.append(str(value)) + for key in ("item_ids", "reconciliation_item_ids"): + value = metadata.get(key) + if isinstance(value, list): + ids.extend(str(v) for v in value if v) + return list(dict.fromkeys(ids)) + + +def _load_opportunity(opportunity_id: Optional[str], document_number: Optional[str]) -> Optional[Dict[str, Any]]: + with engine.begin() as conn: + if opportunity_id: + row = conn.execute(text(""" + SELECT id::text, title, value_amount, product_interest, local_customer_id::text, + customer_name, customer_email, metadata + FROM opportunities + WHERE id = CAST(:id AS UUID) + LIMIT 1 + """), {"id": opportunity_id}).mappings().first() + return dict(row) if row else None + if document_number: + row = conn.execute(text(""" + SELECT id::text, title, value_amount, product_interest, local_customer_id::text, + customer_name, customer_email, metadata + FROM opportunities + WHERE metadata->>'document_number' = :document_number + OR title ILIKE '%' || :document_number || '%' + ORDER BY updated_at DESC + LIMIT 1 + """), {"document_number": document_number}).mappings().first() + return dict(row) if row else None + return None + + +def _load_candidate_items(opportunity: Dict[str, Any]) -> List[Dict[str, Any]]: + metadata = opportunity.get("metadata") if isinstance(opportunity.get("metadata"), dict) else {} + ids = _ids_from_metadata(metadata) + external_id = _as_text(metadata.get("external_id")) + document_number = _as_text(metadata.get("document_number")) + opportunity_id = _as_text(opportunity.get("id")) + + with engine.begin() as conn: + rows = conn.execute(text(""" + SELECT id::text, source_system, external_type, external_id, title, description, + status, priority, suggested_action, confidence, opportunity_id::text, + customer_id::text, customer_name, customer_email, customer_tax_id, + document_number, document_date, amount, currency, payload, + resolution_note, created_at, updated_at, resolved_at + FROM reconciliation_items + WHERE source_system = 'jasmin' + AND ( + opportunity_id = CAST(:opportunity_id AS UUID) + OR (CAST(:ids AS TEXT[]) IS NOT NULL AND id::text = ANY(CAST(:ids AS TEXT[]))) + OR (CAST(:external_id AS TEXT) <> '' AND external_id = CAST(:external_id AS TEXT)) + OR (CAST(:document_number AS TEXT) <> '' AND document_number = CAST(:document_number AS TEXT)) + OR (CAST(:document_number AS TEXT) <> '' AND payload::text ILIKE '%' || CAST(:document_number AS TEXT) || '%') + ) + ORDER BY updated_at DESC, created_at DESC + """), { + "opportunity_id": opportunity_id, + "ids": ids or [], + "external_id": external_id, + "document_number": document_number, + }).mappings().all() + + # De-duplicate while keeping recency order. + seen = set() + result = [] + for row in rows: + item = dict(row) + item_id = item.get("id") + if item_id in seen: + continue + seen.add(item_id) + result.append(item) + return result + + +async def _fetch_jasmin_detail_async(item: Dict[str, Any]) -> Optional[Dict[str, Any]]: + external_type = _as_text(item.get("external_type")) + external_id = _as_text(item.get("external_id")) + if not external_id: + return None + from app.jasmin_client import JasminClient + + client = JasminClient() + if external_type == "jasmin_quotation": + return await client.get_quotation(external_id) + if external_type == "jasmin_invoice": + return await client.get_invoice(external_id) + # Some tenants represent pro-forma as a quotation. Try quotation detail as a + # conservative fallback when the external id is present. + if external_type == "jasmin_proforma": + try: + return await client.get_quotation(external_id) + except Exception: + return None + return None + + +def _with_jasmin_detail(item: Dict[str, Any], *, fetch_detail: bool) -> Dict[str, Any]: + if not fetch_detail: + return item + existing_lines = _jasmin_document_lines_from_item(item) + if existing_lines: + return item + try: + detail = asyncio.run(_fetch_jasmin_detail_async(item)) + except Exception as exc: + item = dict(item) + payload = item.get("payload") if isinstance(item.get("payload"), dict) else {} + item["payload"] = { + **payload, + "detail_fetch_error": f"{type(exc).__name__}: {exc}", + } + return item + if not isinstance(detail, dict): + return item + + payload = item.get("payload") if isinstance(item.get("payload"), dict) else {} + enriched = dict(item) + enriched["payload"] = { + **payload, + "record": detail, + "detail_source": "jasmin_api", + "previous_record": payload.get("record"), + } + + # Fill top-level fields if the detailed document exposes them only there. + record_number = detail.get("documentNumber") or detail.get("naturalKey") or detail.get("number") + if record_number and not enriched.get("document_number"): + enriched["document_number"] = record_number + total = ( + detail.get("payableAmount") + or detail.get("totalAmount") + or detail.get("grossAmount") + or detail.get("amount") + ) + if total and not enriched.get("amount"): + enriched["amount"] = _decimal_or_none(total) or total + return enriched + + +def _summary_for_items(items: Iterable[Dict[str, Any]]) -> List[Dict[str, Any]]: + result = [] + for item in items: + totals = _jasmin_document_totals(item) + lines = _jasmin_document_lines_from_item(item) + result.append({ + "id": item.get("id"), + "external_type": item.get("external_type"), + "external_id": item.get("external_id"), + "document_number": item.get("document_number"), + "amount": item.get("amount"), + "totals": totals, + "lines": len(lines), + "payload_keys": list((item.get("payload") or {}).keys()) if isinstance(item.get("payload"), dict) else [], + }) + return result + + +def _post_import_summary(opportunity_id: str) -> Dict[str, Any]: + with engine.begin() as conn: + opportunity = conn.execute(text(""" + SELECT id::text, title, value_amount, product_interest, metadata + FROM opportunities + WHERE id = CAST(:id AS UUID) + """), {"id": opportunity_id}).mappings().first() + docs = conn.execute(text(""" + SELECT id::text, document_kind, document_number, amount, total_amount, currency, document_date + FROM commercial_documents + WHERE opportunity_id = CAST(:id AS UUID) + ORDER BY created_at DESC + """), {"id": opportunity_id}).mappings().all() + items = conn.execute(text(""" + SELECT product_name, quantity, unit_price, total_price, jasmin_sales_item + FROM opportunity_items + WHERE opportunity_id = CAST(:id AS UUID) + ORDER BY created_at + """), {"id": opportunity_id}).mappings().all() + lines = conn.execute(text(""" + SELECT cdl.description, cdl.quantity, cdl.unit_price, cdl.total_amount, cdl.jasmin_sales_item + FROM commercial_document_lines cdl + JOIN commercial_documents cd ON cd.id = cdl.document_id + WHERE cd.opportunity_id = CAST(:id AS UUID) + ORDER BY cdl.line_index + """), {"id": opportunity_id}).mappings().all() + return { + "opportunity": dict(opportunity or {}), + "documents": [dict(r) for r in docs], + "opportunity_items": [dict(r) for r in items], + "document_lines": [dict(r) for r in lines], + } + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--opportunity-id") + parser.add_argument("--document-number") + parser.add_argument("--no-fetch-jasmin-detail", action="store_true") + parser.add_argument("--dry-run", action="store_true") + parser.add_argument("--actor", default="operator_backfill") + args = parser.parse_args() + + if not args.opportunity_id and not args.document_number: + parser.error("usa --opportunity-id ou --document-number") + + opportunity = _load_opportunity(args.opportunity_id, args.document_number) + if not opportunity: + print(json.dumps({"ok": False, "error": "opportunity_not_found"}, ensure_ascii=False, indent=2)) + return 2 + + items = _load_candidate_items(opportunity) + enriched_items = [ + _with_jasmin_detail(item, fetch_detail=not args.no_fetch_jasmin_detail) + for item in items + ] + print(json.dumps({ + "opportunity_id": opportunity.get("id"), + "title": opportunity.get("title"), + "candidate_items": _summary_for_items(enriched_items), + "dry_run": args.dry_run, + }, ensure_ascii=False, indent=2, default=str)) + + if not enriched_items: + print(json.dumps({"ok": False, "error": "no_jasmin_reconciliation_items_found"}, ensure_ascii=False, indent=2)) + return 3 + + if args.dry_run: + return 0 + + with engine.begin() as conn: + result = _apply_jasmin_documents_to_opportunity( + conn, + enriched_items, + str(opportunity["id"]), + actor=args.actor, + ) + print(json.dumps({"ok": True, "import_result": result}, ensure_ascii=False, indent=2, default=str)) + print(json.dumps(_post_import_summary(str(opportunity["id"])), ensure_ascii=False, indent=2, default=str)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/backfill_opportunity_product_mappings.py b/scripts/backfill_opportunity_product_mappings.py new file mode 100755 index 0000000..e93b3e1 --- /dev/null +++ b/scripts/backfill_opportunity_product_mappings.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +"""Backfill Jasmin product mappings for Odoo-imported opportunity lines. + +Maps Odoo line metadata product ids to the ClientFlow catalogue convention: +``product_id=3`` -> ``products.sku='ODOO-3'`` -> ``jasmin_sales_item``. + +Usage: + PYTHONPATH=. python scripts/backfill_opportunity_product_mappings.py + PYTHONPATH=. python scripts/backfill_opportunity_product_mappings.py --opportunity-id --apply +""" +from __future__ import annotations + +import argparse +from sqlalchemy import text + +from app.db import engine +from app.product_service import ensure_product_schema + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--opportunity-id", default="", help="Optional opportunity UUID to restrict the backfill") + parser.add_argument("--apply", action="store_true", help="Apply changes. Without this flag it runs as dry-run.") + args = parser.parse_args() + + ensure_product_schema() + where_opp = "AND oi.opportunity_id = CAST(:opportunity_id AS UUID)" if args.opportunity_id else "" + params = {"opportunity_id": args.opportunity_id} if args.opportunity_id else {} + + with engine.begin() as conn: + rows = conn.execute(text(f""" + SELECT + oi.id::text AS item_id, + oi.opportunity_id::text AS opportunity_id, + oi.product_name, + oi.sku AS old_sku, + oi.jasmin_sales_item AS old_jasmin_sales_item, + oi.metadata->>'product_id' AS odoo_product_id, + p.id::text AS product_id, + p.sku AS new_sku, + p.jasmin_sales_item AS new_jasmin_sales_item, + p.name AS catalog_name + FROM opportunity_items oi + JOIN products p + ON p.sku = ('ODOO-' || (oi.metadata->>'product_id')) + WHERE oi.metadata->>'source_system' = 'odoo' + AND COALESCE(oi.metadata->>'product_id','') <> '' + AND ( + oi.product_id IS DISTINCT FROM p.id + OR COALESCE(oi.sku,'') IS DISTINCT FROM COALESCE(p.sku,'') + OR COALESCE(oi.jasmin_sales_item,'') IS DISTINCT FROM COALESCE(p.jasmin_sales_item,'') + ) + {where_opp} + ORDER BY oi.created_at DESC + """), params).mappings().all() + + print(f"Candidatos a atualizar: {len(rows)}") + for row in rows: + print(dict(row)) + + if args.apply and rows: + result = conn.execute(text(f""" + UPDATE opportunity_items oi + SET product_id = p.id, + sku = p.sku, + jasmin_sales_item = p.jasmin_sales_item, + metadata = COALESCE(oi.metadata, '{{}}'::jsonb) || jsonb_build_object( + 'resolved_sku', p.sku, + 'resolved_jasmin_sales_item', p.jasmin_sales_item, + 'product_mapping_status', CASE WHEN COALESCE(p.jasmin_sales_item,'') <> '' THEN 'mapped' ELSE 'missing_jasmin' END, + 'catalog_name', p.name, + 'backfilled_at', now()::text + ), + updated_at = now() + FROM products p + WHERE p.sku = ('ODOO-' || (oi.metadata->>'product_id')) + AND oi.metadata->>'source_system' = 'odoo' + AND COALESCE(oi.metadata->>'product_id','') <> '' + {where_opp} + """), params) + print(f"Aplicado: {result.rowcount or 0} linha(s) atualizada(s)") + elif not args.apply: + print("Dry-run. Usa --apply para aplicar.") + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/batch_validate_email_identity.py b/scripts/batch_validate_email_identity.py new file mode 100755 index 0000000..ae3c28e --- /dev/null +++ b/scripts/batch_validate_email_identity.py @@ -0,0 +1,347 @@ +#!/usr/bin/env python3 +"""Batch validation for email identity extraction. + +Safe for LLM runs: prints progress, truncates long bodies, applies a per-message +process timeout, and writes JSONL incrementally so partial results are kept even +if a provider call stalls. +""" +from __future__ import annotations + +import argparse +import csv +import json +import multiprocessing as mp +import os +import re +from datetime import datetime +from pathlib import Path +from typing import Any, Dict, List + +from sqlalchemy import text + +from app.commercial_service import normalize_fiscal_name +from app.db import engine + + +LEGAL_SUFFIX_TOKENS = { + "lda", "limitada", "unipessoal", "sa", "s", "a", "sociedade", + "mediação", "mediacao", "seguro", "seguros", "importação", "importacao", + "exportação", "exportacao", "fabricação", "fabricacao", "representação", + "representacao", "soluções", "solucoes", "metálicas", "metalicas", +} + + +def clean(value: Any) -> str: + return re.sub(r"\s+", " ", str(value or "")).strip() + + +def norm(value: Any) -> str: + return normalize_fiscal_name(value or "") or clean(value).casefold() + + +def meaningful_tokens(value: Any) -> set[str]: + n = norm(value) + tokens = {t for t in re.split(r"[^a-z0-9áàâãéèêíìîóòôõúùûç]+", n) if len(t) >= 3} + return {t for t in tokens if t not in LEGAL_SUFFIX_TOKENS} + + +def mentions_match_fiscal(mentions: List[str], fiscal_name: str | None) -> bool: + if not mentions or not fiscal_name: + return False + + nf = norm(fiscal_name) + fiscal_tokens = meaningful_tokens(fiscal_name) + + for mention in mentions: + nm = norm(mention) + if not nm: + continue + if nm == nf: + return True + if len(nm) >= 4 and (nm in nf or nf in nm): + return True + mention_tokens = meaningful_tokens(mention) + if not mention_tokens or not fiscal_tokens: + continue + overlap = mention_tokens & fiscal_tokens + if len(overlap) >= 2: + return True + if len(mention_tokens) <= 2 and overlap: + return True + return False + + +def classify(identity: Dict[str, Any], fiscal_customer: str | None) -> str: + if identity.get("_error"): + return "EXTRACTION_ERROR" + if identity.get("_timeout"): + return "EXTRACTION_TIMEOUT" + + mentions = identity.get("company_mentions") or [] + domain = identity.get("domain") or "" + person = identity.get("person_name") or "" + + if mentions and fiscal_customer: + if mentions_match_fiscal(mentions, fiscal_customer): + return "OK_MENTION_COMPATIBLE_WITH_FISCAL" + return "CONFLICT_MENTION_DIFFERS_FROM_FISCAL" + if mentions and not fiscal_customer: + return "OK_MENTION_AVAILABLE_NO_FISCAL" + if not mentions and fiscal_customer: + return "WEAK_NO_COMPANY_MENTION_HAS_FISCAL" + if domain and person: + return "WEAK_PERSON_AND_DOMAIN_ONLY" + if domain: + return "WEAK_DOMAIN_ONLY" + return "NO_USEFUL_IDENTITY" + + +def fetch_cases(limit: int, offset: int) -> List[Dict[str, Any]]: + with engine.begin() as conn: + rows = conn.execute(text(""" + WITH ranked AS ( + SELECT + o.id::text AS opportunity_id, + o.title AS opportunity_title, + o.customer_name, + o.customer_email, + c.name AS fiscal_customer, + c.tax_id AS fiscal_tax_id, + c.email AS fiscal_email, + t.id::text AS task_id, + t.action_code, + t.route, + t.created_at AS task_created_at, + m.id::text AS message_id, + COALESCE(NULLIF(m.clean_body, ''), NULLIF(m.raw_body, '')) AS body, + COALESCE( + NULLIF(m.metadata->>'subject', ''), + NULLIF(re.payload->'conversation'->'additional_attributes'->>'mail_subject', ''), + NULLIF(re.payload->'content_attributes'->'email'->>'subject', ''), + NULLIF(re.payload->'conversation'->'messages'->0->'content_attributes'->'email'->>'subject', '') + ) AS subject, + COALESCE( + NULLIF(re.payload->'sender'->>'email', ''), + NULLIF(re.payload->'conversation'->'meta'->'sender'->>'email', ''), + NULLIF(re.payload->'conversation'->'contact_inbox'->>'source_id', ''), + NULLIF(o.customer_email, '') + ) AS sender_email, + ROW_NUMBER() OVER ( + PARTITION BY o.id + ORDER BY t.created_at DESC + ) AS rn + FROM opportunities o + JOIN tasks t ON t.opportunity_id = o.id + LEFT JOIN messages m ON m.id = t.message_id + LEFT JOIN raw_events re ON re.id = t.raw_event_id + LEFT JOIN customers c ON c.id = o.local_customer_id + WHERE COALESCE(NULLIF(m.clean_body, ''), NULLIF(m.raw_body, '')) IS NOT NULL + ) + SELECT * + FROM ranked + WHERE rn = 1 + ORDER BY task_created_at DESC + LIMIT :limit + OFFSET :offset + """), {"limit": limit, "offset": offset}).mappings().all() + return [dict(r) for r in rows] + + +def _domain_from_email(email: str) -> str: + if "@" not in (email or ""): + return "" + return email.rsplit("@", 1)[1].lower().strip() + + +def worker_extract(queue: mp.Queue, body: str, email: str, subject: str, use_llm: bool) -> None: + try: + from app.email_identity_extraction_service import extract_email_identity + + identity = extract_email_identity( + body or "", + email=email or "", + subject=subject or "", + use_llm=use_llm, + ) + queue.put({"ok": True, "identity": identity}) + except Exception as exc: # noqa: BLE001 validation tool should keep going + queue.put({"ok": False, "error": f"{type(exc).__name__}: {exc}"}) + + +def extract_with_timeout(body: str, email: str, subject: str, use_llm: bool, timeout_seconds: int) -> Dict[str, Any]: + if not use_llm: + from app.email_identity_extraction_service import extract_email_identity + return extract_email_identity(body or "", email=email or "", subject=subject or "", use_llm=False) + + queue: mp.Queue = mp.Queue() + proc = mp.Process(target=worker_extract, args=(queue, body, email, subject, use_llm)) + proc.start() + proc.join(timeout_seconds) + + if proc.is_alive(): + proc.terminate() + proc.join(5) + return { + "_timeout": True, + "method": "timeout", + "confidence": 0, + "email": email, + "domain": _domain_from_email(email), + "person_name": "", + "company_mentions": [], + "address": "", + "phones": [], + "websites": [], + "evidence": [f"timeout após {timeout_seconds}s"], + } + + if queue.empty(): + return { + "_error": True, + "method": "error", + "confidence": 0, + "email": email, + "domain": _domain_from_email(email), + "person_name": "", + "company_mentions": [], + "address": "", + "phones": [], + "websites": [], + "evidence": ["processo terminou sem resultado"], + } + + result = queue.get() + if result.get("ok"): + return result["identity"] + + return { + "_error": True, + "method": "error", + "confidence": 0, + "email": email, + "domain": _domain_from_email(email), + "person_name": "", + "company_mentions": [], + "address": "", + "phones": [], + "websites": [], + "evidence": [result.get("error")], + } + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--limit", type=int, default=20) + parser.add_argument("--offset", type=int, default=0) + parser.add_argument("--use-llm", action="store_true") + parser.add_argument("--model", default="", help="Override EMAIL_IDENTITY_LLM_MODEL for this validation run") + parser.add_argument("--fallback-model", default="", help="Override EMAIL_IDENTITY_LLM_FALLBACK_MODEL for this validation run") + parser.add_argument("--timeout-seconds", type=int, default=30) + parser.add_argument("--max-body-chars", type=int, default=3500) + parser.add_argument("--out-dir", default="reports") + args = parser.parse_args() + if args.model: + os.environ["EMAIL_IDENTITY_LLM_MODEL"] = args.model + if args.fallback_model: + os.environ["EMAIL_IDENTITY_LLM_FALLBACK_MODEL"] = args.fallback_model + + cases = fetch_cases(args.limit, args.offset) + out_dir = Path(args.out_dir) + out_dir.mkdir(parents=True, exist_ok=True) + + stamp = datetime.now().strftime("%Y%m%d_%H%M%S") + mode = "llm" if args.use_llm else "regex" + jsonl_path = out_dir / f"email_identity_validation_{mode}_{stamp}.jsonl" + csv_path = out_dir / f"email_identity_validation_{mode}_{stamp}.csv" + + results: List[Dict[str, Any]] = [] + counts: Dict[str, int] = {} + + model_label = os.getenv("EMAIL_IDENTITY_LLM_MODEL") or os.getenv("OPENROUTER_MODEL", "") + fallback_label = os.getenv("EMAIL_IDENTITY_LLM_FALLBACK_MODEL", "") + print( + f"Casos: {len(cases)} | mode={mode} | offset={args.offset} | " + f"timeout={args.timeout_seconds}s | max_body_chars={args.max_body_chars} | " + f"model={model_label if args.use_llm else '-'} | fallback={fallback_label if args.use_llm and fallback_label else '-'}", + flush=True, + ) + print(f"JSONL incremental: {jsonl_path}", flush=True) + + with jsonl_path.open("w", encoding="utf-8") as jf: + for idx, row in enumerate(cases, start=1): + email = row.get("sender_email") or row.get("customer_email") or "" + body = (row.get("body") or "")[: args.max_body_chars] + subject = row.get("subject") or "" + + print(f"\n[{idx}/{len(cases)}] {row.get('opportunity_title')} | {email}", flush=True) + + identity = extract_with_timeout( + body=body, + email=email, + subject=subject, + use_llm=args.use_llm, + timeout_seconds=args.timeout_seconds, + ) + status = classify(identity, row.get("fiscal_customer")) + + result = { + "status": status, + "opportunity_id": row.get("opportunity_id"), + "opportunity_title": row.get("opportunity_title"), + "task_id": row.get("task_id"), + "action_code": row.get("action_code"), + "sender_email": email, + "customer_name": row.get("customer_name"), + "customer_email": row.get("customer_email"), + "fiscal_customer": row.get("fiscal_customer"), + "fiscal_tax_id": row.get("fiscal_tax_id"), + "fiscal_email": row.get("fiscal_email"), + "person_name": identity.get("person_name"), + "company_mentions": identity.get("company_mentions") or [], + "domain": identity.get("domain"), + "address": identity.get("address"), + "phones": identity.get("phones") or [], + "confidence": identity.get("confidence"), + "method": identity.get("method"), + "llm_model": identity.get("llm_model"), + "fallback_used": identity.get("fallback_used"), + "evidence": identity.get("evidence") or [], + } + results.append(result) + counts[status] = counts.get(status, 0) + 1 + jf.write(json.dumps(result, ensure_ascii=False, default=str) + "\n") + jf.flush() + + print( + "status:", status, + "| person:", result["person_name"], + "| companies:", result["company_mentions"], + flush=True, + ) + + csv_fields = [ + "status", "opportunity_id", "opportunity_title", "task_id", "action_code", + "sender_email", "customer_name", "customer_email", "fiscal_customer", + "fiscal_tax_id", "fiscal_email", "person_name", "company_mentions", + "domain", "address", "phones", "confidence", "method", "llm_model", "fallback_used", "evidence", + ] + with csv_path.open("w", encoding="utf-8", newline="") as f: + writer = csv.DictWriter(f, fieldnames=csv_fields) + writer.writeheader() + for r in results: + csv_row = dict(r) + for key in ["company_mentions", "phones", "evidence"]: + csv_row[key] = " | ".join(str(x) for x in csv_row.get(key) or []) + writer.writerow(csv_row) + + print("\nResumo:") + for status, count in sorted(counts.items()): + print(f"{status}: {count}") + + print("\nFicheiros gerados:") + print(jsonl_path) + print(csv_path) + + +if __name__ == "__main__": + main() diff --git a/scripts/batch_validate_email_identity_safe.py b/scripts/batch_validate_email_identity_safe.py new file mode 100755 index 0000000..ae3c28e --- /dev/null +++ b/scripts/batch_validate_email_identity_safe.py @@ -0,0 +1,347 @@ +#!/usr/bin/env python3 +"""Batch validation for email identity extraction. + +Safe for LLM runs: prints progress, truncates long bodies, applies a per-message +process timeout, and writes JSONL incrementally so partial results are kept even +if a provider call stalls. +""" +from __future__ import annotations + +import argparse +import csv +import json +import multiprocessing as mp +import os +import re +from datetime import datetime +from pathlib import Path +from typing import Any, Dict, List + +from sqlalchemy import text + +from app.commercial_service import normalize_fiscal_name +from app.db import engine + + +LEGAL_SUFFIX_TOKENS = { + "lda", "limitada", "unipessoal", "sa", "s", "a", "sociedade", + "mediação", "mediacao", "seguro", "seguros", "importação", "importacao", + "exportação", "exportacao", "fabricação", "fabricacao", "representação", + "representacao", "soluções", "solucoes", "metálicas", "metalicas", +} + + +def clean(value: Any) -> str: + return re.sub(r"\s+", " ", str(value or "")).strip() + + +def norm(value: Any) -> str: + return normalize_fiscal_name(value or "") or clean(value).casefold() + + +def meaningful_tokens(value: Any) -> set[str]: + n = norm(value) + tokens = {t for t in re.split(r"[^a-z0-9áàâãéèêíìîóòôõúùûç]+", n) if len(t) >= 3} + return {t for t in tokens if t not in LEGAL_SUFFIX_TOKENS} + + +def mentions_match_fiscal(mentions: List[str], fiscal_name: str | None) -> bool: + if not mentions or not fiscal_name: + return False + + nf = norm(fiscal_name) + fiscal_tokens = meaningful_tokens(fiscal_name) + + for mention in mentions: + nm = norm(mention) + if not nm: + continue + if nm == nf: + return True + if len(nm) >= 4 and (nm in nf or nf in nm): + return True + mention_tokens = meaningful_tokens(mention) + if not mention_tokens or not fiscal_tokens: + continue + overlap = mention_tokens & fiscal_tokens + if len(overlap) >= 2: + return True + if len(mention_tokens) <= 2 and overlap: + return True + return False + + +def classify(identity: Dict[str, Any], fiscal_customer: str | None) -> str: + if identity.get("_error"): + return "EXTRACTION_ERROR" + if identity.get("_timeout"): + return "EXTRACTION_TIMEOUT" + + mentions = identity.get("company_mentions") or [] + domain = identity.get("domain") or "" + person = identity.get("person_name") or "" + + if mentions and fiscal_customer: + if mentions_match_fiscal(mentions, fiscal_customer): + return "OK_MENTION_COMPATIBLE_WITH_FISCAL" + return "CONFLICT_MENTION_DIFFERS_FROM_FISCAL" + if mentions and not fiscal_customer: + return "OK_MENTION_AVAILABLE_NO_FISCAL" + if not mentions and fiscal_customer: + return "WEAK_NO_COMPANY_MENTION_HAS_FISCAL" + if domain and person: + return "WEAK_PERSON_AND_DOMAIN_ONLY" + if domain: + return "WEAK_DOMAIN_ONLY" + return "NO_USEFUL_IDENTITY" + + +def fetch_cases(limit: int, offset: int) -> List[Dict[str, Any]]: + with engine.begin() as conn: + rows = conn.execute(text(""" + WITH ranked AS ( + SELECT + o.id::text AS opportunity_id, + o.title AS opportunity_title, + o.customer_name, + o.customer_email, + c.name AS fiscal_customer, + c.tax_id AS fiscal_tax_id, + c.email AS fiscal_email, + t.id::text AS task_id, + t.action_code, + t.route, + t.created_at AS task_created_at, + m.id::text AS message_id, + COALESCE(NULLIF(m.clean_body, ''), NULLIF(m.raw_body, '')) AS body, + COALESCE( + NULLIF(m.metadata->>'subject', ''), + NULLIF(re.payload->'conversation'->'additional_attributes'->>'mail_subject', ''), + NULLIF(re.payload->'content_attributes'->'email'->>'subject', ''), + NULLIF(re.payload->'conversation'->'messages'->0->'content_attributes'->'email'->>'subject', '') + ) AS subject, + COALESCE( + NULLIF(re.payload->'sender'->>'email', ''), + NULLIF(re.payload->'conversation'->'meta'->'sender'->>'email', ''), + NULLIF(re.payload->'conversation'->'contact_inbox'->>'source_id', ''), + NULLIF(o.customer_email, '') + ) AS sender_email, + ROW_NUMBER() OVER ( + PARTITION BY o.id + ORDER BY t.created_at DESC + ) AS rn + FROM opportunities o + JOIN tasks t ON t.opportunity_id = o.id + LEFT JOIN messages m ON m.id = t.message_id + LEFT JOIN raw_events re ON re.id = t.raw_event_id + LEFT JOIN customers c ON c.id = o.local_customer_id + WHERE COALESCE(NULLIF(m.clean_body, ''), NULLIF(m.raw_body, '')) IS NOT NULL + ) + SELECT * + FROM ranked + WHERE rn = 1 + ORDER BY task_created_at DESC + LIMIT :limit + OFFSET :offset + """), {"limit": limit, "offset": offset}).mappings().all() + return [dict(r) for r in rows] + + +def _domain_from_email(email: str) -> str: + if "@" not in (email or ""): + return "" + return email.rsplit("@", 1)[1].lower().strip() + + +def worker_extract(queue: mp.Queue, body: str, email: str, subject: str, use_llm: bool) -> None: + try: + from app.email_identity_extraction_service import extract_email_identity + + identity = extract_email_identity( + body or "", + email=email or "", + subject=subject or "", + use_llm=use_llm, + ) + queue.put({"ok": True, "identity": identity}) + except Exception as exc: # noqa: BLE001 validation tool should keep going + queue.put({"ok": False, "error": f"{type(exc).__name__}: {exc}"}) + + +def extract_with_timeout(body: str, email: str, subject: str, use_llm: bool, timeout_seconds: int) -> Dict[str, Any]: + if not use_llm: + from app.email_identity_extraction_service import extract_email_identity + return extract_email_identity(body or "", email=email or "", subject=subject or "", use_llm=False) + + queue: mp.Queue = mp.Queue() + proc = mp.Process(target=worker_extract, args=(queue, body, email, subject, use_llm)) + proc.start() + proc.join(timeout_seconds) + + if proc.is_alive(): + proc.terminate() + proc.join(5) + return { + "_timeout": True, + "method": "timeout", + "confidence": 0, + "email": email, + "domain": _domain_from_email(email), + "person_name": "", + "company_mentions": [], + "address": "", + "phones": [], + "websites": [], + "evidence": [f"timeout após {timeout_seconds}s"], + } + + if queue.empty(): + return { + "_error": True, + "method": "error", + "confidence": 0, + "email": email, + "domain": _domain_from_email(email), + "person_name": "", + "company_mentions": [], + "address": "", + "phones": [], + "websites": [], + "evidence": ["processo terminou sem resultado"], + } + + result = queue.get() + if result.get("ok"): + return result["identity"] + + return { + "_error": True, + "method": "error", + "confidence": 0, + "email": email, + "domain": _domain_from_email(email), + "person_name": "", + "company_mentions": [], + "address": "", + "phones": [], + "websites": [], + "evidence": [result.get("error")], + } + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--limit", type=int, default=20) + parser.add_argument("--offset", type=int, default=0) + parser.add_argument("--use-llm", action="store_true") + parser.add_argument("--model", default="", help="Override EMAIL_IDENTITY_LLM_MODEL for this validation run") + parser.add_argument("--fallback-model", default="", help="Override EMAIL_IDENTITY_LLM_FALLBACK_MODEL for this validation run") + parser.add_argument("--timeout-seconds", type=int, default=30) + parser.add_argument("--max-body-chars", type=int, default=3500) + parser.add_argument("--out-dir", default="reports") + args = parser.parse_args() + if args.model: + os.environ["EMAIL_IDENTITY_LLM_MODEL"] = args.model + if args.fallback_model: + os.environ["EMAIL_IDENTITY_LLM_FALLBACK_MODEL"] = args.fallback_model + + cases = fetch_cases(args.limit, args.offset) + out_dir = Path(args.out_dir) + out_dir.mkdir(parents=True, exist_ok=True) + + stamp = datetime.now().strftime("%Y%m%d_%H%M%S") + mode = "llm" if args.use_llm else "regex" + jsonl_path = out_dir / f"email_identity_validation_{mode}_{stamp}.jsonl" + csv_path = out_dir / f"email_identity_validation_{mode}_{stamp}.csv" + + results: List[Dict[str, Any]] = [] + counts: Dict[str, int] = {} + + model_label = os.getenv("EMAIL_IDENTITY_LLM_MODEL") or os.getenv("OPENROUTER_MODEL", "") + fallback_label = os.getenv("EMAIL_IDENTITY_LLM_FALLBACK_MODEL", "") + print( + f"Casos: {len(cases)} | mode={mode} | offset={args.offset} | " + f"timeout={args.timeout_seconds}s | max_body_chars={args.max_body_chars} | " + f"model={model_label if args.use_llm else '-'} | fallback={fallback_label if args.use_llm and fallback_label else '-'}", + flush=True, + ) + print(f"JSONL incremental: {jsonl_path}", flush=True) + + with jsonl_path.open("w", encoding="utf-8") as jf: + for idx, row in enumerate(cases, start=1): + email = row.get("sender_email") or row.get("customer_email") or "" + body = (row.get("body") or "")[: args.max_body_chars] + subject = row.get("subject") or "" + + print(f"\n[{idx}/{len(cases)}] {row.get('opportunity_title')} | {email}", flush=True) + + identity = extract_with_timeout( + body=body, + email=email, + subject=subject, + use_llm=args.use_llm, + timeout_seconds=args.timeout_seconds, + ) + status = classify(identity, row.get("fiscal_customer")) + + result = { + "status": status, + "opportunity_id": row.get("opportunity_id"), + "opportunity_title": row.get("opportunity_title"), + "task_id": row.get("task_id"), + "action_code": row.get("action_code"), + "sender_email": email, + "customer_name": row.get("customer_name"), + "customer_email": row.get("customer_email"), + "fiscal_customer": row.get("fiscal_customer"), + "fiscal_tax_id": row.get("fiscal_tax_id"), + "fiscal_email": row.get("fiscal_email"), + "person_name": identity.get("person_name"), + "company_mentions": identity.get("company_mentions") or [], + "domain": identity.get("domain"), + "address": identity.get("address"), + "phones": identity.get("phones") or [], + "confidence": identity.get("confidence"), + "method": identity.get("method"), + "llm_model": identity.get("llm_model"), + "fallback_used": identity.get("fallback_used"), + "evidence": identity.get("evidence") or [], + } + results.append(result) + counts[status] = counts.get(status, 0) + 1 + jf.write(json.dumps(result, ensure_ascii=False, default=str) + "\n") + jf.flush() + + print( + "status:", status, + "| person:", result["person_name"], + "| companies:", result["company_mentions"], + flush=True, + ) + + csv_fields = [ + "status", "opportunity_id", "opportunity_title", "task_id", "action_code", + "sender_email", "customer_name", "customer_email", "fiscal_customer", + "fiscal_tax_id", "fiscal_email", "person_name", "company_mentions", + "domain", "address", "phones", "confidence", "method", "llm_model", "fallback_used", "evidence", + ] + with csv_path.open("w", encoding="utf-8", newline="") as f: + writer = csv.DictWriter(f, fieldnames=csv_fields) + writer.writeheader() + for r in results: + csv_row = dict(r) + for key in ["company_mentions", "phones", "evidence"]: + csv_row[key] = " | ".join(str(x) for x in csv_row.get(key) or []) + writer.writerow(csv_row) + + print("\nResumo:") + for status, count in sorted(counts.items()): + print(f"{status}: {count}") + + print("\nFicheiros gerados:") + print(jsonl_path) + print(csv_path) + + +if __name__ == "__main__": + main() diff --git a/scripts/check_clientflow_health.py b/scripts/check_clientflow_health.py new file mode 100755 index 0000000..7d90c50 --- /dev/null +++ b/scripts/check_clientflow_health.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +"""Operational health check for ClientFlow deployments.""" +from __future__ import annotations + +import json +import os +import sys +from pathlib import Path + +PROJECT_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(PROJECT_ROOT)) +os.chdir(PROJECT_ROOT) +from sqlalchemy import text + +from app.config import settings +from app.db import engine + + +def scalar(sql: str, **params): + with engine.begin() as conn: + return conn.execute(text(sql), params).scalar() + + +def rows(sql: str, **params): + with engine.begin() as conn: + return [dict(r) for r in conn.execute(text(sql), params).mappings().all()] + + +def main() -> int: + result = { + "app": settings.app_name, + "env": settings.env, + "database_ok": False, + "jasmin_enabled": bool(settings.jasmin_enabled), + "packlink_enabled": bool(settings.packlink_enabled), + "outbox": {}, + "documents": {}, + "warnings": [], + } + + try: + result["database_ok"] = bool(scalar("SELECT 1")) + except Exception as exc: + result["warnings"].append(f"database_error: {exc}") + print(json.dumps(result, ensure_ascii=False, indent=2, default=str)) + return 2 + + for row in rows(""" + SELECT target_system, status, count(*)::int AS total + FROM integration_outbox + GROUP BY target_system, status + ORDER BY target_system, status + """): + result["outbox"].setdefault(row["target_system"], {})[row["status"]] = row["total"] + + for row in rows(""" + SELECT document_kind, status, count(*)::int AS total + FROM commercial_documents + GROUP BY document_kind, status + ORDER BY document_kind, status + """): + result["documents"].setdefault(row["document_kind"], {})[row["status"]] = row["total"] + + missing_jasmin = scalar(""" + SELECT count(*) + FROM products + WHERE active = TRUE + AND (jasmin_sales_item IS NULL OR jasmin_sales_item = '') + """) + if missing_jasmin: + result["warnings"].append(f"active_products_without_jasmin_sales_item={missing_jasmin}") + + failed_outbox = scalar("SELECT count(*) FROM integration_outbox WHERE status = 'failed'") + if failed_outbox: + result["warnings"].append(f"failed_outbox_items={failed_outbox}") + + print(json.dumps(result, ensure_ascii=False, indent=2, default=str)) + return 1 if result["warnings"] else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/cleanup_invalid_email_identity_suggestions.py b/scripts/cleanup_invalid_email_identity_suggestions.py new file mode 100755 index 0000000..69ef1f6 --- /dev/null +++ b/scripts/cleanup_invalid_email_identity_suggestions.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python3 +"""Clean stale invalid email-identity suggestions/extractions. + +Use after strengthening company-mention filters. It targets suggestions and +stored extractions created from invalid fragments such as ``pt`` or ``com``. +""" +from __future__ import annotations + +import argparse +import json +from typing import Any + +from sqlalchemy import text + +from app.db import engine +from app.email_identity_extraction_service import is_plausible_company_mention + +INVALID = {"pt", "com", "net", "org", "www", "http", "https", "mail", "email"} + + +def _clean(value: Any) -> str: + return str(value or "").strip() + + +def _valid_companies(values: Any) -> list[str]: + out: list[str] = [] + if not isinstance(values, list): + return out + for value in values: + v = _clean(value).strip(" ,.;:-") + if v and is_plausible_company_mention(v): + out.append(v) + return out + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--opportunity-id", default="") + parser.add_argument("--apply", action="store_true") + parser.add_argument("--include-accepted", action="store_true") + parser.add_argument("--fix-extractions", action="store_true", help="Also rewrite stored email_identity_extractions company_mentions after current filters") + args = parser.parse_args() + + where = ["lookup_type LIKE 'email_identity%'"] + params: dict[str, Any] = {} + if args.opportunity_id: + where.append("opportunity_id = CAST(:opportunity_id AS UUID)") + params["opportunity_id"] = args.opportunity_id + if args.include_accepted: + where.append("status IN ('pending', 'accepted', 'rejected')") + else: + where.append("status = 'pending'") + invalid_sql = ", ".join("'" + v.replace("'", "") + "'" for v in sorted(INVALID)) + where.append(f"lower(trim(COALESCE(lookup_value, ''))) IN ({invalid_sql})") + + sql_where = " AND ".join(where) + with engine.begin() as conn: + rows = conn.execute(text(f""" + SELECT id::text, opportunity_id::text, suggested_name, suggested_nif, + lookup_type, lookup_value, confidence, status, reason, created_at + FROM fiscal_customer_suggestions + WHERE {sql_where} + ORDER BY created_at DESC + """), params).mappings().all() + + print(json.dumps({"invalid_suggestions": len(rows), "apply": args.apply}, ensure_ascii=False, indent=2, default=str)) + for row in rows: + print(json.dumps(dict(row), ensure_ascii=False, indent=2, default=str)) + + if args.apply and rows: + ids = [r["id"] for r in rows] + conn.execute(text(""" + UPDATE fiscal_customer_suggestions + SET status = 'rejected', + reason = COALESCE(reason, '') || ' | rejected_invalid_email_identity_token', + resolved_by = 'cleanup_invalid_email_identity_suggestions', + resolved_at = now(), + updated_at = now() + WHERE id = ANY(CAST(:ids AS UUID[])) + """), {"ids": ids}) + print(json.dumps({"rejected": len(ids)}, ensure_ascii=False, indent=2)) + + if args.fix_extractions: + extraction_where = [] + extraction_params: dict[str, Any] = {} + if args.opportunity_id: + extraction_where.append("opportunity_id = CAST(:opportunity_id AS UUID)") + extraction_params["opportunity_id"] = args.opportunity_id + extraction_sql = "WHERE " + " AND ".join(extraction_where) if extraction_where else "" + ex_rows = conn.execute(text(f""" + SELECT id::text, opportunity_id::text, company_mentions, confidence, raw_payload + FROM email_identity_extractions + {extraction_sql} + ORDER BY updated_at DESC + """), extraction_params).mappings().all() + changed = [] + for row in ex_rows: + original = row.get("company_mentions") or [] + valid = _valid_companies(original) + if list(original or []) == valid: + continue + changed.append({"id": row["id"], "opportunity_id": row["opportunity_id"], "before": original, "after": valid}) + if args.apply: + try: + confidence = float(row.get("confidence") or 0) + except Exception: + confidence = 0.0 + if not valid: + confidence = min(confidence, 0.45) + raw_payload = dict(row.get("raw_payload") or {}) if isinstance(row.get("raw_payload"), dict) else {} + raw_payload["filtered_invalid_company_mentions"] = list(original or []) + conn.execute(text(""" + UPDATE email_identity_extractions + SET company_mentions = CAST(:company_mentions AS JSONB), + confidence = :confidence, + raw_payload = COALESCE(raw_payload, '{}'::jsonb) || CAST(:raw_payload AS JSONB), + updated_at = now() + WHERE id = CAST(:id AS UUID) + """), { + "id": row["id"], + "company_mentions": json.dumps(valid, ensure_ascii=False), + "confidence": confidence, + "raw_payload": json.dumps(raw_payload, ensure_ascii=False, default=str), + }) + print(json.dumps({"extractions_to_fix": len(changed), "items": changed}, ensure_ascii=False, indent=2, default=str)) + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/cleanup_non_commercial_opportunities.py b/scripts/cleanup_non_commercial_opportunities.py new file mode 100755 index 0000000..83cae60 --- /dev/null +++ b/scripts/cleanup_non_commercial_opportunities.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 +"""Identify or close opportunities that were probably created from system messages. + +Conservative by default: use --dry-run to list candidates. Use --apply to mark +safe candidates as LOST with a metadata reason. It only targets opportunities +with value 0, no commercial documents and no shipments. +""" +from __future__ import annotations + +import argparse +import json +import uuid + +from sqlalchemy import text + +from app.db import engine +from app.opportunity_service import ensure_opportunity_schema + +SYSTEM_TERMS = ( + "mail delivery subsystem", + "mailer-daemon", + "postmaster", + "returned mail", + "undelivered mail", + "delivery status notification", + "failure notice", + "mail delivery failed", +) + + +def _json(value: object) -> str: + return json.dumps(value or {}, ensure_ascii=False, default=str) + + +def find_candidates(limit: int = 100) -> list[dict]: + ensure_opportunity_schema() + like_sql = " OR ".join( + ["lower(coalesce(o.title,'') || ' ' || coalesce(o.customer_name,'') || ' ' || coalesce(o.customer_email,'') || ' ' || coalesce(o.metadata::text,'')) LIKE :term_{}".format(i) for i, _ in enumerate(SYSTEM_TERMS)] + ) + params = {f"term_{i}": f"%{term}%" for i, term in enumerate(SYSTEM_TERMS)} + params["limit"] = int(limit) + sql = text(f""" + SELECT o.id::text, o.title, o.customer_name, o.customer_email, o.value_amount, o.stage, o.status, o.created_at + FROM opportunities o + WHERE o.status = 'open' + AND COALESCE(o.value_amount, 0) = 0 + AND NOT EXISTS (SELECT 1 FROM commercial_documents cd WHERE cd.opportunity_id = o.id) + AND NOT EXISTS (SELECT 1 FROM shipments s WHERE s.opportunity_id = o.id) + AND ({like_sql}) + ORDER BY o.created_at DESC + LIMIT :limit + """) + with engine.begin() as conn: + rows = conn.execute(sql, params).mappings().all() + return [dict(row) for row in rows] + + +def close_candidates(candidates: list[dict]) -> int: + if not candidates: + return 0 + ids = [row["id"] for row in candidates] + with engine.begin() as conn: + for opportunity_id in ids: + conn.execute(text(""" + UPDATE opportunities + SET status = 'closed', + stage = 'LOST', + closed_at = COALESCE(closed_at, now()), + updated_at = now(), + metadata = COALESCE(metadata, '{}'::jsonb) || CAST(:metadata AS JSONB) + WHERE id = CAST(:opportunity_id AS UUID) + """), { + "opportunity_id": opportunity_id, + "metadata": _json({"closed_reason": "system_or_bounce_created_by_mistake", "closed_by": "cleanup_non_commercial_opportunities"}), + }) + conn.execute(text(""" + INSERT INTO opportunity_events (id, opportunity_id, event_type, from_stage, to_stage, note, payload, created_by) + VALUES (CAST(:id AS UUID), CAST(:opportunity_id AS UUID), 'cleanup_closed_non_commercial', NULL, 'LOST', :note, CAST(:payload AS JSONB), 'system') + """), { + "id": str(uuid.uuid4()), + "opportunity_id": opportunity_id, + "note": "Oportunidade fechada por parecer mensagem automática/bounce sem atividade comercial.", + "payload": _json({"reason": "system_or_bounce_created_by_mistake"}), + }) + return len(ids) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--limit", type=int, default=100) + parser.add_argument("--apply", action="store_true", help="Apply changes. Without this, only prints candidates.") + args = parser.parse_args() + candidates = find_candidates(limit=args.limit) + print(f"Found {len(candidates)} candidate(s).") + for row in candidates: + print(f"- {row.get('id')} | {row.get('title')} | {row.get('customer_name')} | {row.get('created_at')}") + if args.apply: + total = close_candidates(candidates) + print(f"Closed {total} candidate(s).") + else: + print("Dry-run only. Re-run with --apply to close candidates.") + + +if __name__ == "__main__": + main() diff --git a/scripts/cleanup_operations_noise.py b/scripts/cleanup_operations_noise.py new file mode 100755 index 0000000..fdb3762 --- /dev/null +++ b/scripts/cleanup_operations_noise.py @@ -0,0 +1,158 @@ +#!/usr/bin/env python3 +"""Clean noisy pending Operations items created from mailbox/system messages. + +Dry-run by default. With --apply, marks obvious bounce/NDR/system tasks as +``skipped`` and stores a cleanup marker in task metadata. This does not delete +messages, raw events, or Chatwoot conversations. +""" +from __future__ import annotations + +import argparse +import json + +from sqlalchemy import text + +from app.db import engine + +SQL_TERMS = [ + "postmaster", + "mailer-daemon", + "mail delivery subsystem", + "mail delivery system", + "microsoft exchange", + "office 365", + "undeliverable", + "returned mail", + "delivery status notification", + "non-delivery report", + "non delivery report", + "your message couldn't be delivered", + "your message couldnt be delivered", + "recipient wasn't found", + "recipient was not found", + "unknown to address", + "delivery has failed", + "mail delivery failed", + "remote server returned", + "550 5.1.1", + "5.1.10", + "wasn't found at", +] + +NOISE_ACTION_CODES = [ + "IGNORE_BOUNCE", + "IGNORE_SPAM", + "NO_ACTION", +] + + +def _json(value: object) -> str: + return json.dumps(value or {}, ensure_ascii=False, default=str) + + +def find_candidates(limit: int = 200) -> list[dict]: + term_params = {f"term_{idx}": f"%{term}%" for idx, term in enumerate(SQL_TERMS)} + term_sql = " OR ".join([f"noise_text ILIKE :term_{idx}" for idx, _ in enumerate(SQL_TERMS)]) + sql = text(f""" + WITH task_context AS ( + SELECT + t.id::text, + t.created_at, + t.action_code, + t.route, + t.priority, + t.status, + t.conversation_id, + t.contact_id, + t.opportunity_id::text, + COALESCE(t.note, '') AS note, + COALESCE(t.action, '') AS action, + COALESCE(re.payload->'sender'->>'name', '') AS sender_name, + COALESCE(re.payload->'sender'->>'email', '') AS sender_email, + COALESCE( + re.payload->'conversation'->'additional_attributes'->>'mail_subject', + re.payload->'content_attributes'->'email'->>'subject', + re.payload->'conversation'->'messages'->0->'content_attributes'->'email'->>'subject', + '' + ) AS subject, + COALESCE(m.clean_body, m.raw_body, re.payload->>'content', '') AS body, + lower( + COALESCE(t.action_code, '') || ' ' || COALESCE(t.route, '') || ' ' || + COALESCE(t.action, '') || ' ' || COALESCE(t.note, '') || ' ' || + COALESCE(re.payload->'sender'->>'name', '') || ' ' || + COALESCE(re.payload->'sender'->>'email', '') || ' ' || + COALESCE(re.payload->'conversation'->'additional_attributes'->>'mail_subject', '') || ' ' || + COALESCE(re.payload->'content_attributes'->'email'->>'subject', '') || ' ' || + COALESCE(re.payload->'conversation'->'messages'->0->'content_attributes'->'email'->>'subject', '') || ' ' || + COALESCE(m.clean_body, '') || ' ' || COALESCE(m.raw_body, '') || ' ' || COALESCE(re.payload->>'content', '') + ) AS noise_text + FROM tasks t + LEFT JOIN messages m ON m.id = t.message_id + LEFT JOIN raw_events re ON re.id = t.raw_event_id + WHERE t.status = 'pending' + ) + SELECT id, created_at, action_code, route, priority, status, conversation_id, contact_id, + opportunity_id, sender_name, sender_email, subject, action, note + FROM task_context + WHERE ({term_sql} OR upper(coalesce(action_code,'')) = ANY(:noise_action_codes)) + ORDER BY created_at DESC + LIMIT :limit + """) + params = dict(term_params) + params["noise_action_codes"] = NOISE_ACTION_CODES + params["limit"] = int(limit) + with engine.begin() as conn: + rows = conn.execute(sql, params).mappings().all() + return [dict(row) for row in rows] + + +def apply_cleanup(candidates: list[dict]) -> int: + if not candidates: + return 0 + ids = [row["id"] for row in candidates] + payload = _json({ + "cleanup_reason": "operations_noise_bounce_or_system_message", + "cleanup_by": "cleanup_operations_noise", + "clientflow_version": "v4.9.0", + }) + with engine.begin() as conn: + for task_id in ids: + conn.execute(text(""" + UPDATE tasks + SET status = 'skipped', + updated_at = now(), + done_at = COALESCE(done_at, now()), + done_by = 'cleanup_operations_noise', + metadata = COALESCE(metadata, '{}'::jsonb) || CAST(:payload AS JSONB) + WHERE id = CAST(:task_id AS UUID) + AND status = 'pending' + """), {"task_id": task_id, "payload": payload}) + conn.execute(text(""" + INSERT INTO task_events (task_id, event_type, payload, created_by) + VALUES (CAST(:task_id AS UUID), 'task_skipped_noise_cleanup', CAST(:payload AS JSONB), 'system') + """), {"task_id": task_id, "payload": payload}) + return len(ids) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Clean pending Operations noise created from bounces/NDRs/system messages.") + parser.add_argument("--limit", type=int, default=200) + parser.add_argument("--apply", action="store_true", help="Apply cleanup. Without this, only prints candidates.") + args = parser.parse_args() + + candidates = find_candidates(limit=args.limit) + print(f"Found {len(candidates)} candidate task(s).") + for row in candidates: + subject = (row.get("subject") or row.get("note") or "")[:90] + sender = row.get("sender_email") or row.get("sender_name") or row.get("contact_id") or "sem remetente" + print(f"- {row.get('id')} | {row.get('action_code')} | {sender} | {subject}") + + if args.apply: + total = apply_cleanup(candidates) + print(f"Marked {total} task(s) as skipped.") + else: + print("Dry-run only. Re-run with --apply to mark candidates as skipped.") + + +if __name__ == "__main__": + main() diff --git a/scripts/cleanup_stale_reconciliation_items.py b/scripts/cleanup_stale_reconciliation_items.py new file mode 100755 index 0000000..76dffa2 --- /dev/null +++ b/scripts/cleanup_stale_reconciliation_items.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +"""Ignore reconciliation items outside the intended working window. + +Dry-run by default. Use this after an external sync staged too much history. + +Examples: + PYTHONPATH=. python scripts/cleanup_stale_reconciliation_items.py --days 3 + PYTHONPATH=. python scripts/cleanup_stale_reconciliation_items.py --days 3 --apply + PYTHONPATH=. python scripts/cleanup_stale_reconciliation_items.py --source odoo --days 3 --apply +""" +from __future__ import annotations + +import argparse + +from sqlalchemy import text + +from app.db import engine +from app.reconciliation_service import ( + cleanup_reconciliation_outside_window, + ensure_reconciliation_schema, +) + + +def _preview(*, days: int, source_system: str | None, limit: int) -> dict: + ensure_reconciliation_schema() + from datetime import datetime, timedelta, timezone + + days = max(int(days or 3), 1) + cutoff = (datetime.now(timezone.utc).date() - timedelta(days=days)).isoformat() + params = {"cutoff": cutoff, "limit": int(limit)} + source_sql = "" + if source_system: + source_sql = "AND source_system = :source_system" + params["source_system"] = source_system + + with engine.begin() as conn: + rows = conn.execute(text(f""" + SELECT id::text, source_system, external_type, document_number, + customer_name, document_date, title, status + FROM reconciliation_items + WHERE status IN ('open', 'needs_review') + {source_sql} + AND document_date IS NOT NULL + AND document_date < CAST(:cutoff AS DATE) + ORDER BY document_date ASC, updated_at DESC + LIMIT :limit + """), params).mappings().all() + return {"days": days, "cutoff": cutoff, "source_system": source_system or "all", "matched": len(rows), "items": [dict(r) for r in rows]} + + +def main() -> None: + parser = argparse.ArgumentParser(description="Ignore stale reconciliation candidates outside a recent working window.") + parser.add_argument("--jasmin", action="store_true", help="same as --source jasmin") + parser.add_argument("--source", choices=["jasmin", "odoo", "packlink", "manual"], help="only clean one source system") + parser.add_argument("--days", type=int, default=3, help="keep open items from the last N days") + parser.add_argument("--limit", type=int, default=1000, help="maximum rows to inspect/update") + parser.add_argument("--apply", action="store_true", help="apply changes; otherwise dry-run") + args = parser.parse_args() + + source = "jasmin" if args.jasmin else args.source + if args.apply: + result = cleanup_reconciliation_outside_window(days=args.days, source_system=source, limit=args.limit, actor="cleanup_stale_reconciliation_items") + else: + result = _preview(days=args.days, source_system=source, limit=args.limit) + + print(f"Janela operacional: manter itens >= {result['cutoff']} ({result['days']} dias)") + print(f"Fonte: {result['source_system']}") + print(f"Encontrados para ignorar: {result['matched']}") + for row in result.get("items", [])[:50]: + print(f"- {row.get('document_date')} · {row.get('source_system')} · {row.get('external_type')} · {row.get('document_number')} · {row.get('customer_name') or ''}") + + if args.apply: + print(f"Aplicado: {result['ignored']} itens marcados como ignored") + else: + print("Dry-run. Para aplicar, repetir com --apply") + + +if __name__ == "__main__": + main() diff --git a/scripts/create_automatic_followups.py b/scripts/create_automatic_followups.py new file mode 100755 index 0000000..13f7cdf --- /dev/null +++ b/scripts/create_automatic_followups.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python3 +"""Automatic follow-up task creation is disabled in the clean architecture. + +The current design keeps LLM triage action_codes short and uses +opportunity/business events for pipeline follow-up state. +""" + + +def main() -> int: + print("Automatic legacy follow-up task creation is disabled.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/enrich_fiscal_customers.py b/scripts/enrich_fiscal_customers.py new file mode 100755 index 0000000..a9c5aa6 --- /dev/null +++ b/scripts/enrich_fiscal_customers.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 +"""Run the ClientFlow fiscal enrichment worker. + +Typical production use: + + python scripts/enrich_fiscal_customers.py --incremental --limit 100 + +The worker is idempotent: it enriches/suggests fiscal customers for open +opportunities missing local_customer_id and never creates opportunities. +""" +from __future__ import annotations + +import argparse +import json +import os +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from app.fiscal_enrichment_service import enrich_open_opportunities, ensure_fiscal_enrichment_schema + + +def main() -> int: + parser = argparse.ArgumentParser(description="Enriquecer oportunidades sem cliente fiscal") + parser.add_argument("--limit", type=int, default=100, help="Máximo de oportunidades abertas a analisar") + parser.add_argument("--no-auto-apply", action="store_true", help="Criar apenas sugestões, sem auto-associação forte") + parser.add_argument("--daily", action="store_true", help="Marcar execução como diária/batch") + parser.add_argument("--incremental", action="store_true", help="Marcar execução como incremental") + args = parser.parse_args() + + ensure_fiscal_enrichment_schema() + mode = "daily" if args.daily else "incremental" + result = enrich_open_opportunities(limit=args.limit, apply_safe=not args.no_auto_apply, mode=mode) + print(json.dumps(result, ensure_ascii=False, indent=2, default=str)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/evaluate_action_decider.py b/scripts/evaluate_action_decider.py new file mode 100755 index 0000000..5a574db --- /dev/null +++ b/scripts/evaluate_action_decider.py @@ -0,0 +1,217 @@ +#!/usr/bin/env python3 +import json +import os +import sys +import urllib.request +import urllib.error +from pathlib import Path +from typing import Any, Dict, List + +try: + import psycopg +except Exception: + psycopg = None + + +def env(name: str, default: str = "") -> str: + return os.getenv(name, default).strip() + + +API_URL = env("EVAL_API_URL", "http://127.0.0.1:8020/analyze") +DATASET = Path(env("EVAL_DATASET", "data/action_eval_cases.jsonl")) +PSQL_DATABASE_URL = env("PSQL_DATABASE_URL") +EVAL_CLEANUP = env("EVAL_CLEANUP", "true").lower() == "true" +EVAL_LIMIT = int(env("EVAL_LIMIT", "0") or "0") +EVAL_LLM_ONLY = env("EVAL_LLM_ONLY", "false").lower() == "true" + + +def load_cases() -> List[Dict[str, Any]]: + cases = [] + with DATASET.open("r", encoding="utf-8") as f: + for line in f: + line = line.strip() + if line: + cases.append(json.loads(line)) + + if EVAL_LIMIT > 0: + return cases[:EVAL_LIMIT] + + return cases + + +def post_json(url: str, payload: Dict[str, Any]) -> Dict[str, Any]: + raw = json.dumps(payload, ensure_ascii=False).encode("utf-8") + + req = urllib.request.Request( + url, + data=raw, + headers={"Content-Type": "application/json"}, + method="POST", + ) + + try: + with urllib.request.urlopen(req, timeout=90) as resp: + body = resp.read().decode("utf-8", errors="replace") + return { + "ok": 200 <= resp.status < 300, + "status": resp.status, + "json": json.loads(body) if body else {}, + "raw": body, + } + except urllib.error.HTTPError as e: + body = e.read().decode("utf-8", errors="replace") + return { + "ok": False, + "status": e.code, + "json": None, + "raw": body, + } + + +def cleanup_eval_data() -> None: + if not EVAL_CLEANUP or not PSQL_DATABASE_URL or psycopg is None: + return + + # Limpeza defensiva: algumas tabelas não têm source_system/source_event_id. + # Por isso verificamos as colunas reais antes de construir o DELETE. + tables = [ + "integration_outbox", + "tasks", + "action_runs", + "messages", + "raw_events", + ] + + def table_columns(cur, table: str) -> set[str]: + cur.execute( + """ + select column_name + from information_schema.columns + where table_name = %s + """, + (table,), + ) + return {row[0] for row in cur.fetchall()} + + with psycopg.connect(PSQL_DATABASE_URL) as conn: + with conn.cursor() as cur: + for table in tables: + cols = table_columns(cur, table) + conditions = [] + + if "source_system" in cols: + conditions.append("source_system like 'action_eval%'") + + if "conversation_id" in cols: + conditions.append("conversation_id like 'eval-%'") + + if "contact_id" in cols: + conditions.append("contact_id like 'eval-contact-%'") + + if "source_event_id" in cols: + conditions.append("source_event_id like 'eval-%'") + + if "idempotency_key" in cols: + conditions.append("idempotency_key like '%eval-%'") + + if "metadata" in cols: + conditions.append("metadata::text like '%action_eval%'") + + if not conditions: + continue + + sql = f"delete from {table} where " + " or ".join(conditions) + cur.execute(sql) + + conn.commit() + + +def main() -> int: + cases = load_cases() + + cleanup_eval_data() + + results = [] + ok_count = 0 + + print(f"Dataset: {DATASET}") + print(f"Cases: {len(cases)}") + print(f"API: {API_URL}") + print(f"EVAL_LLM_ONLY={EVAL_LLM_ONLY}") + print("---") + + for i, case in enumerate(cases, start=1): + conv_id = f"eval-{case['id']}" + payload = { + "last_customer_message": case["message"], + "previous_context": case.get("context", ""), + "source": "action_eval_llm_only" if EVAL_LLM_ONLY else "action_eval", + "conversation_id": conv_id, + "contact_id": f"eval-contact-{i}", + } + + result = post_json(API_URL, payload) + + if not result["ok"]: + got = "HTTP_ERROR" + confidence = None + provider = None + note = result["raw"][:300] + else: + data = result["json"] or {} + decision = data.get("action_decision") or {} + usage = data.get("usage") or {} + got = decision.get("action_code") or data.get("action_result", {}).get("action_code") or "UNKNOWN" + confidence = decision.get("confidence") + provider = usage.get("provider") + note = decision.get("note") + + expected = case["expected"] + passed = got == expected + ok_count += 1 if passed else 0 + + results.append({ + "id": case["id"], + "expected": expected, + "got": got, + "ok": passed, + "confidence": confidence, + "provider": provider, + "note": note, + }) + + status = "OK" if passed else "FAIL" + print(f"{status:4} {case['id']}") + print(f" expected={expected}") + print(f" got ={got}") + print(f" conf ={confidence} provider={provider}") + if not passed: + print(f" note ={note}") + print("---") + + accuracy = ok_count / len(cases) if cases else 0 + + print(f"accuracy={accuracy:.1%} ({ok_count}/{len(cases)})") + + by_expected = {} + for r in results: + item = by_expected.setdefault(r["expected"], {"ok": 0, "total": 0}) + item["total"] += 1 + item["ok"] += 1 if r["ok"] else 0 + + print("--- by expected") + for code, stats in sorted(by_expected.items()): + print(f"{code}: {stats['ok']}/{stats['total']}") + + Path("data/action_eval_last_results.json").write_text( + json.dumps(results, ensure_ascii=False, indent=2), + encoding="utf-8", + ) + + cleanup_eval_data() + + return 0 if accuracy >= 0.90 else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/extract_email_identity.py b/scripts/extract_email_identity.py new file mode 100755 index 0000000..649aea2 --- /dev/null +++ b/scripts/extract_email_identity.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 +"""Extract identity signals from an opportunity email body/signature. + +Usage: + PYTHONPATH=. python scripts/extract_email_identity.py --opportunity-id + PYTHONPATH=. python scripts/extract_email_identity.py --text-file /tmp/email.txt --email user@example.pt +""" +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +from app.email_identity_extraction_service import extract_email_identity, extract_identity_for_opportunity + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--opportunity-id") + parser.add_argument("--text-file") + parser.add_argument("--email", default="") + parser.add_argument("--subject", default="") + parser.add_argument("--no-llm", action="store_true") + parser.add_argument("--refresh", action="store_true") + args = parser.parse_args() + + if args.opportunity_id: + result = extract_identity_for_opportunity(args.opportunity_id, refresh=args.refresh, use_llm=not args.no_llm) + elif args.text_file: + result = extract_email_identity( + Path(args.text_file).read_text(encoding="utf-8"), + email=args.email, + subject=args.subject, + use_llm=not args.no_llm, + ) + else: + parser.error("use --opportunity-id or --text-file") + + print(json.dumps(result or {}, ensure_ascii=False, indent=2, default=str)) + + +if __name__ == "__main__": + main() diff --git a/scripts/ignore_manually_cleaned_outbox.py b/scripts/ignore_manually_cleaned_outbox.py new file mode 100755 index 0000000..9c0b2e6 --- /dev/null +++ b/scripts/ignore_manually_cleaned_outbox.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +"""Mark manually cleaned outbox failures as ignored. + +Use when old Jasmin/Packlink test or duplicate failures were already handled +outside the worker and should no longer pollute /operations. +""" +from __future__ import annotations + +import argparse +from sqlalchemy import text + +from app.db import engine + + +MATCH_SQL = """ + status = 'failed' + AND ( + COALESCE(last_error, '') ILIKE '%limpo manualmente%' + OR COALESCE(last_error, '') ILIKE '%resolvido manualmente%' + ) +""" + + +def main() -> int: + parser = argparse.ArgumentParser(description="Ignore outbox failures already resolved manually.") + parser.add_argument("--dry-run", action="store_true", help="Only show matching rows; do not update.") + parser.add_argument("--limit", type=int, default=200, help="Maximum rows to inspect/update.") + args = parser.parse_args() + + with engine.begin() as conn: + rows = conn.execute(text(f""" + SELECT id::text, target_system, action_type, status, last_error, created_at + FROM integration_outbox + WHERE {MATCH_SQL} + ORDER BY created_at DESC + LIMIT :limit + """), {"limit": args.limit}).mappings().all() + + print(f"Matched {len(rows)} manually cleaned failed outbox item(s).") + for row in rows: + print(f"- {row['id']} {row['target_system']}.{row['action_type']} :: {row['last_error']}") + + if args.dry_run or not rows: + print("Dry-run/no-op; no rows updated.") + return 0 + + ids = [row["id"] for row in rows] + conn.execute(text(""" + UPDATE integration_outbox + SET status = 'ignored', + last_error = COALESCE(NULLIF(last_error, ''), 'Ignorado por limpeza operacional manual.'), + updated_at = now() + WHERE id::text = ANY(:ids) + """), {"ids": ids}) + print(f"Updated {len(ids)} item(s) to ignored.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/inspect_customer_tax_id_conflict.py b/scripts/inspect_customer_tax_id_conflict.py new file mode 100644 index 0000000..22b375c --- /dev/null +++ b/scripts/inspect_customer_tax_id_conflict.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +"""Inspect customers/opportunities involved in a duplicate NIF conflict.""" +from __future__ import annotations + +import argparse +import json +from sqlalchemy import text + +from app.db import engine +from app.commercial_service import normalize_tax_id + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--tax-id", required=True) + args = parser.parse_args() + tax_id = normalize_tax_id(args.tax_id) + with engine.begin() as conn: + customers = conn.execute(text(""" + SELECT c.id::text, c.name, c.tax_id, c.email, c.phone, c.street_name, c.postal_zone, c.city_name, + c.country, c.created_at, c.updated_at, + (SELECT count(*) FROM opportunities o WHERE o.local_customer_id = c.id)::int AS opportunities, + (SELECT count(*) FROM commercial_documents cd WHERE cd.customer_id = c.id)::int AS documents + FROM customers c + WHERE c.tax_id = :tax_id OR c.name ILIKE '%' || :tax_id || '%' + ORDER BY c.tax_id NULLS LAST, c.updated_at DESC + """), {"tax_id": tax_id}).mappings().all() + print(json.dumps({"tax_id": tax_id, "customers": [dict(c) for c in customers]}, ensure_ascii=False, indent=2, default=str)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/inspect_jasmin_candidates_for_opportunity.py b/scripts/inspect_jasmin_candidates_for_opportunity.py new file mode 100755 index 0000000..03fe042 --- /dev/null +++ b/scripts/inspect_jasmin_candidates_for_opportunity.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 +"""Inspect Jasmin document candidates for an opportunity. + +Optionally sync recent Jasmin documents first, then prints open/valid candidates first +and ignored closed/completed/cancelled documents afterwards. +""" +from __future__ import annotations + +import argparse +import asyncio +import json +from decimal import Decimal +from typing import Any + + +def _json_default(value: Any) -> str: + if isinstance(value, Decimal): + return str(value) + return str(value) + + +async def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--opportunity-id", required=True) + parser.add_argument("--limit", type=int, default=20) + parser.add_argument("--sync", action="store_true", help="Sync recent Jasmin reconciliation candidates before inspection") + parser.add_argument("--days", type=int, default=30) + args = parser.parse_args() + + if args.sync: + from app.external_reconciliation_sync import sync_jasmin_reconciliation_candidates + sync_result = await sync_jasmin_reconciliation_candidates(limit=100, days=args.days) + print(json.dumps({"sync_jasmin": sync_result}, ensure_ascii=False, indent=2, default=_json_default)) + + from app.jasmin_backfill_service import find_jasmin_document_candidates_for_opportunity + candidates = find_jasmin_document_candidates_for_opportunity(args.opportunity_id, limit=args.limit) + print(json.dumps({"opportunity_id": args.opportunity_id, "candidates": candidates}, ensure_ascii=False, indent=2, default=_json_default)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(asyncio.run(main())) diff --git a/scripts/install_systemd_timers.sh b/scripts/install_systemd_timers.sh new file mode 100755 index 0000000..334ac22 --- /dev/null +++ b/scripts/install_systemd_timers.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="${1:-/mnt/ssd/home/plx/clientflow_backend}" +SYSTEMD_DIR="/etc/systemd/system" + +if [[ ! -d "$ROOT_DIR" ]]; then + echo "Project directory not found: $ROOT_DIR" >&2 + exit 1 +fi + +cd "$ROOT_DIR" + +sudo cp deploy/systemd/clientflow-outbox-jasmin.service "$SYSTEMD_DIR/" +sudo cp deploy/systemd/clientflow-outbox-jasmin.timer "$SYSTEMD_DIR/" + +if [[ "${ENABLE_FISCAL_ENRICHMENT_TIMER:-false}" == "true" ]]; then + sudo cp deploy/systemd/clientflow-fiscal-enrichment.service "$SYSTEMD_DIR/" + sudo cp deploy/systemd/clientflow-fiscal-enrichment.timer "$SYSTEMD_DIR/" +fi + +if [[ "${ENABLE_PACKLINK_TIMER:-false}" == "true" ]]; then + sudo cp deploy/systemd/clientflow-outbox-packlink.service "$SYSTEMD_DIR/" + sudo cp deploy/systemd/clientflow-outbox-packlink.timer "$SYSTEMD_DIR/" +fi + +sudo systemctl daemon-reload +sudo systemctl enable --now clientflow-outbox-jasmin.timer + +if [[ "${ENABLE_FISCAL_ENRICHMENT_TIMER:-false}" == "true" ]]; then + sudo systemctl enable --now clientflow-fiscal-enrichment.timer +fi + +if [[ "${ENABLE_PACKLINK_TIMER:-false}" == "true" ]]; then + sudo systemctl enable --now clientflow-outbox-packlink.timer +fi + +systemctl list-timers | grep clientflow || true diff --git a/scripts/prepare_task.py b/scripts/prepare_task.py new file mode 100755 index 0000000..731ebc4 --- /dev/null +++ b/scripts/prepare_task.py @@ -0,0 +1,512 @@ +#!/usr/bin/env python3 +import argparse +import json +import os +import re +import urllib.request +import urllib.error +from typing import Any, Dict, List, Optional, Tuple + +import psycopg + + +def env(name: str, default: str = "") -> str: + return os.getenv(name, default).strip() + + +PSQL_DATABASE_URL = env("PSQL_DATABASE_URL") +OPENROUTER_API_KEY = env("OPENROUTER_API_KEY") +OPENROUTER_URL = env("OPENROUTER_URL", "https://openrouter.ai/api/v1/chat/completions") +OPENROUTER_MODEL = env("EXTRACTION_MODEL", env("OPENROUTER_MODEL", "qwen/qwen3-30b-a3b")) + + +def extract_first_json_object(raw: str) -> str: + s = str(raw or "").strip() + + if s.startswith("```"): + lines = s.splitlines() + if lines and lines[0].strip().startswith("```"): + lines = lines[1:] + if lines and lines[-1].strip().startswith("```"): + lines = lines[:-1] + s = "\n".join(lines).strip() + + start = s.find("{") + if start == -1: + raise ValueError(f"no JSON object found: {s[:300]}") + + in_string = False + escaped = False + depth = 0 + + for i in range(start, len(s)): + ch = s[i] + + if escaped: + escaped = False + continue + + if ch == "\\": + escaped = True + continue + + if ch == '"': + in_string = not in_string + continue + + if in_string: + continue + + if ch == "{": + depth += 1 + elif ch == "}": + depth -= 1 + if depth == 0: + return s[start:i + 1] + + raise ValueError(f"incomplete JSON object: {s[:500]}") + + +def parse_json(raw: str) -> Dict[str, Any]: + try: + return json.loads(raw) + except Exception: + return json.loads(extract_first_json_object(raw)) + + +def strip_html(text: str) -> str: + text = re.sub(r"", "\n", text or "", flags=re.I) + text = re.sub(r"", "\n", text, flags=re.I) + text = re.sub(r"<[^>]+>", " ", text) + text = text.replace(" ", " ") + text = text.replace("&", "&") + text = text.replace(""", '"') + text = text.replace("'", "'") + return re.sub(r"[ \t]+", " ", text).strip() + + +def remove_quoted_text(text: str) -> str: + if not text: + return "" + + markers = [ + "\nÀs ", + "\nEm ", + "\nOn ", + "\n-----Original Message-----", + "\nDe:", + "\nFrom:", + ] + + cut = len(text) + for marker in markers: + idx = text.find(marker) + if idx != -1: + cut = min(cut, idx) + + lines = [] + for line in text[:cut].splitlines(): + if line.strip().startswith(">"): + continue + lines.append(line) + + return "\n".join(lines).strip() + + +def extract_message_content(payload: Dict[str, Any]) -> Optional[Dict[str, Any]]: + msg = payload.get("message") or payload.get("messages") or {} + + if isinstance(msg, list): + msg = msg[0] if msg else {} + + if not isinstance(msg, dict): + return None + + content = msg.get("content") or payload.get("content") or "" + message_type = str(msg.get("message_type") or payload.get("message_type") or "").lower() + private = bool(msg.get("private") or payload.get("private")) + + if not content: + return None + + return { + "content": remove_quoted_text(strip_html(str(content))), + "message_type": message_type, + "private": private, + "id": str(msg.get("id") or payload.get("id") or ""), + } + + +def fetch_task(conn, task_id: Optional[str], conversation_id: Optional[str]) -> Dict[str, Any]: + if task_id: + sql = """ + select id::text, conversation_id, contact_id, action_code, route, status, action, note + from tasks + where id = %s + limit 1 + """ + params = (task_id,) + else: + sql = """ + select id::text, conversation_id, contact_id, action_code, route, status, action, note + from tasks + where conversation_id = %s + order by created_at desc + limit 1 + """ + params = (conversation_id,) + + with conn.cursor(row_factory=psycopg.rows.dict_row) as cur: + cur.execute(sql, params) + row = cur.fetchone() + + if not row: + raise SystemExit("ERRO: task não encontrada.") + + return dict(row) + + +def fetch_conversation_messages(conn, conversation_id: str, limit: int = 20) -> List[Dict[str, Any]]: + with conn.cursor(row_factory=psycopg.rows.dict_row) as cur: + cur.execute( + """ + select created_at, payload + from raw_events + where source_system = 'chatwoot' + and conversation_id = %s + order by created_at asc + limit %s + """, + (conversation_id, limit), + ) + rows = cur.fetchall() + + messages = [] + for row in rows: + payload = row.get("payload") or {} + if isinstance(payload, str): + try: + payload = json.loads(payload) + except Exception: + payload = {} + + msg = extract_message_content(payload) + if not msg: + continue + + # Para extração, manter incoming e outgoing públicos, ignorar notas privadas. + if msg["private"]: + continue + + messages.append({ + "created_at": str(row["created_at"]), + **msg, + }) + + return messages + + +def build_prompt(task: Dict[str, Any], messages: List[Dict[str, Any]], prep_type: str) -> List[Dict[str, str]]: + conversation_text = "\n\n".join( + f"[{m['created_at']}] {m.get('message_type') or 'message'}:\n{m['content']}" + for m in messages + if m.get("content") + ) + + if prep_type == "proforma": + objective = """ +Objetivo: preparar dados para emitir fatura pró-forma. +Extrai apenas dados relevantes para faturação/proforma: +cliente, empresa, email, telefone, NIF, morada fiscal, produto, quantidade, preço, condições comerciais e dados em falta para emitir a pró-forma. + +Para prep_type=proforma: +- NÃO peças morada de entrega, destinatário ou telefone para transportadora, exceto se a conversa indicar que são necessários para a pró-forma. +- Morada de entrega/recolha pertence à fase de envio/recolha, não à fase de pró-forma. +- Se já houver NIF, morada fiscal, nome/empresa de faturação, email, produto e preço, missing_fields deve ser []. +""" + elif prep_type == "shipment": + objective = """ +Objetivo: preparar envio. +Extrai apenas dados relevantes para logística de entrega: +morada de entrega, contacto no local, telefone, produto/equipamento, quantidade, instruções, estado do pagamento e dados em falta. + +Para prep_type=shipment: +- Usa shipment.delivery_address para a morada de entrega. +- Usa shipment.recipient_name e shipment.recipient_phone para contacto da entrega. +- Os campos obrigatórios são morada de entrega, contacto, telefone, produto/equipamento, quantidade e estado do pagamento. +- NÃO coloques customer.tax_id, billing.tax_id, billing_address ou customer.email em missing_fields, exceto se forem explicitamente necessários para a transportadora. +- Se já existir morada de entrega, contacto e telefone, não peças esses dados novamente. +- Se faltar produto, quantidade ou comprovativo/estado de pagamento, pede apenas esses dados. +- A suggested_reply deve falar em envio/entrega, nunca em recolha. +""" + elif prep_type == "pickup": + objective = """ +Objetivo: preparar recolha. +Extrai apenas dados relevantes para recolha ou assistência logística: +morada de recolha, contacto no local, telefone, produto/equipamento a recolher, motivo/instruções, data preferida e dados em falta. + +Para prep_type=pickup: +- Usa shipment.pickup_address para a morada de recolha. +- Usa shipment.recipient_name e shipment.recipient_phone para a pessoa de contacto da recolha. +- Se a conversa só tiver uma morada e o objetivo é recolha, coloca essa morada em shipment.pickup_address, não em shipment.delivery_address. +- NÃO coloques sale.total_estimate, customer.tax_id, billing.tax_id, billing_address ou customer.email em missing_fields. +- Os campos importantes são pickup_address, recipient_name, recipient_phone, produto/equipamento, motivo/instruções da recolha. +- payment.status só é obrigatório se a tarefa for claramente sobre pagamento ou envio após pagamento. +- A suggested_reply deve falar em recolha, nunca em envio. +""" + else: + objective = """ +Objetivo: preparar execução operacional da tarefa. +Extrai dados úteis para a ação, dados em falta e resposta sugerida. +""" + + system = f""" +És o ClientFlow Sales Assistant. +A tua função é extrair dados operacionais de conversas B2B para ajudar a preparar pró-forma, fatura, envio ou recolha. + +Regras: +- Devolve apenas JSON puro. +- Não inventes dados. +- Se um dado não existir, usa null. +- Ignora texto citado antigo, dados da BLIF, IBANs e assinaturas da BLIF. +- Não associes IBAN ao cliente. +- Distingue morada fiscal de morada de entrega/recolha. +- Usa evidências curtas. +- Se faltar dado necessário, coloca em missing_fields. +- suggested_reply deve ser uma resposta curta em português para pedir dados em falta ou indicar o próximo passo. +- Não digas que a pró-forma/fatura/envio já foi emitida, enviada, agendada ou concluída. +- O assistente apenas prepara dados para revisão; usa linguagem como "vamos preparar", "podemos avançar", "dados suficientes para preparar". +""" + + user = f""" +{objective} + +Tarefa: +action_code: {task.get('action_code')} +route: {task.get('route')} +action: {task.get('action')} +note: {task.get('note')} + +Conversa: +{conversation_text} + +Schema obrigatório: +{{ + "customer": {{ + "name": null, + "company": null, + "email": null, + "phone": null, + "tax_id": null + }}, + "billing": {{ + "billing_name": null, + "tax_id": null, + "billing_address": null, + "billing_email": null + }}, + "sale": {{ + "products": [ + {{ + "name": null, + "description": null, + "quantity": null, + "unit_price": null, + "currency": "EUR" + }} + ], + "total_estimate": null, + "commercial_terms": null + }}, + "payment": {{ + "status": null, + "proof_mentioned": false + }}, + "shipment": {{ + "delivery_address": null, + "pickup_address": null, + "recipient_name": null, + "recipient_phone": null, + "instructions": null + }}, + "missing_fields": [], + "suggested_reply": "", + "confidence": 0.0, + "evidence": [] +}} +""" + + return [ + {"role": "system", "content": system}, + {"role": "user", "content": user}, + ] + + +def call_openrouter(messages: List[Dict[str, str]]) -> Tuple[Dict[str, Any], Dict[str, Any], str]: + body = { + "model": OPENROUTER_MODEL, + "messages": messages, + "temperature": 0, + } + + req = urllib.request.Request( + OPENROUTER_URL, + data=json.dumps(body, ensure_ascii=False).encode("utf-8"), + headers={ + "Authorization": f"Bearer {OPENROUTER_API_KEY}", + "Content-Type": "application/json", + "HTTP-Referer": "https://clientflow.blif.pt", + "X-Title": "ClientFlow", + }, + method="POST", + ) + + try: + with urllib.request.urlopen(req, timeout=120) as resp: + raw = resp.read().decode("utf-8", errors="replace") + parsed = json.loads(raw) + except urllib.error.HTTPError as e: + raw = e.read().decode("utf-8", errors="replace") + raise RuntimeError(f"OpenRouter HTTP {e.code}: {raw[:1000]}") + + content = parsed["choices"][0]["message"].get("content") or "{}" + extracted = parse_json(content) + return extracted, parsed, content + + +def save_preparation(conn, task: Dict[str, Any], prep_type: str, extracted: Dict[str, Any], raw_response: Dict[str, Any]) -> str: + usage = raw_response.get("usage") or {} + model = raw_response.get("model") or OPENROUTER_MODEL + provider = raw_response.get("provider") or raw_response.get("provider_name") + + missing_fields = extracted.get("missing_fields") or [] + suggested_reply = extracted.get("suggested_reply") or "" + confidence = extracted.get("confidence") + + with conn.cursor() as cur: + cur.execute( + """ + insert into task_preparations ( + task_id, + conversation_id, + contact_id, + prep_type, + status, + extracted_data, + missing_fields, + suggested_reply, + confidence, + model, + provider, + total_tokens, + cost, + raw_response + ) + values ( + %s, %s, %s, %s, 'draft', + %s::jsonb, + %s::jsonb, + %s, + %s, + %s, + %s, + %s, + %s, + %s::jsonb + ) + returning id::text + """, + ( + task["id"], + task["conversation_id"], + task.get("contact_id"), + prep_type, + json.dumps(extracted, ensure_ascii=False), + json.dumps(missing_fields, ensure_ascii=False), + suggested_reply, + confidence, + model, + provider, + int(usage.get("total_tokens") or 0), + usage.get("cost") or 0, + json.dumps(raw_response, ensure_ascii=False), + ), + ) + prep_id = cur.fetchone()[0] + + conn.commit() + return prep_id + + +def run_preparation( + *, + task_id: Optional[str] = None, + conversation_id: Optional[str] = None, + prep_type: str, + database_url: Optional[str] = None, +) -> Dict[str, Any]: + db_url = database_url or PSQL_DATABASE_URL + if not task_id and not conversation_id: + raise ValueError("Usa task_id ou conversation_id.") + if not db_url: + raise RuntimeError("PSQL_DATABASE_URL/DATABASE_URL não definida.") + if not OPENROUTER_API_KEY: + raise RuntimeError("OPENROUTER_API_KEY não definida.") + + with psycopg.connect(db_url) as conn: + task = fetch_task(conn, task_id, conversation_id) + messages = fetch_conversation_messages(conn, task["conversation_id"]) + if not messages: + raise RuntimeError("Não encontrei mensagens públicas da conversa.") + prompt = build_prompt(task, messages, prep_type) + extracted, raw_response, raw_content = call_openrouter(prompt) + prep_id = save_preparation(conn, task, prep_type, extracted, raw_response) + + return { + "preparation_id": prep_id, + "task_id": task["id"], + "conversation_id": task["conversation_id"], + "prep_type": prep_type, + "extracted": extracted, + } + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--task-id") + parser.add_argument("--conversation-id") + parser.add_argument("--type", choices=["proforma", "shipment", "pickup", "generic"], required=True) + args = parser.parse_args() + + if not args.task_id and not args.conversation_id: + raise SystemExit("Usa --task-id ou --conversation-id.") + + if not PSQL_DATABASE_URL: + raise SystemExit("PSQL_DATABASE_URL não definida.") + + if not OPENROUTER_API_KEY: + raise SystemExit("OPENROUTER_API_KEY não definida.") + + with psycopg.connect(PSQL_DATABASE_URL) as conn: + task = fetch_task(conn, args.task_id, args.conversation_id) + messages = fetch_conversation_messages(conn, task["conversation_id"]) + + if not messages: + raise SystemExit("ERRO: não encontrei mensagens públicas da conversa.") + + prompt = build_prompt(task, messages, args.type) + extracted, raw_response, raw_content = call_openrouter(prompt) + prep_id = save_preparation(conn, task, args.type, extracted, raw_response) + + print(f"OK: preparation_id={prep_id}") + print(f"task_id={task['id']}") + print(f"conversation_id={task['conversation_id']}") + print(f"prep_type={args.type}") + print("--- extracted") + print(json.dumps(extracted, ensure_ascii=False, indent=2)) + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/process_outbox.py b/scripts/process_outbox.py new file mode 100644 index 0000000..4219587 --- /dev/null +++ b/scripts/process_outbox.py @@ -0,0 +1,318 @@ +import asyncio +import os +import sys +from pathlib import Path +from typing import Any, Dict, Literal + +import httpx + + +PROJECT_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(PROJECT_ROOT)) +os.chdir(PROJECT_ROOT) + +from app.config import settings +from app.integration_outbox_service import ( + claim_pending_outbox, + recover_stale_processing_outbox, + mark_outbox_blocked, + mark_outbox_dry_run, + mark_outbox_failed, + mark_outbox_sent, +) + + +Outcome = Literal["processed", "skipped"] + + +def env_bool(name: str, default: bool = False) -> bool: + fallback = "true" if default else "false" + value = os.getenv(name, fallback).strip().lower() + return value in {"true", "1", "yes", "on"} + + +def is_dry_run() -> bool: + return env_bool("OUTBOX_DRY_RUN", True) + + +def integration_enabled(target_system: str) -> bool: + key = f"{str(target_system or '').upper()}_OUTBOX_ENABLED" + return env_bool(key, False) + + +def build_chatwoot_note(payload: Dict[str, Any]) -> str: + event_type = payload.get("event_type", "") + action = payload.get("action", "") + note = payload.get("note", "") + conversation_id = payload.get("conversation_id", "") + + return f"""🤖 ClientFlow + +Evento: +{event_type} + +Ação: +{action} + +Nota: +{note} + +Conversa: +{conversation_id} +""" + + +async def process_chatwoot_add_private_note(item: Dict[str, Any]) -> Outcome: + payload = item.get("payload") or {} + conversation_id = payload.get("conversation_id") + + if not conversation_id: + raise RuntimeError("conversation_id em falta no payload") + + if is_dry_run(): + message = f"DRY-RUN chatwoot.add_private_note conversation_id={conversation_id}" + print(message) + mark_outbox_dry_run(item["id"], message) + return "processed" + + if not settings.chatwoot_write_enabled: + raise RuntimeError("CHATWOOT_WRITE_ENABLED=false") + + if not settings.chatwoot_base_url or not settings.chatwoot_account_id or not settings.chatwoot_api_token: + raise RuntimeError("Configuração Chatwoot incompleta") + + url = ( + settings.chatwoot_base_url.rstrip("/") + + f"/api/v1/accounts/{settings.chatwoot_account_id}" + + f"/conversations/{conversation_id}/messages" + ) + + body = { + "content": build_chatwoot_note(payload), + "message_type": "outgoing", + "private": True, + "content_type": "text", + "content_attributes": {}, + } + + headers = { + "Content-Type": "application/json", + "api_access_token": settings.chatwoot_api_token, + } + + async with httpx.AsyncClient(timeout=30) as client: + response = await client.post(url, headers=headers, json=body) + + if response.status_code >= 400: + raise RuntimeError(f"Chatwoot error {response.status_code}: {response.text}") + + mark_outbox_sent(item["id"]) + return "processed" + + +async def process_mautic_add_tag(item: Dict[str, Any]) -> Outcome: + payload = item.get("payload") or {} + + if is_dry_run(): + message = ( + "DRY-RUN mautic.add_tag " + f"conversation_id={payload.get('conversation_id')} " + f"tag={payload.get('tag')}" + ) + print(message) + mark_outbox_dry_run(item["id"], message) + return "processed" + + from app.mautic_client import add_tag_from_outbox_payload + + add_tag_from_outbox_payload(payload) + mark_outbox_sent(item["id"]) + return "processed" + + +async def process_mautic_remove_tag(item: Dict[str, Any]) -> Outcome: + payload = item.get("payload") or {} + + if is_dry_run(): + message = ( + "DRY-RUN mautic.remove_tag " + f"conversation_id={payload.get('conversation_id')} " + f"tag={payload.get('tag')}" + ) + print(message) + mark_outbox_dry_run(item["id"], message) + return "processed" + + from app.mautic_client import remove_tag_from_outbox_payload + + remove_tag_from_outbox_payload(payload) + mark_outbox_sent(item["id"]) + return "processed" + + +async def process_packlink_create_shipment(item: Dict[str, Any]) -> Outcome: + payload = item.get("payload") or {} + opportunity_id = payload.get("opportunity_id") + + if is_dry_run(): + message = f"DRY-RUN packlink.create_shipment opportunity_id={opportunity_id}" + print(message) + mark_outbox_dry_run(item["id"], message) + return "processed" + + if not settings.packlink_enabled: + raise RuntimeError("PACKLINK_ENABLED=false") + + from app.packlink_service import create_shipment_from_outbox_payload + + result = await create_shipment_from_outbox_payload(payload) + print(f"Packlink shipment created reference={result.get('reference')}") + mark_outbox_sent(item["id"]) + return "processed" + + + + +async def process_jasmin_create_quotation(item: Dict[str, Any]) -> Outcome: + payload = item.get("payload") or {} + opportunity_id = payload.get("opportunity_id") + + if is_dry_run(): + message = f"DRY-RUN jasmin.create_quotation opportunity_id={opportunity_id}" + print(message) + mark_outbox_dry_run(item["id"], message) + return "processed" + + if not settings.jasmin_enabled: + raise RuntimeError("JASMIN_ENABLED=false") + + from app.jasmin_service import process_create_quotation_outbox + + result = await process_create_quotation_outbox(payload) + print(f"Jasmin quotation created id={result.get('quotation_id')}") + mark_outbox_sent(item["id"]) + return "processed" + + +async def process_jasmin_convert_invoice(item: Dict[str, Any]) -> Outcome: + payload = item.get("payload") or {} + opportunity_id = payload.get("opportunity_id") + + if is_dry_run(): + message = f"DRY-RUN jasmin.convert_quotation_to_invoice opportunity_id={opportunity_id}" + print(message) + mark_outbox_dry_run(item["id"], message) + return "processed" + + if not settings.jasmin_enabled: + raise RuntimeError("JASMIN_ENABLED=false") + + from app.jasmin_service import process_convert_invoice_outbox + + result = await process_convert_invoice_outbox(payload) + print(f"Jasmin invoice created id={result.get('invoice_id')}") + mark_outbox_sent(item["id"]) + return "processed" + +async def process_item(item: Dict[str, Any]) -> Outcome: + target_system = item.get("target_system") + action_type = item.get("action_type") + + print(f"Processing {item['id']} {target_system}.{action_type}") + + if not integration_enabled(target_system): + message = f"Integração desativada: {target_system}.{action_type}. Ative {str(target_system or '').upper()}_OUTBOX_ENABLED=true para processar." + print(f"BLOCKED {message}") + mark_outbox_blocked(item["id"], message) + return "skipped" + + if target_system == "chatwoot" and action_type == "add_private_note": + return await process_chatwoot_add_private_note(item) + + if target_system == ("t" + "wenty"): + print(f"SKIP legacy external CRM outbox item {item.get('id')}: integração removida") + return "skipped" + + if target_system == "mautic" and action_type == "add_tag": + return await process_mautic_add_tag(item) + + if target_system == "mautic" and action_type == "remove_tag": + return await process_mautic_remove_tag(item) + + if target_system == "packlink" and action_type == "create_shipment": + return await process_packlink_create_shipment(item) + + if target_system == "jasmin" and action_type == "create_quotation": + return await process_jasmin_create_quotation(item) + + if target_system == "jasmin" and action_type == "convert_quotation_to_invoice": + return await process_jasmin_convert_invoice(item) + + if is_dry_run(): + message = f"DRY-RUN unsupported-now {target_system}.{action_type}" + print(message) + mark_outbox_dry_run(item["id"], message) + return "processed" + + raise RuntimeError(f"Handler não implementado: {target_system}.{action_type}") + + +async def main() -> int: + + # Nota: as notas privadas do Chatwoot podem estar desligadas sem bloquear + # outras integrações como Packlink, Mautic ou Jasmin. A decisão de processar + # cada target_system fica em integration_enabled() e no handler específico. + if os.getenv("CLIENTFLOW_DISABLE_CHATWOOT_PRIVATE_NOTES", "true").lower() in {"1", "true", "yes", "sim"}: + print("ClientFlow Chatwoot private notes disabled by env; non-Chatwoot outbox will still run.") + + limit = int(os.getenv("OUTBOX_LIMIT", "50")) + target_system = os.getenv("OUTBOX_TARGET_SYSTEM", "").strip() or None + + worker_id = os.getenv("OUTBOX_WORKER_ID", f"process_outbox:{os.getpid()}") + + if env_bool("OUTBOX_RECOVER_STALE_BEFORE_PROCESS", True): + recovered = recover_stale_processing_outbox( + mode=os.getenv("OUTBOX_STALE_RECOVERY_MODE", "manual_only"), + actor=worker_id, + ) + if recovered: + print(f"Recovered stale processing outbox items: {len(recovered)}") + + items = claim_pending_outbox( + limit=limit, + target_system=target_system, + lock_owner=worker_id, + ) + + print(f"Claimed outbox items: {len(items)}") + print(f"OUTBOX_WORKER_ID={worker_id}") + print(f"OUTBOX_DRY_RUN={is_dry_run()}") + print(f"OUTBOX_TARGET_SYSTEM={target_system or 'all'}") + + processed = 0 + skipped = 0 + failed = 0 + + for item in items: + try: + outcome = await process_item(item) + + if outcome == "processed": + processed += 1 + else: + skipped += 1 + + except Exception as exc: + failed += 1 + print(f"FAILED {item.get('id')}: {exc}") + mark_outbox_failed(item["id"], str(exc)) + + print(f"Processed: {processed}") + print(f"Skipped: {skipped}") + print(f"Failed: {failed}") + + return 0 if failed == 0 else 1 + + +if __name__ == "__main__": + raise SystemExit(asyncio.run(main())) diff --git a/scripts/recover_stale_outbox.py b/scripts/recover_stale_outbox.py new file mode 100755 index 0000000..e29bd18 --- /dev/null +++ b/scripts/recover_stale_outbox.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 +"""Recover or expose stale outbox processing rows. + +Usage examples: + OUTBOX_STALE_RECOVERY_MODE=manual_only python scripts/recover_stale_outbox.py + OUTBOX_STALE_RECOVERY_MODE=retry_pending python scripts/recover_stale_outbox.py +""" +from __future__ import annotations + +import os +import sys +from pathlib import Path + +PROJECT_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(PROJECT_ROOT)) +os.chdir(PROJECT_ROOT) + +from app.integration_outbox_service import recover_stale_processing_outbox, outbox_stale_minutes + + +def main() -> int: + mode = os.getenv("OUTBOX_STALE_RECOVERY_MODE", "manual_only") + actor = os.getenv("OUTBOX_WORKER_ID", f"recover_stale_outbox:{os.getpid()}") + limit = int(os.getenv("OUTBOX_STALE_RECOVERY_LIMIT", "100")) + stale_minutes = outbox_stale_minutes() + recovered = recover_stale_processing_outbox( + mode=mode, + actor=actor, + limit=limit, + stale_minutes=stale_minutes, + ) + print(f"Mode: {mode}") + print(f"Stale threshold minutes: {stale_minutes}") + print(f"Recovered/exposed stale items: {len(recovered)}") + for item in recovered: + print(f"- {item.get('id')} {item.get('target_system')}.{item.get('action_type')} -> {item.get('status')}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/reopen_chatwoot_review_tasks.py b/scripts/reopen_chatwoot_review_tasks.py new file mode 100755 index 0000000..48548b8 --- /dev/null +++ b/scripts/reopen_chatwoot_review_tasks.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +"""Reabre tasks Chatwoot que foram classificadas como revisão/remoção mas ficaram skipped. + +Uso seguro: + PYTHONPATH=. python scripts/reopen_chatwoot_review_tasks.py --dry-run + PYTHONPATH=. python scripts/reopen_chatwoot_review_tasks.py --days 7 + +Por defeito só olha para os últimos 7 dias e não toca em spam/NO_ACTION. +""" +from __future__ import annotations + +import argparse +import json +from sqlalchemy import text + +from app.db import engine + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--days", type=int, default=7, help="Janela de dias a corrigir") + parser.add_argument("--dry-run", action="store_true", help="Mostra o que faria sem alterar") + args = parser.parse_args() + + params = {"days": int(args.days)} + select_sql = text(""" + SELECT id::text, created_at, action_code, route, action, status, conversation_id, contact_id + FROM tasks + WHERE source_system = 'chatwoot' + AND status = 'skipped' + AND action_code IN ('REVIEW_MANUALLY', 'REMOVE_FROM_LIST') + AND created_at >= now() - (:days * interval '1 day') + ORDER BY created_at DESC + """) + + with engine.begin() as conn: + rows = conn.execute(select_sql, params).mappings().all() + + print(f"Encontradas {len(rows)} task(s) Chatwoot a reabrir.") + for row in rows[:50]: + print(f"- {row['created_at']} {row['action_code']} conversa={row['conversation_id']} task={row['id']}") + + if args.dry_run or not rows: + print("Dry-run: nenhuma alteração aplicada." if args.dry_run else "Nada para alterar.") + return 0 + + ids = [row["id"] for row in rows] + update_sql = text(""" + UPDATE tasks + SET status = 'pending', + priority = CASE WHEN action_code = 'REVIEW_MANUALLY' THEN 'alta' ELSE COALESCE(priority, 'normal') END, + route = CASE WHEN action_code = 'REMOVE_FROM_LIST' THEN 'marketing' ELSE route END, + updated_at = now(), + metadata = COALESCE(metadata, '{}'::jsonb) || CAST(:patch AS JSONB) + WHERE id = ANY(CAST(:ids AS uuid[])) + """) + + patch = json.dumps({ + "v46_reopened": True, + "v46_reason": "REVIEW_MANUALLY/REMOVE_FROM_LIST devem gerar trabalho humano pendente", + }, ensure_ascii=False) + + with engine.begin() as conn: + conn.execute(update_sql, {"ids": ids, "patch": patch}) + + print(f"Reabertas {len(ids)} task(s) como pending.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/repair_odoo_sale_order_customer_names.py b/scripts/repair_odoo_sale_order_customer_names.py new file mode 100644 index 0000000..811d4a3 --- /dev/null +++ b/scripts/repair_odoo_sale_order_customer_names.py @@ -0,0 +1,89 @@ +"""Repair Odoo sale-order references accidentally stored as customer names. + +v4.9.25.3 fixes the source of the issue. This script repairs rows already +created by older v4.9.25.x builds where customers.name became S00xxx although +metadata.raw_customer_payload.partner_name contains the real fiscal customer. + +Dry-run by default. Use --apply to update rows. +""" +from __future__ import annotations + +import argparse +from typing import Any, Dict, List + +from sqlalchemy import text + +from app.db import engine + + +def _partner_name_from_metadata(metadata: Any) -> str: + if not isinstance(metadata, dict): + return "" + raw = metadata.get("raw_customer_payload") + if not isinstance(raw, dict): + return "" + partner_name = str(raw.get("partner_name") or "").strip() + if partner_name: + return partner_name + partner_id = raw.get("partner_id") + if isinstance(partner_id, (list, tuple)) and len(partner_id) > 1: + return str(partner_id[1] or "").strip() + return "" + + +def find_rows() -> List[Dict[str, Any]]: + with engine.begin() as conn: + rows = conn.execute(text(""" + SELECT id::text, name, tax_id, email, metadata + FROM customers + WHERE name ~ '^S[0-9]{4,}' + AND metadata->>'source_system' = 'odoo' + ORDER BY updated_at DESC + """)).mappings().all() + result: List[Dict[str, Any]] = [] + for row in rows: + partner_name = _partner_name_from_metadata(row.get("metadata")) + if partner_name and not partner_name.upper().startswith("S00"): + data = dict(row) + data["new_name"] = partner_name + result.append(data) + return result + + +def apply(rows: List[Dict[str, Any]]) -> int: + updated = 0 + with engine.begin() as conn: + for row in rows: + conn.execute(text(""" + UPDATE customers + SET name = :new_name, + metadata = COALESCE(metadata, '{}'::jsonb) || jsonb_build_object( + 'odoo_sale_order_name_repaired', true, + 'previous_customer_name', CAST(:old_name AS TEXT) + ), + updated_at = now() + WHERE id = CAST(:id AS UUID) + """), {"id": row["id"], "old_name": row["name"], "new_name": row["new_name"]}) + updated += 1 + return updated + + +def main() -> None: + parser = argparse.ArgumentParser(description="Repair Odoo S00xxx customer names.") + parser.add_argument("--apply", action="store_true", help="Apply updates. Default is dry-run.") + args = parser.parse_args() + + rows = find_rows() + print(f"Clientes Odoo com nome S00xxx reparáveis: {len(rows)}") + for row in rows[:50]: + print(f"- {row['name']} -> {row['new_name']} | NIF {row.get('tax_id') or '-'} | email {row.get('email') or '-'}") + if len(rows) > 50: + print(f"... mais {len(rows)-50}") + if not args.apply: + print("Dry-run. Para aplicar: repetir com --apply") + return + print(f"Atualizados: {apply(rows)}") + + +if __name__ == "__main__": + main() diff --git a/scripts/replace_jasmin_document_for_opportunity.py b/scripts/replace_jasmin_document_for_opportunity.py new file mode 100755 index 0000000..8ae0c53 --- /dev/null +++ b/scripts/replace_jasmin_document_for_opportunity.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +"""Replace imported Jasmin quotation/proforma details for an opportunity. + +Use this when an old/closed quotation was linked by mistake and a newer open +reconciliation item should become the active document for the opportunity. +This deletes only ClientFlow imported Jasmin quotation/proforma artifacts; it does +not delete documents in Jasmin. +""" +from __future__ import annotations + +import argparse +import asyncio +import json +from decimal import Decimal +from typing import Any + + +def _json_default(value: Any) -> str: + if isinstance(value, Decimal): + return str(value) + return str(value) + + +async def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--opportunity-id", required=True) + parser.add_argument("--item-id", required=True, help="reconciliation_items.id for the open/valid Jasmin candidate") + parser.add_argument("--dry-run", action="store_true") + args = parser.parse_args() + + from app.jasmin_backfill_service import replace_jasmin_document_for_opportunity_async + + result = await replace_jasmin_document_for_opportunity_async( + opportunity_id=args.opportunity_id, + item_id=args.item_id, + actor="operator_cli_replace_jasmin_document", + dry_run=args.dry_run, + ) + print(json.dumps(result, ensure_ascii=False, indent=2, default=_json_default)) + return 0 if result.get("ok") else 2 + + +if __name__ == "__main__": + raise SystemExit(asyncio.run(main())) diff --git a/scripts/reset_reconciliation_generated.py b/scripts/reset_reconciliation_generated.py new file mode 100755 index 0000000..d6b02ee --- /dev/null +++ b/scripts/reset_reconciliation_generated.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python3 +"""Reset generated reconciliation staging items so they can be rebuilt. + +Safe by default: dry-run only and only affects generated external candidates +from Jasmin/Odoo/Packlink in open/needs_review/ignored states. It does not +remove opportunities, commercial documents, payment proofs, operation links or +external system records. + +Examples: + PYTHONPATH=. python scripts/reset_reconciliation_generated.py + PYTHONPATH=. python scripts/reset_reconciliation_generated.py --apply + PYTHONPATH=. python scripts/reset_reconciliation_generated.py --source jasmin --source odoo --apply + PYTHONPATH=. python scripts/reset_reconciliation_generated.py --days 30 --apply + PYTHONPATH=. python scripts/reset_reconciliation_generated.py --include-manual --apply +""" +from __future__ import annotations + +import argparse +from datetime import datetime, timedelta, timezone +from typing import Any, Dict, List + +from sqlalchemy import text + +from app.db import engine, init_db +from app.reconciliation_service import ensure_reconciliation_schema + +DEFAULT_SOURCES = ["jasmin", "odoo", "packlink"] +DEFAULT_STATUSES = ["open", "needs_review", "ignored"] + + +def _backup_table_name() -> str: + return "reconciliation_items_reset_backup_" + datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S") + + +def _build_where(args: argparse.Namespace) -> tuple[str, Dict[str, Any]]: + sources = list(args.source or DEFAULT_SOURCES) + if args.include_manual and "manual" not in sources: + sources.append("manual") + statuses = list(args.status or DEFAULT_STATUSES) + params: Dict[str, Any] = {"sources": sources, "statuses": statuses} + clauses = [ + "source_system = ANY(CAST(:sources AS TEXT[]))", + "status = ANY(CAST(:statuses AS TEXT[]))", + ] + # Extra guard: never touch items already linked to an opportunity unless + # the operator explicitly changes the status list and unlinks manually. + clauses.append("opportunity_id IS NULL") + + if args.days is not None: + days = max(int(args.days), 1) + cutoff = (datetime.now(timezone.utc).date() - timedelta(days=days - 1)).isoformat() + params["cutoff"] = cutoff + clauses.append("(document_date IS NULL OR document_date >= CAST(:cutoff AS DATE))") + return " AND ".join(clauses), params + + +def _summarize(where_sql: str, params: Dict[str, Any], limit: int) -> Dict[str, Any]: + with engine.begin() as conn: + counts = conn.execute(text(f""" + SELECT source_system, external_type, status, COUNT(*) AS total + FROM reconciliation_items + WHERE {where_sql} + GROUP BY source_system, external_type, status + ORDER BY source_system, external_type, status + """), params).mappings().all() + rows = conn.execute(text(f""" + SELECT id::text, source_system, external_type, status, document_number, + customer_name, customer_tax_id, document_date, amount, title + FROM reconciliation_items + WHERE {where_sql} + ORDER BY updated_at DESC, created_at DESC + LIMIT :limit + """), {**params, "limit": int(limit)}).mappings().all() + return {"counts": [dict(r) for r in counts], "items": [dict(r) for r in rows]} + + +def _apply_reset(where_sql: str, params: Dict[str, Any]) -> Dict[str, Any]: + backup_table = _backup_table_name() + # backup_table is generated internally from digits/underscore only. + with engine.begin() as conn: + total = conn.execute(text(f"SELECT COUNT(*) FROM reconciliation_items WHERE {where_sql}"), params).scalar() or 0 + conn.execute(text(f"CREATE TABLE {backup_table} AS SELECT * FROM reconciliation_items WHERE {where_sql}"), params) + deleted = conn.execute(text(f"DELETE FROM reconciliation_items WHERE {where_sql}"), params).rowcount or 0 + return {"matched": int(total), "deleted": int(deleted), "backup_table": backup_table} + + +def main() -> None: + parser = argparse.ArgumentParser(description="Reset generated reconciliation candidates and keep a DB backup table.") + parser.add_argument("--source", action="append", choices=["jasmin", "odoo", "packlink", "manual"], help="source_system to reset; repeatable. Default: jasmin, odoo, packlink") + parser.add_argument("--status", action="append", choices=["open", "needs_review", "ignored"], help="status to reset; repeatable. Default: open, needs_review, ignored") + parser.add_argument("--days", type=int, help="only reset candidates inside the last N days; default is all dates") + parser.add_argument("--include-manual", action="store_true", help="also include source_system=manual; use with care") + parser.add_argument("--limit", type=int, default=50, help="preview sample size") + parser.add_argument("--apply", action="store_true", help="delete matched staging rows after creating a backup table") + args = parser.parse_args() + + init_db() + ensure_reconciliation_schema() + where_sql, params = _build_where(args) + summary = _summarize(where_sql, params, args.limit) + + print("Alvo do reset:") + print(f" fontes: {', '.join(params['sources'])}") + print(f" estados: {', '.join(params['statuses'])}") + print(" proteção: opportunity_id IS NULL") + if args.days is not None: + print(f" janela: >= {params['cutoff']} ({args.days} dias)") + print("\nContagens:") + if not summary["counts"]: + print(" 0 itens encontrados") + for row in summary["counts"]: + print(f" {row['source_system']} · {row['external_type']} · {row['status']}: {row['total']}") + + print("\nAmostra:") + for row in summary["items"]: + print( + f" {row.get('document_date') or '-'} · {row.get('source_system')} · {row.get('external_type')} · " + f"{row.get('status')} · {row.get('document_number') or '-'} · {row.get('customer_name') or '-'}" + ) + + if not args.apply: + print("\nDry-run. Para aplicar: repetir com --apply") + return + + result = _apply_reset(where_sql, params) + print("\nReset aplicado:") + print(f" encontrados: {result['matched']}") + print(f" apagados: {result['deleted']}") + print(f" backup: {result['backup_table']}") + + +if __name__ == "__main__": + main() diff --git a/scripts/review_domain_auto_applied_suggestions.py b/scripts/review_domain_auto_applied_suggestions.py new file mode 100755 index 0000000..5ca6536 --- /dev/null +++ b/scripts/review_domain_auto_applied_suggestions.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +"""List/revert risky fiscal suggestions auto-applied from domain-only matches.""" +from __future__ import annotations + +import argparse +import json + +from sqlalchemy import text + +from app.db import engine + +DOMAIN_MATCHES = ( + "email_principal_dominio", + "email_dominio_empresa_associada", + "contacto_email_dominio", + "dominio", + "email_dominio", + "website_dominio", +) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--apply", action="store_true", help="revert accepted domain-only suggestions and unlink matching opportunity customer") + args = parser.parse_args() + + with engine.begin() as conn: + rows = conn.execute(text(""" + SELECT + s.id::text, + s.opportunity_id::text, + s.suggested_customer_id::text, + s.suggested_name, + s.suggested_nif, + s.match_type, + s.confidence, + o.customer_name, + o.customer_email, + o.local_customer_id::text AS current_customer_id + FROM fiscal_customer_suggestions s + JOIN opportunities o ON o.id = s.opportunity_id + WHERE s.status = 'accepted' + AND s.auto_applied = TRUE + AND s.match_type = ANY(:matches) + ORDER BY s.updated_at DESC + """), {"matches": list(DOMAIN_MATCHES)}).mappings().all() + + print(f"Sugestões por domínio auto-aplicadas: {len(rows)}") + for r in rows: + print(f"- {r['customer_name']} <{r['customer_email']}> -> {r['suggested_name']} / {r['suggested_nif']} | {r['match_type']} | {r['confidence']} | opp={r['opportunity_id']}") + + if not args.apply: + print("Dry-run. Para reverter: repetir com --apply") + return + + updated = 0 + with engine.begin() as conn: + for r in rows: + # Only unlink if the opportunity is still linked to the same customer suggested by this risky suggestion. + if r["suggested_customer_id"] and r["current_customer_id"] == r["suggested_customer_id"]: + conn.execute(text(""" + UPDATE opportunities + SET local_customer_id = NULL, + metadata = COALESCE(metadata, '{}'::jsonb) || jsonb_build_object( + 'domain_match_auto_apply_reverted', true, + 'domain_match_reverted_suggestion_id', :suggestion_id, + 'domain_match_reverted_customer_name', :customer_name, + 'domain_match_reverted_at', now() + ), + updated_at = now() + WHERE id = CAST(:opportunity_id AS UUID) + """), { + "opportunity_id": r["opportunity_id"], + "suggestion_id": r["id"], + "customer_name": r["suggested_name"], + }) + conn.execute(text(""" + UPDATE fiscal_customer_suggestions + SET status = 'rejected', auto_applied = FALSE, resolved_by = 'domain_match_safety_review', + resolved_at = now(), reason = COALESCE(reason, '') || ' | reverted: domain-only auto-apply is unsafe', + updated_at = now() + WHERE id = CAST(:id AS UUID) + """), {"id": r["id"]}) + updated += 1 + print(f"Revertidas: {updated}") + + +if __name__ == "__main__": + main() diff --git a/scripts/run_dev.sh b/scripts/run_dev.sh new file mode 100755 index 0000000..937d38c --- /dev/null +++ b/scripts/run_dev.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -euo pipefail + +python -m uvicorn app.main:app --reload --host 127.0.0.1 --port 8000 diff --git a/scripts/run_reconciliation_pipeline.py b/scripts/run_reconciliation_pipeline.py new file mode 100755 index 0000000..083b491 --- /dev/null +++ b/scripts/run_reconciliation_pipeline.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +"""Run the recommended ClientFlow pipeline: enrichment -> external sync. + +This orchestrator is safe to run periodically. It does not apply low-confidence +reconciliation decisions; it prepares fiscal identities first and then rebuilds +external candidates for the operator. +""" +from __future__ import annotations + +import argparse +import asyncio +import json +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from app.fiscal_enrichment_service import enrich_open_opportunities, ensure_fiscal_enrichment_schema +from app.external_reconciliation_sync import sync_all_external_reconciliation_candidates + + +async def _run(args: argparse.Namespace) -> dict: + ensure_fiscal_enrichment_schema() + enrichment = enrich_open_opportunities( + limit=args.enrichment_limit, + apply_safe=not args.no_auto_apply, + mode="pipeline", + ) + reconciliation = await sync_all_external_reconciliation_candidates(limit=args.limit, days=args.days) + return {"enrichment": enrichment, "reconciliation": reconciliation} + + +def main() -> int: + parser = argparse.ArgumentParser(description="Correr pipeline ClientFlow: enriquecimento fiscal + reconciliação") + parser.add_argument("--days", type=int, default=7, help="Janela de reconciliação externa") + parser.add_argument("--limit", type=int, default=100, help="Limite por fonte externa") + parser.add_argument("--enrichment-limit", type=int, default=100, help="Limite de oportunidades a enriquecer antes da reconciliação") + parser.add_argument("--no-auto-apply", action="store_true", help="Não auto-associar sugestões fiscais fortes") + args = parser.parse_args() + result = asyncio.run(_run(args)) + print(json.dumps(result, ensure_ascii=False, indent=2, default=str)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/sync_external_reconciliation.py b/scripts/sync_external_reconciliation.py new file mode 100755 index 0000000..eae705d --- /dev/null +++ b/scripts/sync_external_reconciliation.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +"""Sync external systems into reconciliation candidates. + +Examples: + PYTHONPATH=. python scripts/sync_external_reconciliation.py --all + PYTHONPATH=. python scripts/sync_external_reconciliation.py --jasmin --limit 50 --days 3 + PYTHONPATH=. python scripts/sync_external_reconciliation.py --odoo --days 3 + PYTHONPATH=. python scripts/sync_external_reconciliation.py --customers --limit 200 + +The full --all pipeline first creates/updates fiscal customers from Jasmin/Odoo +and then stages reconciliation_items. It never creates opportunities or confirms +payments, so the operator can link/create/ignore in the UI. +""" +from __future__ import annotations + +import argparse +import asyncio +import json +from typing import Any, Dict, List + +from app.external_reconciliation_sync import ( + sync_all_external_reconciliation_candidates, + sync_external_fiscal_customers_for_reconciliation, + sync_jasmin_reconciliation_candidates, + sync_odoo_reconciliation_candidates, + sync_packlink_reconciliation_candidates, +) + + +def _print(result: Dict[str, Any]) -> None: + print(json.dumps(result, ensure_ascii=False, indent=2, default=str)) + + +async def main() -> None: + parser = argparse.ArgumentParser(description="Sync external APIs into ClientFlow reconciliation candidates.") + parser.add_argument("--all", action="store_true", help="sync all enabled external systems") + parser.add_argument("--jasmin", action="store_true", help="sync Jasmin quotations/invoices") + parser.add_argument("--odoo", action="store_true", help="sync Odoo sale orders") + parser.add_argument("--packlink", action="store_true", help="sync Packlink shipments") + parser.add_argument("--customers", action="store_true", help="seed fiscal customers from Jasmin/Odoo before document reconciliation") + parser.add_argument("--limit", type=int, default=50, help="maximum records per source") + parser.add_argument("--days", type=int, default=3, help="lookback window for Jasmin/Odoo/Packlink syncs") + args = parser.parse_args() + + if args.all or not (args.jasmin or args.odoo or args.packlink or args.customers): + _print(await sync_all_external_reconciliation_candidates(limit=args.limit, days=args.days)) + return + + if args.customers and not (args.jasmin or args.odoo or args.packlink): + _print(await sync_external_fiscal_customers_for_reconciliation(limit=max(args.limit, 200))) + return + + results: List[Dict[str, Any]] = [] + customer_result = None + if args.customers: + customer_result = await sync_external_fiscal_customers_for_reconciliation(limit=max(args.limit, 200)) + if args.jasmin: + results.append(await sync_jasmin_reconciliation_candidates(limit=args.limit, days=args.days)) + if args.odoo: + # Odoo XML-RPC client is sync; the service itself stays sync for easier reuse. + results.append(sync_odoo_reconciliation_candidates(limit=args.limit, days=args.days)) + if args.packlink: + results.append(await sync_packlink_reconciliation_candidates(limit=args.limit, days=args.days)) + output = { + "seen": sum(int(r.get("seen") or 0) for r in results), + "created_or_updated": sum(int(r.get("created_or_updated") or 0) for r in results), + "results": results, + } + if customer_result is not None: + output["customer_seen"] = customer_result.get("seen", 0) + output["customers_created_or_updated"] = customer_result.get("created_or_updated", 0) + output["customer_results"] = customer_result.get("results", []) + _print(output) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/scripts/sync_reconciliation_candidates.py b/scripts/sync_reconciliation_candidates.py new file mode 100755 index 0000000..0bc2199 --- /dev/null +++ b/scripts/sync_reconciliation_candidates.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 +"""Create reconciliation candidates from local external records. + +This script is safe to run from cron/systemd timer. It is idempotent and only +creates/updates reconciliation_items for information already known locally, such +as Jasmin commercial documents without opportunity_id. +""" +from __future__ import annotations + +from app.db import init_db +from app.reconciliation_service import sync_local_documents_without_opportunity + + +def main() -> None: + init_db() + result = sync_local_documents_without_opportunity(limit=500) + print( + "Reconciliation sync finished: " + f"seen={result.get('seen', 0)} " + f"created_or_updated={result.get('created_or_updated', 0)}" + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/test_action_core_http.py b/scripts/test_action_core_http.py new file mode 100644 index 0000000..1d415f0 --- /dev/null +++ b/scripts/test_action_core_http.py @@ -0,0 +1,95 @@ +import json +import os +import sys +from pathlib import Path + +import requests + + +BASE_URL = os.getenv("CLIENTFLOW_BASE_URL", "http://127.0.0.1:8000") + + +CASES = [ + { + "conversation_id": "case-send-invoice", + "message": "Recebemos o equipamento. Agradecemos o envio da factura.", + }, + { + "conversation_id": "case-payment", + "message": "Segue comprovativo de pagamento em anexo.", + }, + { + "conversation_id": "case-info", + "message": "Bom dia, podem enviar mais informações sobre carregadores monofásicos?", + }, + { + "conversation_id": "case-shipment", + "message": "Boa tarde, gostava de saber se já enviaram o carregador.", + }, +] + + +def main() -> int: + out_dir = Path("resultados-action-core") + out_dir.mkdir(exist_ok=True) + + rows = [] + + for case in CASES: + payload = { + "last_customer_message": case["message"], + "previous_context": "Teste Action Core.", + "source": "manual_test", + "conversation_id": case["conversation_id"], + "contact_id": "test-contact", + } + + response = requests.post( + f"{BASE_URL}/analyze", + headers={"Content-Type": "application/json"}, + json=payload, + timeout=90, + ) + + response.raise_for_status() + data = response.json() + + action_decision = data.get("action_decision") or {} + action_result = data.get("action_result") or {} + + row = { + "conversation_id": case["conversation_id"], + "action_code": action_decision.get("action_code"), + "route": action_result.get("route"), + "action": action_result.get("action"), + "safe_to_post": action_result.get("safe_to_post"), + "task_id": data.get("task_id"), + "needs_review": data.get("needs_review"), + } + + rows.append(row) + + print("=" * 80) + print(case["conversation_id"]) + print("action_code:", row["action_code"]) + print("route:", row["route"]) + print("action:", row["action"]) + print("safe_to_post:", row["safe_to_post"]) + print("task_id:", row["task_id"]) + + (out_dir / f"{case['conversation_id']}.json").write_text( + json.dumps(data, ensure_ascii=False, indent=2), + encoding="utf-8", + ) + + (out_dir / "summary.json").write_text( + json.dumps(rows, ensure_ascii=False, indent=2), + encoding="utf-8", + ) + + print("\nResumo:", out_dir / "summary.json") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/test_jasmin_connection.py b/scripts/test_jasmin_connection.py new file mode 100755 index 0000000..5b4dcd1 --- /dev/null +++ b/scripts/test_jasmin_connection.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +"""Teste não destrutivo da integração Jasmin. + +Não cria clientes/documentos. Valida OAuth, versão, OData de clientes, +produtos, orçamentos e faturas. +""" + +from __future__ import annotations + +import asyncio +import json +import os +import sys +from pathlib import Path + +PROJECT_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(PROJECT_ROOT)) +os.chdir(PROJECT_ROOT) + +__test__ = False + + +def show(title: str, value, limit: int = 2500) -> None: + print(f"\n=== {title} ===") + try: + text = json.dumps(value, indent=2, ensure_ascii=False, default=str) + except Exception: + text = str(value) + print(text[:limit]) + + +async def main() -> int: + from app.jasmin_client import JasminClient, JasminError + + client = JasminClient() + try: + token = await client.get_token() + print(f"OAuth OK: token_length={len(token)}") + show("Versões", await client.get_versions()) + show("Clientes OData", await client.list_customers_odata(top=5)) + show("Produtos OData", await client.list_sales_items(top=10)) + show("Últimos orçamentos", await client.list_quotations(top=5)) + show("Últimas faturas", await client.list_invoices(top=5)) + except JasminError as exc: + print(f"ERRO Jasmin: {exc}") + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(asyncio.run(main())) diff --git a/scripts/test_operational_flow.py b/scripts/test_operational_flow.py new file mode 100644 index 0000000..490ec1d --- /dev/null +++ b/scripts/test_operational_flow.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python3 +"""Smoke placeholder for the clean ClientFlow operational architecture. + +Operational progression is represented by business_events and operation_links, +not by extra triage action_codes. +""" + + +def main() -> int: + print("Use API/UI smoke tests against current action codes and operation_links.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/test_packlink_connection.py b/scripts/test_packlink_connection.py new file mode 100755 index 0000000..87a2232 --- /dev/null +++ b/scripts/test_packlink_connection.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +"""Smoke test Packlink PRO. + +Uso: + export PACKLINK_API_KEY=... + export PACKLINK_BASE_URL=https://api.packlink.com/v1 + python scripts/test_packlink_connection.py +""" + +from __future__ import annotations + +import asyncio +import json +import os +import sys +from pathlib import Path + +PROJECT_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(PROJECT_ROOT)) +os.chdir(PROJECT_ROOT) + +# Permite que o ficheiro seja importado por pytest sem exigir .env real. +os.environ.setdefault("OPENROUTER_API_KEY", "dummy") +os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://clientflow:password@127.0.0.1:5432/clientflow") + +from app.packlink_client import PacklinkClient + + +def normalize_packlink_zip(country: str, zip_code: str, *, for_quote: bool = True) -> str: + country = str(country or "").upper().strip() + zip_code = str(zip_code or "").strip() + if country == "PT" and for_quote: + import re + match = re.search(r"\d{4}", zip_code) + if match: + return match.group(0) + return zip_code + + +def default_package() -> dict: + return { + "height": int(float(os.getenv("PACKLINK_DEFAULT_PACKAGE_HEIGHT", "10"))), + "width": int(float(os.getenv("PACKLINK_DEFAULT_PACKAGE_WIDTH", "20"))), + "length": int(float(os.getenv("PACKLINK_DEFAULT_PACKAGE_LENGTH", "30"))), + "weight": float(os.getenv("PACKLINK_DEFAULT_PACKAGE_WEIGHT", "2")), + } + + +def dump(title: str, value) -> None: + print(f"\n=== {title} ===") + print(json.dumps(value, ensure_ascii=False, indent=2, default=str)[:5000]) + + +async def main() -> int: + if not os.getenv("PACKLINK_API_KEY"): + print("ERRO: PACKLINK_API_KEY em falta") + return 2 + + client = PacklinkClient() + account = await client.get_client() + dump("Conta", account) + + warehouses = await client.get_warehouses() + dump("Armazéns", warehouses[:3]) + + parcels = await client.get_parcels() + dump("Volumes", parcels[:3]) + + from_zip = normalize_packlink_zip("PT", os.getenv("PACKLINK_TEST_FROM_ZIP", "3650-219"), for_quote=True) + to_zip = normalize_packlink_zip("PT", os.getenv("PACKLINK_TEST_TO_ZIP", "4000-001"), for_quote=True) + services = await client.quote_services( + from_country="PT", + from_zip=from_zip, + to_country="PT", + to_zip=to_zip, + source=os.getenv("PACKLINK_SOURCE", "PRO"), + packages=[default_package()], + ) + + simple = [ + { + "id": s.get("id"), + "carrier": s.get("carrier_name"), + "service": s.get("name"), + "price": (s.get("price") or {}).get("total_price") or s.get("base_price"), + "currency": s.get("currency") or (s.get("price") or {}).get("currency"), + "dropoff": s.get("dropoff"), + "parcelshop": s.get("delivery_to_parcelshop"), + } + for s in services + ] + dump("Serviços", simple) + + service_id = os.getenv("PACKLINK_DEFAULT_SERVICE_ID", "20571") + details = await client.get_service_details(service_id) + dump(f"Detalhes serviço {service_id}", details) + return 0 + + +if __name__ == "__main__": + raise SystemExit(asyncio.run(main())) diff --git a/scripts/test_sample.sh b/scripts/test_sample.sh new file mode 100755 index 0000000..bcb3aeb --- /dev/null +++ b/scripts/test_sample.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -euo pipefail +curl -sS -X POST http://127.0.0.1:8000/analyze \ + -H "Content-Type: application/json" \ + -d @tests/sample_cases/sample_analyze.json | jq . diff --git a/scripts/validate_v45_operational_core.py b/scripts/validate_v45_operational_core.py new file mode 100755 index 0000000..bc26fa1 --- /dev/null +++ b/scripts/validate_v45_operational_core.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +"""Lightweight v4.5 validation checks for a deployed ClientFlow backend.""" +from __future__ import annotations + +import sys + +from sqlalchemy import text + +from app.db import engine, init_db + + +REQUIRED_TABLES = ["communications", "timeline_events", "tasks", "integration_outbox", "opportunities"] +REQUIRED_TASK_COLUMNS = ["communication_id", "document_id", "shipment_id", "outbox_id", "priority", "assigned_to"] + + +def main() -> int: + init_db() + with engine.begin() as conn: + tables = {r[0] for r in conn.execute(text(""" + SELECT table_name + FROM information_schema.tables + WHERE table_schema = 'public' + """))} + missing_tables = [t for t in REQUIRED_TABLES if t not in tables] + + cols = {r[0] for r in conn.execute(text(""" + SELECT column_name + FROM information_schema.columns + WHERE table_schema = 'public' AND table_name = 'tasks' + """))} + missing_cols = [c for c in REQUIRED_TASK_COLUMNS if c not in cols] + + comm_count = conn.execute(text("SELECT COUNT(*) FROM communications")).scalar() + timeline_count = conn.execute(text("SELECT COUNT(*) FROM timeline_events")).scalar() + + if missing_tables or missing_cols: + print("ClientFlow v4.5 validation failed") + print("Missing tables:", ", ".join(missing_tables) or "none") + print("Missing task columns:", ", ".join(missing_cols) or "none") + return 1 + + print("ClientFlow v4.5 validation OK") + print(f"communications={comm_count} timeline_events={timeline_count}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/test_clientflow_e2e.sh b/test_clientflow_e2e.sh new file mode 100755 index 0000000..b868ef0 --- /dev/null +++ b/test_clientflow_e2e.sh @@ -0,0 +1,407 @@ +#!/usr/bin/env bash +set -u +set -o pipefail + +BASE_URL="${BASE_URL:-http://127.0.0.1:8020}" +PGHOST="${PGHOST:-192.168.1.110}" +PGUSER="${PGUSER:-clientflow}" +PGDATABASE="${PGDATABASE:-clientflow}" +TEST_TAG="e2e-$(date +%Y%m%d-%H%M%S)" +CONTACT_ID="${TEST_TAG}-contact" + +PASS=0 +FAIL=0 + +ok() { PASS=$((PASS+1)); echo "✅ $*"; } +fail() { FAIL=$((FAIL+1)); echo "❌ $*"; } + +if [ -z "${PGPASSWORD:-}" ]; then + echo "Define primeiro: export PGPASSWORD='password_da_bd'" + exit 2 +fi + +psql_q() { + PGPASSWORD="$PGPASSWORD" psql -h "$PGHOST" -U "$PGUSER" -d "$PGDATABASE" -Atc "$1" +} + +request() { + NAME="$1" + EXPECT="$2" + shift 2 + + echo + echo "==============================" + echo "$NAME" + echo "------------------------------" + + BODY="$(mktemp)" + CODE="$(curl -sS -o "$BODY" -w "%{http_code}" "$@" || true)" + + echo "HTTP $CODE" + + if echo "$EXPECT" | grep -qw "$CODE"; then + ok "$NAME" + else + fail "$NAME esperado [$EXPECT], recebeu HTTP $CODE" + echo "Body:" + cat "$BODY" + echo + fi + + LAST_BODY="$BODY" + LAST_CODE="$CODE" +} + +json_get() { + FILE="$1" + KEY="$2" + python - "$FILE" "$KEY" <<'PY' +import json, sys +path, key = sys.argv[1], sys.argv[2] +try: + data = json.load(open(path, encoding="utf-8")) +except Exception: + print("") + raise SystemExit + +cur = data +for part in key.split("."): + if isinstance(cur, dict): + cur = cur.get(part) + else: + cur = None + break + +print("" if cur is None else cur) +PY +} + +echo "============================================================" +echo "ClientFlow E2E" +echo "BASE_URL=$BASE_URL" +echo "TEST_TAG=$TEST_TAG" +echo "============================================================" + +echo +echo "== 0. BD ==" + +DB_OK="$(psql_q "SELECT current_database() || ':' || current_user;")" +if [ -n "$DB_OK" ]; then + ok "BD OK: $DB_OK" +else + fail "Falha ligação BD" + exit 2 +fi + +for T in raw_events messages action_runs tasks opportunities opportunity_events products opportunity_items operation_links; do + EXISTS="$(psql_q "SELECT to_regclass('public.$T') IS NOT NULL;")" + if [ "$EXISTS" = "t" ]; then + ok "Tabela existe: $T" + else + fail "Tabela em falta: $T" + fi +done + +PRODUCT_ID="$(psql_q "SELECT id FROM products WHERE active=true ORDER BY name LIMIT 1;")" +if [ -n "$PRODUCT_ID" ]; then + ok "Produto para teste: $PRODUCT_ID" +else + fail "Sem produto ativo para teste" +fi + +echo +echo "== 1. Páginas principais ==" + +request "GET /" "200" "$BASE_URL/" +request "GET /tasks" "200" "$BASE_URL/tasks" +request "GET /opportunities" "200" "$BASE_URL/opportunities" +request "GET /products" "200" "$BASE_URL/products" +request "GET /orders" "200" "$BASE_URL/orders" +request "GET /finance" "200" "$BASE_URL/finance" +request "GET /customers" "200" "$BASE_URL/customers" + +echo +echo "== 2. Analyze SEND_INFO ==" + +request "POST /analyze SEND_INFO" "200" \ + -X POST "$BASE_URL/analyze" \ + -H "Content-Type: application/json" \ + -d "{ + \"last_customer_message\": \"Enviar informação detalhada sobre carregador.\", + \"previous_context\": \"Assunto: Carregador\", + \"source\": \"e2e_test\", + \"conversation_id\": \"$TEST_TAG-info\", + \"contact_id\": \"$CONTACT_ID\" + }" + +TASK_INFO_ID="$(json_get "$LAST_BODY" "task_id")" +if [ -z "$TASK_INFO_ID" ]; then + TASK_INFO_ID="$(psql_q "SELECT id FROM tasks WHERE conversation_id='$TEST_TAG-info' ORDER BY created_at DESC LIMIT 1;")" +fi + +if [ -n "$TASK_INFO_ID" ]; then + ok "Task criada: $TASK_INFO_ID" +else + fail "Task SEND_INFO não criada" +fi + +OPP_INFO_ID="$(psql_q "SELECT opportunity_id FROM tasks WHERE id::text='$TASK_INFO_ID' LIMIT 1;")" +if [ -n "$OPP_INFO_ID" ]; then + ok "Oportunidade criada/ligada: $OPP_INFO_ID" +else + fail "Sem oportunidade ligada à task" +fi + +if [ -n "$TASK_INFO_ID" ]; then + request "GET task criada" "200 302 303" "$BASE_URL/tasks/$TASK_INFO_ID" +fi + +if [ -n "$OPP_INFO_ID" ]; then + request "GET oportunidade criada" "200" "$BASE_URL/opportunities/$OPP_INFO_ID" +fi + +echo +echo "== 3. Produto na oportunidade ==" + +if [ -n "$OPP_INFO_ID" ] && [ -n "$PRODUCT_ID" ]; then + request "POST adicionar produto" "302 303" \ + -X POST "$BASE_URL/opportunities/$OPP_INFO_ID/items/add" \ + -H "Content-Type: application/x-www-form-urlencoded" \ + --data-urlencode "product_id=$PRODUCT_ID" \ + --data-urlencode "quantity=1" \ + --data-urlencode "unit_price=" \ + --data-urlencode "discount_amount=0" \ + --data-urlencode "status=INTERESTED" + + ITEM_ID="$(psql_q "SELECT id FROM opportunity_items WHERE opportunity_id::text='$OPP_INFO_ID' ORDER BY created_at DESC LIMIT 1;")" + if [ -n "$ITEM_ID" ]; then + ok "Produto adicionado: $ITEM_ID" + else + fail "Produto não foi adicionado" + fi +fi + +echo +echo "== 4. Stage oportunidade ==" + +if [ -n "$OPP_INFO_ID" ]; then + request "POST stage INFO_SENT" "302 303" \ + -X POST "$BASE_URL/opportunities/$OPP_INFO_ID/stage" \ + -H "Content-Type: application/x-www-form-urlencoded" \ + --data-urlencode "stage=INFO_SENT" \ + --data-urlencode "note=E2E informação enviada" + + STAGE="$(psql_q "SELECT stage FROM opportunities WHERE id::text='$OPP_INFO_ID' LIMIT 1;")" + if [ "$STAGE" = "INFO_SENT" ]; then + ok "Stage INFO_SENT confirmado" + else + fail "Stage esperado INFO_SENT, recebido $STAGE" + fi +fi + +echo +echo "== 5. Complete task com nota ==" + +if [ -n "$TASK_INFO_ID" ]; then + request "POST complete-with-note" "302 303" \ + -X POST "$BASE_URL/tasks/$TASK_INFO_ID/complete-with-note" \ + -H "Content-Type: application/x-www-form-urlencoded" \ + --data-urlencode "done_note=E2E tarefa concluída" \ + --data-urlencode "done_note_extra=teste automático" + + STATUS="$(psql_q "SELECT status FROM tasks WHERE id::text='$TASK_INFO_ID' LIMIT 1;")" + if [ "$STATUS" = "done" ]; then + ok "Task concluída" + else + fail "Task deveria estar done, está $STATUS" + fi +fi + +echo + +echo "== 6. Cockpit operacional com workflow guard ==" + +if [ -n "$OPP_INFO_ID" ]; then + + echo + echo "-- 6.1 Bloqueio esperado: pagamento antes de pró-forma --" + + request "POST operation payment_confirmed antes de pró-forma" "409" \ + -X POST "$BASE_URL/opportunities/$OPP_INFO_ID/operations/payment_confirmed" \ + -H "Content-Type: application/x-www-form-urlencoded" \ + --data-urlencode "note=E2E bloqueio esperado" + + echo + echo "-- 6.2 Fluxo correto até venda Odoo --" + + request "POST operation jasmin_proforma" "302 303" \ + -X POST "$BASE_URL/opportunities/$OPP_INFO_ID/operations/jasmin_proforma" \ + -H "Content-Type: application/x-www-form-urlencoded" \ + --data-urlencode "external_id=$TEST_TAG-proforma" \ + --data-urlencode "external_name=PF-$TEST_TAG" \ + --data-urlencode "external_url=https://jasmin.blif.pt/e2e/proforma" \ + --data-urlencode "note=E2E pró-forma emitida" + + request "POST operation payment_confirmed" "302 303" \ + -X POST "$BASE_URL/opportunities/$OPP_INFO_ID/operations/payment_confirmed" \ + -H "Content-Type: application/x-www-form-urlencoded" \ + --data-urlencode "note=E2E pagamento confirmado" + + request "POST operation odoo_sale_order" "302 303" \ + -X POST "$BASE_URL/opportunities/$OPP_INFO_ID/operations/odoo_sale_order" \ + -H "Content-Type: application/x-www-form-urlencoded" \ + --data-urlencode "external_id=$TEST_TAG-sale-order" \ + --data-urlencode "external_name=SO-$TEST_TAG" \ + --data-urlencode "external_url=https://odoo.plixin.com/web#id=999999&model=sale.order&view_type=form" \ + --data-urlencode "note=E2E venda Odoo registada" + + echo + echo "-- 6.3 Bloqueio esperado: Packlink antes de fatura/Odoo pronto --" + + request "POST operation packlink_shipment bloqueado" "409" \ + -X POST "$BASE_URL/opportunities/$OPP_INFO_ID/operations/packlink_shipment" \ + -H "Content-Type: application/x-www-form-urlencoded" \ + --data-urlencode "external_id=$TEST_TAG-packlink-blocked" \ + --data-urlencode "external_name=PACKLINK-BLOQUEADO" \ + --data-urlencode "note=E2E bloqueio packlink esperado" + + echo + echo "-- 6.4 Simular Odoo pronto para despacho no teste --" + + PGPASSWORD="$PGPASSWORD" psql -h "$PGHOST" -U "$PGUSER" -d "$PGDATABASE" -c " + INSERT INTO operation_links ( + opportunity_id, + system, + external_type, + external_id, + external_name, + status, + payload, + last_synced_at, + updated_at + ) + VALUES ( + '$OPP_INFO_ID', + 'odoo', + 'physical_status', + '$TEST_TAG-physical-ready', + 'Estado físico E2E', + 'ready_to_ship', + jsonb_build_object( + 'label', 'Pronta para despacho', + 'reason', 'Simulação E2E: encomenda pronta.', + 'next_action', 'Criar envio Packlink.', + 'ready_to_ship', true, + 'sale_order', jsonb_build_object('id', 999999, 'name', 'SO-' || '$TEST_TAG'), + 'pickings', '[]'::jsonb, + 'productions', '[]'::jsonb + ), + now(), + now() + ) + ON CONFLICT (opportunity_id, system, external_type) + DO UPDATE SET + external_id = EXCLUDED.external_id, + external_name = EXCLUDED.external_name, + status = EXCLUDED.status, + payload = EXCLUDED.payload, + last_synced_at = now(), + updated_at = now(); + " + + request "POST operation odoo_physical_validated" "302 303" \ + -X POST "$BASE_URL/opportunities/$OPP_INFO_ID/operations/odoo_physical_validated" \ + -H "Content-Type: application/x-www-form-urlencoded" \ + --data-urlencode "external_id=$TEST_TAG-physical-validated" \ + --data-urlencode "external_name=Validação física E2E" \ + --data-urlencode "note=E2E encomenda física validada" + + request "POST operation jasmin_invoice" "302 303" \ + -X POST "$BASE_URL/opportunities/$OPP_INFO_ID/operations/jasmin_invoice" \ + -H "Content-Type: application/x-www-form-urlencoded" \ + --data-urlencode "external_id=$TEST_TAG-invoice" \ + --data-urlencode "external_name=FT-$TEST_TAG" \ + --data-urlencode "external_url=https://jasmin.blif.pt/e2e/invoice" \ + --data-urlencode "note=E2E fatura emitida" + + request "POST operation packlink_shipment" "302 303" \ + -X POST "$BASE_URL/opportunities/$OPP_INFO_ID/operations/packlink_shipment" \ + -H "Content-Type: application/x-www-form-urlencoded" \ + --data-urlencode "external_id=$TEST_TAG-shipment" \ + --data-urlencode "external_name=PACKLINK-$TEST_TAG" \ + --data-urlencode "external_url=https://pro.packlink.pt/e2e/shipment" \ + --data-urlencode "note=E2E envio criado" + + request "POST operation tracking_sent" "302 303" \ + -X POST "$BASE_URL/opportunities/$OPP_INFO_ID/operations/tracking_sent" \ + -H "Content-Type: application/x-www-form-urlencoded" \ + --data-urlencode "external_id=$TEST_TAG-tracking" \ + --data-urlencode "external_name=TRK-$TEST_TAG" \ + --data-urlencode "external_url=https://tracking.example/e2e" \ + --data-urlencode "note=E2E tracking enviado" + + request "POST operation delivered" "302 303" \ + -X POST "$BASE_URL/opportunities/$OPP_INFO_ID/operations/delivered" \ + -H "Content-Type: application/x-www-form-urlencoded" \ + --data-urlencode "external_id=$TEST_TAG-delivered" \ + --data-urlencode "external_name=Entrega E2E" \ + --data-urlencode "note=E2E entregue" + + LINKS="$(psql_q "SELECT count(*) FROM operation_links WHERE opportunity_id::text='$OPP_INFO_ID';")" + if [ "$LINKS" -ge 8 ]; then + ok "operation_links criados: $LINKS" + else + fail "operation_links insuficientes: $LINKS" + fi + + FINAL_STAGE="$(psql_q "SELECT stage FROM opportunities WHERE id::text='$OPP_INFO_ID' LIMIT 1;")" + if [ "$FINAL_STAGE" = "WON" ]; then + ok "Stage final WON" + else + fail "Stage final esperado WON, recebido $FINAL_STAGE" + fi + +else + fail "Sem OPP_INFO_ID; não foi possível testar cockpit operacional" +fi + +echo "== 7. Resumo ==" + +psql -h "$PGHOST" -U "$PGUSER" -d "$PGDATABASE" -c " +SELECT action_code, status, count(*) +FROM tasks +GROUP BY action_code, status +ORDER BY action_code, status; +" + +psql -h "$PGHOST" -U "$PGUSER" -d "$PGDATABASE" -c " +SELECT stage, status, count(*) +FROM opportunities +GROUP BY stage, status +ORDER BY stage, status; +" + +psql -h "$PGHOST" -U "$PGUSER" -d "$PGDATABASE" -c " +SELECT system, external_type, status, count(*) +FROM operation_links +GROUP BY system, external_type, status +ORDER BY system, external_type, status; +" + +echo +echo "== 8. Limpeza ignorada para inspeção manual ==" + + +echo +echo "============================================================" +echo "PASS=$PASS" +echo "FAIL=$FAIL" +echo "============================================================" + +if [ "$FAIL" -eq 0 ]; then + echo "✅ E2E concluído sem falhas críticas" + exit 0 +else + echo "❌ E2E encontrou falhas" + exit 1 +fi diff --git a/test_odoo_integration.sh b/test_odoo_integration.sh new file mode 100755 index 0000000..c9a14d8 --- /dev/null +++ b/test_odoo_integration.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +set -u +set -o pipefail + +BASE_URL="${BASE_URL:-http://127.0.0.1:8020}" +PGHOST="${PGHOST:-192.168.1.110}" +PGUSER="${PGUSER:-clientflow}" +PGDATABASE="${PGDATABASE:-clientflow}" + +if [ -z "${PGPASSWORD:-}" ]; then + echo "Define PGPASSWORD antes de correr." + exit 2 +fi + +code() { + local method="$1" + local url="$2" + shift 2 + curl -s -o /dev/null -w "%{http_code}\n" -X "$method" "$url" "$@" +} + +echo "== Odoo ClientFlow integration test ==" + +echo -n "GET /integrations -> " +code GET "$BASE_URL/integrations" + +echo -n "POST /integrations/odoo/test -> " +code POST "$BASE_URL/integrations/odoo/test" -H "Content-Type: application/x-www-form-urlencoded" + +echo -n "POST /integrations/odoo/sync-products -> " +code POST "$BASE_URL/integrations/odoo/sync-products" -H "Content-Type: application/x-www-form-urlencoded" + +echo +echo "Produtos com metadata Odoo:" +PGPASSWORD="$PGPASSWORD" psql -h "$PGHOST" -U "$PGUSER" -d "$PGDATABASE" -c " +SELECT + count(*) FILTER (WHERE metadata ? 'odoo') AS synced_products, + count(*) FILTER (WHERE COALESCE((metadata->'odoo'->>'has_bom')::boolean, false)) AS with_bom, + count(*) FILTER (WHERE COALESCE((metadata->'odoo'->'stock'->>'available')::numeric, 0) > 0) AS with_available_stock, + max(metadata->'odoo'->>'last_synced_at') AS last_synced_at +FROM products; +" + +echo +echo "Amostra:" +PGPASSWORD="$PGPASSWORD" psql -h "$PGHOST" -U "$PGUSER" -d "$PGDATABASE" -c " +SELECT + sku, + name, + metadata->'odoo'->>'product_id' AS odoo_id, + metadata->'odoo'->'stock'->>'available' AS available, + metadata->'odoo'->>'has_bom' AS has_bom +FROM products +WHERE metadata ? 'odoo' +ORDER BY name +LIMIT 15; +" + +echo +echo "Sync runs:" +PGPASSWORD="$PGPASSWORD" psql -h "$PGHOST" -U "$PGUSER" -d "$PGDATABASE" -c " +SELECT sync_type, status, total_seen, total_changed, total_errors, message, started_at, finished_at +FROM odoo_sync_runs +ORDER BY started_at DESC +LIMIT 10; +" diff --git a/test_odoo_order_status.sh b/test_odoo_order_status.sh new file mode 100755 index 0000000..337be0e --- /dev/null +++ b/test_odoo_order_status.sh @@ -0,0 +1,104 @@ +#!/usr/bin/env bash +set -u +set -o pipefail + +BASE_URL="${BASE_URL:-http://127.0.0.1:8020}" +PGHOST="${PGHOST:-192.168.1.110}" +PGUSER="${PGUSER:-clientflow}" +PGDATABASE="${PGDATABASE:-clientflow}" +OPP_ID="${OPP_ID:-}" + +if [ -z "${PGPASSWORD:-}" ]; then + echo "Define PGPASSWORD antes de correr." + exit 2 +fi + +psqlc() { + PGPASSWORD="$PGPASSWORD" PAGER=cat psql \ + -h "$PGHOST" \ + -U "$PGUSER" \ + -d "$PGDATABASE" \ + -v ON_ERROR_STOP=1 \ + -c "$1" +} + +psqlq() { + PGPASSWORD="$PGPASSWORD" PAGER=cat psql \ + -h "$PGHOST" \ + -U "$PGUSER" \ + -d "$PGDATABASE" \ + -At \ + -v ON_ERROR_STOP=1 \ + -c "$1" +} + +if [ -z "$OPP_ID" ]; then + OPP_ID="$(psqlq " + SELECT opportunity_id + FROM operation_links + WHERE system='odoo' + AND external_type='sale_order' + ORDER BY updated_at DESC + LIMIT 1; + ")" +fi + +echo "== Teste estado físico Odoo ==" +echo "OPP_ID=$OPP_ID" +echo + +echo "Venda Odoo ligada:" +psqlc " +SELECT + opportunity_id, + external_id, + external_name, + external_url, + status, + updated_at +FROM operation_links +WHERE opportunity_id::text = '$OPP_ID' + AND system='odoo' + AND external_type='sale_order'; +" + +echo +echo -n "POST sync-status -> " +curl -s -o /dev/null -w "%{http_code}\n" \ + -X POST "$BASE_URL/opportunities/$OPP_ID/odoo/sync-status" + +echo +echo "Estado físico guardado:" +psqlc " +SELECT + opportunity_id, + external_id, + external_name, + status, + payload->>'label' AS label, + payload->>'reason' AS reason, + payload->>'next_action' AS next_action, + payload->>'ready_to_ship' AS ready_to_ship, + updated_at +FROM operation_links +WHERE opportunity_id::text = '$OPP_ID' + AND system='odoo' + AND external_type='physical_status' +ORDER BY updated_at DESC +LIMIT 1; +" + +echo +echo "Resumo Odoo:" +psqlc " +SELECT + payload->'sale_order' AS sale_order, + jsonb_array_length(COALESCE(payload->'pickings', '[]'::jsonb)) AS pickings, + jsonb_array_length(COALESCE(payload->'productions', '[]'::jsonb)) AS productions +FROM operation_links +WHERE opportunity_id::text = '$OPP_ID' + AND system='odoo' + AND external_type='physical_status' +ORDER BY updated_at DESC +LIMIT 1; +" diff --git a/tests/sample_cases/sample_analyze.json b/tests/sample_cases/sample_analyze.json new file mode 100644 index 0000000..1bebcdf --- /dev/null +++ b/tests/sample_cases/sample_analyze.json @@ -0,0 +1,13 @@ +{ + "last_customer_message": "Boa tarde, gostávamos de avançar com a proposta. Pode enviar a fatura pró-forma?", + "previous_context": "A empresa enviou proposta para 1 carregador monofásico 7,4 kW com cabo de 5 metros.", + "current_state": { + "conversation_state": "a_aguardar_cliente", + "commercial_state": "proposta_enviada", + "relationship_state": "lead_existente", + "last_action_by": "empresa" + }, + "source": "email", + "conversation_id": "sample-001", + "contact_id": "contact-001" +} \ No newline at end of file diff --git a/tests/test_clean_architecture_static.py b/tests/test_clean_architecture_static.py new file mode 100644 index 0000000..a1c0f49 --- /dev/null +++ b/tests/test_clean_architecture_static.py @@ -0,0 +1,38 @@ +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +LEGACY_ACTION_CODES = { + "CONFIRM_PAYMENT_AND_PREPARE_SHIPMENT", + "CHECK_SHIPMENT_STATUS", + "CHECK_MISSING_MATERIAL", + "HANDLE_TECHNICAL_SUPPORT", + "HANDLE_WARRANTY_SUPPORT", + "PREPARE_ORDER", + "VALIDATE_SHIPMENT", + "FOLLOW_UP_INFO", + "CONTACT_CLIENT", + "SCHEDULE_MEETING", +} + + +def test_action_catalog_contains_only_current_triage_codes(): + text = (ROOT / "app" / "action_catalog.py").read_text() + for code in LEGACY_ACTION_CODES: + assert code not in text + + +def test_no_absolute_dashboard_paths(): + text = (ROOT / "app" / "admin_dashboard.py").read_text() + assert "/mnt/ssd/home/plx/clientflow_backend" not in text + assert "/mnt/ssd/backups/clientflow" not in text + + +def test_task_preparations_schema_exists(): + text = (ROOT / "app" / "db.py").read_text() + assert "CREATE TABLE IF NOT EXISTS task_preparations" in text + + +def test_sqlite_default_removed(): + text = (ROOT / "app" / "config.py").read_text() + assert "sqlite:///" not in text + assert "database_url: str" in text diff --git a/tests/test_v40_operational_files.py b/tests/test_v40_operational_files.py new file mode 100644 index 0000000..903de08 --- /dev/null +++ b/tests/test_v40_operational_files.py @@ -0,0 +1,20 @@ +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def test_migrations_exist(): + assert (ROOT / "migrations" / "001_core_indexes.sql").exists() + assert (ROOT / "scripts" / "apply_migrations.py").exists() + + +def test_admin_ui_router_exists(): + assert (ROOT / "app" / "admin_ui" / "router.py").exists() + main = (ROOT / "app" / "main.py").read_text() + assert "app.admin_ui.router" in main + + +def test_internal_api_exists(): + assert (ROOT / "app" / "api" / "internal.py").exists() + main = (ROOT / "app" / "main.py").read_text() + assert "internal_api_router" in main diff --git a/tests/test_v42_review_fixes.py b/tests/test_v42_review_fixes.py new file mode 100644 index 0000000..4b058d7 --- /dev/null +++ b/tests/test_v42_review_fixes.py @@ -0,0 +1,40 @@ +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def test_outbox_dry_run_has_explicit_non_sent_state(): + text = (ROOT / "scripts" / "process_outbox.py").read_text() + service = (ROOT / "app" / "integration_outbox_service.py").read_text() + assert "mark_outbox_dry_run" in text + assert 'status="dry_run"' in service + assert "OUTBOX_DRY_RUN=true; ação não executada" in service + + +def test_jasmin_quotation_idempotency_is_deterministic(): + text = (ROOT / "app" / "jasmin_service.py").read_text() + assert 'idempotency_key=f"jasmin:quotation:{opportunity_id}"' in text + assert 'idempotency_key=f"jasmin:quotation:{opportunity_id}:{uuid.uuid4()}"' not in text + + +def test_admin_token_setting_is_wired(): + config = (ROOT / "app" / "config.py").read_text() + dashboard = (ROOT / "app" / "admin_dashboard.py").read_text() + assert "clientflow_admin_token" in config + assert "require_admin_access" in dashboard + assert "X-ClientFlow-Admin-Token" in dashboard + + +def test_outbox_operational_states_are_visible(): + service = (ROOT / "app" / "integration_outbox_service.py").read_text() + dashboard = (ROOT / "app" / "admin_dashboard.py").read_text() + for state in ["blocked", "dry_run", "cancelled", "processing"]: + assert state in service + assert state in dashboard + + +def test_internal_api_uses_admin_token_when_configured(): + text = (ROOT / "app" / "api" / "internal.py").read_text() + assert "require_internal_access" in text + assert "X-ClientFlow-Admin-Token" in text + assert "dependencies=[Depends(require_internal_access)]" in text diff --git a/tests/test_v451_ui_alignment.py b/tests/test_v451_ui_alignment.py new file mode 100644 index 0000000..8e5a33f --- /dev/null +++ b/tests/test_v451_ui_alignment.py @@ -0,0 +1,46 @@ +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def read(path: str) -> str: + return (ROOT / path).read_text(encoding="utf-8") + + +def test_v453_ui_alignment_styles_are_present(): + admin = read("app/admin_dashboard.py") + styles = read("app/admin_ui/styles.py") + layout = read("app/admin_ui/layout.py") + navigation = read("app/admin_ui/navigation.py") + components = read("app/admin_ui/components.py") + assert "ADMIN_UI_V451_CSS" in admin + assert "--cf-sidebar-width:284px" in styles + assert "background:linear-gradient(180deg,#fff 0,#f8fbff 100%)" in styles + assert "cf-global-search" in layout + assert "v4.7 Operator Workbench" in navigation + assert "cf-kpi-icon" in components + assert "kpi_icon_for" in components + + +def test_v452_nav_matches_operational_information_architecture(): + navigation = read("app/admin_ui/navigation.py") + for label in [ + "Dashboard", + "Centro de trabalho", + "Oportunidades", + "Clientes", + "Produtos", + "Encomendas", + "Financeiro", + "Integrações", + "Admin", + "Comunicações", + "Outbox", + "Configuração", + ]: + assert label in navigation + + +def test_ui_css_endpoint_uses_v451_css(): + admin = read("app/admin_dashboard.py") + assert "return Response(ADMIN_UI_V451_CSS" in admin diff --git a/tests/test_v45_operational_core.py b/tests/test_v45_operational_core.py new file mode 100644 index 0000000..f96590f --- /dev/null +++ b/tests/test_v45_operational_core.py @@ -0,0 +1,49 @@ +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def read(path: str) -> str: + return (ROOT / path).read_text(encoding="utf-8") + + +def test_v45_communications_service_and_schema_exist(): + service = read("app/communication_service.py") + db = read("app/db.py") + migration = read("migrations/006_v45_operational_core.sql") + + assert "CREATE TABLE IF NOT EXISTS communications" in service + assert "CREATE TABLE IF NOT EXISTS timeline_events" in service + assert "communication_id" in service + assert "ensure_communication_schema" in db + assert "communications" in migration + assert "timeline_events" in migration + + +def test_v45_admin_ui_routes_are_registered(): + communications = read("app/admin_ui/pages/communications.py") + operations = read("app/admin_ui/pages/operations.py") + dashboard = read("app/admin_ui/pages/dashboard.py") + assert '@router.get("/communications"' in communications + assert '@router.get("/comunicacoes"' in communications + assert '@router.get("/communications/{communication_id}"' in communications + assert 'Centro de trabalho' in operations + assert 'Dashboard = visibilidade' in dashboard + assert 'Inbox classificada' in communications + + +def test_v45_operations_surface_work_items(): + operations = read("app/operations_service.py") + operations_page = read("app/admin_ui/pages/operations.py") + assert "work_items" in operations + assert "recent_communications" in operations + assert "communications_open" in operations + assert "Fila operacional priorizada" in operations_page + + +def test_v45_opportunity_has_communication_and_timeline_sections(): + opportunities = read("app/admin_ui/pages/opportunities.py") + assert 'href="#comunicacoes"' in opportunities + assert 'id="comunicacoes"' in opportunities + assert 'id="timeline"' in opportunities + assert 'list_communications_for_opportunity' in opportunities diff --git a/tests/test_v461_customer_mismatch_warning.py b/tests/test_v461_customer_mismatch_warning.py new file mode 100644 index 0000000..7c6c4d0 --- /dev/null +++ b/tests/test_v461_customer_mismatch_warning.py @@ -0,0 +1,22 @@ +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def read(path: str) -> str: + return (ROOT / path).read_text(encoding="utf-8") + + +def test_customer_mismatch_warning_no_longer_blocks_by_name(): + admin = read("app/admin_dashboard.py") + + assert "def opportunity_customer_mismatch" in admin + assert "return False" in admin + assert "Nome de contacto ≠ cliente fiscal não é um erro fiável" in admin + assert "Atenção: contacto e cliente fiscal não coincidem" not in admin + + +def test_customer_mismatch_setting_remains_accepted_for_env_compatibility(): + config = read("app/config.py") + assert "clientflow_customer_mismatch_warning_enabled" in config + assert "clientflow_customer_mismatch_warning_enabled: bool = False" in config diff --git a/tests/test_v462_disable_customer_mismatch_warning.py b/tests/test_v462_disable_customer_mismatch_warning.py new file mode 100644 index 0000000..d793591 --- /dev/null +++ b/tests/test_v462_disable_customer_mismatch_warning.py @@ -0,0 +1,27 @@ +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def read(path: str) -> str: + return (ROOT / path).read_text(encoding="utf-8") + + +def test_customer_mismatch_warning_disabled_by_default(): + config = read("app/config.py") + assert "clientflow_customer_mismatch_warning_enabled: bool = False" in config + assert "legitimate contacts use abbreviated names" in config + + +def test_opportunity_detail_switches_to_single_column_earlier(): + styles = read("app/admin_ui/styles.py") + assert "@media (max-width: 1400px)" in styles + assert ".detail-grid" in styles + assert "grid-template-columns: 1fr !important" in styles + + +def test_v462_notes_exist(): + notes = read("docs/CLIENTFLOW_V462_DISABLE_CUSTOMER_MISMATCH_WARNING.md") + assert "CLIENTFLOW_CUSTOMER_MISMATCH_WARNING_ENABLED=false" in notes + assert "Riotec elec" in notes + assert "Bruno Oliveira" in notes diff --git a/tests/test_v462_mark_no_interest.py b/tests/test_v462_mark_no_interest.py new file mode 100644 index 0000000..10a2ee2 --- /dev/null +++ b/tests/test_v462_mark_no_interest.py @@ -0,0 +1,36 @@ +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def read(path: str) -> str: + return (ROOT / path).read_text(encoding="utf-8") + + +def test_mark_no_interest_is_available_action(): + catalog = read("app/action_catalog.py") + prompt = read("app/action_prompt.py") + task_service = read("app/task_service.py") + admin = read("app/admin_dashboard.py") + + assert '"MARK_NO_INTEREST"' in catalog + assert '"action": "Marcar sem interesse"' in catalog + assert "não tem veículos elétricos" in prompt + assert '"MARK_NO_INTEREST": ("vendas", "Marcar sem interesse")' in task_service + assert '"MARK_NO_INTEREST": "Marcar sem interesse"' in admin + + +def test_mark_no_interest_has_deterministic_rule(): + decider = read("app/action_decider.py") + assert "detect_deterministic_action" in decider + assert "MARK_NO_INTEREST" in decider + assert "não temos" in decider or "n[aã]o" in decider + assert "deterministic-rule" in decider + + +def test_no_interest_closes_existing_opportunity_only_when_task_done(): + opp = read("app/opportunity_service.py") + assert '"MARK_NO_INTEREST",' in opp + assert '"MARK_NO_INTEREST": "REVIEW"' in opp + assert '"MARK_NO_INTEREST": "NO_INTEREST"' in opp + assert 'if action_code == "MARK_NO_INTEREST" and not existing' in opp diff --git a/tests/test_v463_operations_queue_cleanup.py b/tests/test_v463_operations_queue_cleanup.py new file mode 100644 index 0000000..4c7260c --- /dev/null +++ b/tests/test_v463_operations_queue_cleanup.py @@ -0,0 +1,43 @@ +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def read(path: str) -> str: + return (ROOT / path).read_text(encoding="utf-8") + + +def test_no_interest_is_distinct_from_lost_opportunity(): + opp = read("app/opportunity_service.py") + admin = read("app/admin_dashboard.py") + + assert '"NO_INTEREST": "Sem interesse"' in opp + assert '"MARK_NO_INTEREST": "NO_INTEREST"' in opp + assert '"NO_INTEREST": 118' in opp + assert '("closed", "Fechadas", ["WON", "LOST", "NO_INTEREST"])' in opp + assert '"NO_INTEREST": "Sem interesse"' in admin + assert 'não é o mesmo que oportunidade perdida' in admin + + +def test_operations_queue_filters_manually_cleaned_outbox_noise(): + ops = read("app/operations_service.py") + assert "limpo manualmente" in ops + assert "resolvido manualmente" in ops + assert "_is_manually_resolved_error" in ops + assert "COALESCE(last_error,'') NOT ILIKE '%limpo manualmente%'" in ops + assert "Classificação da mensagem falhou" in ops + + +def test_operations_labels_are_not_inbox_labels(): + operations_page = read("app/admin_ui/pages/operations.py") + assert "Mensagens para revisão" in operations_page + assert "Mensagens sem cliente" in operations_page + assert "Emails por tratar" not in operations_page + assert "Sem cliente', counts.get('communications_without_customer'" not in operations_page + + +def test_ignore_manually_cleaned_outbox_script_exists(): + script = read("scripts/ignore_manually_cleaned_outbox.py") + assert "limpo manualmente" in script + assert "status = 'ignored'" in script + assert "--dry-run" in script diff --git a/tests/test_v465_operations_work_queue.py b/tests/test_v465_operations_work_queue.py new file mode 100644 index 0000000..6eba4dd --- /dev/null +++ b/tests/test_v465_operations_work_queue.py @@ -0,0 +1,38 @@ +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def read(path: str) -> str: + return (ROOT / path).read_text(encoding="utf-8") + + +def test_operations_is_single_work_queue_not_mini_dashboard(): + operations = read("app/admin_ui/pages/operations.py") + assert "Lista única de trabalho do operador" in operations + assert "o que tenho de fazer agora?" in operations + assert "Não é um mini-dashboard técnico" in operations + assert "Outbox que precisa de atenção" not in operations + assert "Documentos que podem exigir ação" not in operations + assert "Produtos sem Artigo Jasmin" not in operations + assert "Clientes incompletos" not in operations + + +def test_operations_has_operator_filters_and_primary_buttons(): + operations = read("app/admin_ui/pages/operations.py") + for label in ["Todas", "Vendas", "Financeiro", "Logística", "Revisão", "Bloqueadas", "Concluídas hoje"]: + assert label in operations + for label in ["Preparar orçamento", "Preparar pró-forma", "Confirmar pagamento", "Rever mensagem", "Associar cliente"]: + assert label in operations + assert "Abrir Chatwoot ↗" in operations + assert "Ver oportunidade" in operations + + +def test_operations_service_returns_compact_queue_counters_and_action_codes(): + ops = read("app/operations_service.py") + assert "work_queue_total" in ops + assert "overdue_tasks" in ops + assert "review_tasks" in ops + assert "blocked_outbox" in ops + assert "t.action_code" in ops and "AS action_code" in ops + assert "upper(io.target_system || '_' || io.action_type) AS action_code" in ops diff --git a/tests/test_v466_safe_opportunity_linking.py b/tests/test_v466_safe_opportunity_linking.py new file mode 100644 index 0000000..9240197 --- /dev/null +++ b/tests/test_v466_safe_opportunity_linking.py @@ -0,0 +1,47 @@ +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def read(path: str) -> str: + return (ROOT / path).read_text(encoding="utf-8") + + +def test_opportunity_linking_uses_conversation_as_strong_match_and_contact_as_weak_match(): + service = read("app/opportunity_service.py") + assert "_find_open_opportunity_by_conversation" in service + assert "conversation_id = :conversation_id" in service + assert "contact_id_unique_recent" in service + assert "CONTACT_MATCH_RECENT_DAYS = 45" in service + assert "len(_open_opportunities_for_contact(task.get(\"contact_id\"), limit=2)) > 1" in service + assert "find_open_opportunity_for_task" in service + old_risky_pattern = "AND ({' OR '.join(filters)})" + assert old_risky_pattern not in service + + +def test_ambiguous_contact_match_does_not_create_new_opportunity_and_marks_task_for_review(): + service = read("app/opportunity_service.py") + assert "has_ambiguous_opportunity_match(task)" in service + assert "mark_task_opportunity_link_ambiguous" in service + assert "multiple_recent_open_opportunities_for_chatwoot_contact" in service + assert "return None" in service + + +def test_task_detail_does_not_link_chatwoot_contact_id_as_local_customer(): + tasks_page = read("app/admin_ui/pages/tasks.py") + task_service = read("app/task_service.py") + assert "linked_customer_id" in task_service + assert "LEFT JOIN customers cu ON cu.id = o.local_customer_id" in task_service + assert "safe_customer_id" in tasks_page + assert "Ver cliente fiscal" in tasks_page + assert 'href="/customers/{esc(contact_id)}"' not in tasks_page + assert "Contacto Chatwoot" in tasks_page + + +def test_operations_surfaces_ambiguous_opportunity_linking_as_operator_work(): + ops = read("app/operations_service.py") + operations_page = read("app/admin_ui/pages/operations.py") + assert "opportunity_linking_status" in ops + assert "ASSOCIATE_OPPORTUNITY" in ops + assert "Confirmar associação da oportunidade" in ops + assert "Associar oportunidade" in operations_page diff --git a/tests/test_v467_chatwoot_autocomplete_guardrails.py b/tests/test_v467_chatwoot_autocomplete_guardrails.py new file mode 100644 index 0000000..6ef209e --- /dev/null +++ b/tests/test_v467_chatwoot_autocomplete_guardrails.py @@ -0,0 +1,40 @@ +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def read(path: str) -> str: + return (ROOT / path).read_text(encoding="utf-8") + + +def test_chatwoot_auto_complete_excludes_payment_and_review_by_default(): + service = read("app/task_service.py") + start = service.index("# Guardrail v4.6.7") + end = service.index('configured = os.getenv("CHATWOOT_AUTO_COMPLETE_ACTION_CODES"', start) + defaults = service[start:end] + assert '"SEND_INFO"' in defaults + assert '"SEND_QUOTE"' in defaults + assert '"SUPPORT"' in defaults + assert '"CONFIRM_PAYMENT"' not in defaults + assert '"MARK_NO_INTEREST"' not in defaults + assert '"REVIEW_MANUALLY"' not in defaults + + +def test_chatwoot_auto_complete_blocks_ambiguous_opportunity_linking(): + service = read("app/task_service.py") + assert "blocked_ambiguous_opportunity" in service + assert "operator_must_confirm_opportunity_before_auto_complete" in service + assert "opportunity_linking_status" in service + + +def test_task_done_records_timeline_event_after_completion(): + service = read("app/task_service.py") + assert "task_auto_completed" in service + assert "Task concluída automaticamente" in service + assert "_record_task_timeline_event(task_id, event_type=timeline_event_type)" in service + + +def test_v467_documentation_exists(): + doc = read("docs/CLIENTFLOW_V467_CHATWOOT_AUTOCOMPLETE_GUARDRAILS.md") + assert "Chatwoot Auto-complete Guardrails" in doc + assert "CONFIRM_PAYMENT" in doc diff --git a/tests/test_v468_operational_safety_consolidation.py b/tests/test_v468_operational_safety_consolidation.py new file mode 100644 index 0000000..1a5d4b0 --- /dev/null +++ b/tests/test_v468_operational_safety_consolidation.py @@ -0,0 +1,60 @@ +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def read(path: str) -> str: + return (ROOT / path).read_text(encoding="utf-8") + + +def test_auto_complete_has_hard_policy_deny_list_even_when_env_is_configured(): + service = read("app/task_service.py") + assert "never_auto_complete_codes" in service + assert '"CONFIRM_PAYMENT"' in service + assert '"CONFIRM_PAYMENT_AND_PREPARE_SHIPMENT"' in service + assert '"REMOVE_FROM_LIST"' in service + assert "blocked_by_policy" in service + assert "ClientFlow auto-complete policy ignored action_codes" in service + + +def test_outbox_worker_claims_rows_with_skip_locked_instead_of_listing_pending(): + service = read("app/integration_outbox_service.py") + script = read("scripts/process_outbox.py") + assert "def claim_pending_outbox" in service + assert "FOR UPDATE SKIP LOCKED" in service + assert "status = 'processing'" in service + assert "locked_at = now()" in service + assert "lock_owner = :lock_owner" in service + assert "claim_pending_outbox" in script + assert "list_pending_outbox" not in script + assert "OUTBOX_WORKER_ID" in script + + +def test_outbox_completion_releases_processing_lock(): + service = read("app/integration_outbox_service.py") + assert "def mark_outbox_sent" in service + assert "locked_at = NULL" in service + assert "lock_owner = NULL" in service + assert "def mark_outbox_failed" in service + + +def test_system_health_exposes_operational_metrics_and_autocomplete_config(): + ops = read("app/operations_service.py") + assert "operational_metrics" in ops + assert "tasks_auto_completed_24h" in ops + assert "ambiguous_opportunity_tasks" in ops + assert "outbox_processing_stale" in ops + assert "chatwoot_auto_complete_action_codes" in ops + + +def test_tasks_route_still_exists_but_is_not_primary_navigation(): + operations_page = read("app/admin_ui/pages/operations.py") + tasks_page = read("app/admin_ui/pages/tasks.py") + navigation = read("app/admin_ui/navigation.py") + primary_start = navigation.index("PRIMARY_NAV_ITEMS") + primary_end = navigation.index("ADMIN_NAV_ITEMS", primary_start) + primary_nav = navigation[primary_start:primary_end] + assert 'NavItem("tasks", "/tasks"' not in primary_nav + assert 'NavItem("tasks", "/tasks", "bi-list-check", "Tasks", "admin")' in navigation + assert '@router.get("/tasks", response_class=HTMLResponse)' in tasks_page + assert "Ver lista completa de tarefas" in operations_page diff --git a/tests/test_v46_chatwoot_workflow.py b/tests/test_v46_chatwoot_workflow.py new file mode 100644 index 0000000..a6d1680 --- /dev/null +++ b/tests/test_v46_chatwoot_workflow.py @@ -0,0 +1,53 @@ +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def read(path: str) -> str: + return (ROOT / path).read_text(encoding="utf-8") + + +def test_review_manually_and_remove_from_list_are_operational_pending_actions(): + task_service = read("app/task_service.py") + action_catalog = read("app/action_catalog.py") + + assert "def _initial_task_status" in task_service + assert 'code in {"IGNORE_SPAM", "NO_ACTION"}' in task_service + assert 'code in {"REVIEW_MANUALLY", "REMOVE_FROM_LIST"}' in task_service + assert '"REVIEW_MANUALLY": {\n "route": "rever",\n "action_required": True' in action_catalog + assert '"REMOVE_FROM_LIST": {\n "route": "marketing",\n "action_required": True' in action_catalog + + +def test_reclassify_review_stays_pending_and_spam_is_skipped(): + task_service = read("app/task_service.py") + assert 'new_status = "skipped" if new_action_code in {"IGNORE_SPAM", "NO_ACTION"} else "pending"' in task_service + assert '"REMOVE_FROM_LIST": ("marketing", "Remover contacto da lista")' in task_service + + +def test_operations_work_items_include_chatwoot_context_and_url(): + operations = read("app/operations_service.py") + operations_page = read("app/admin_ui/pages/operations.py") + + assert "build_chatwoot_conversation_url" in operations + assert "chatwoot_url" in operations + assert "t.source_system" in operations + assert "t.conversation_id" in operations + assert "Chatwoot ↗" in operations_page + assert "O Chatwoot é a inbox" in operations_page + + +def test_communications_not_primary_navigation_but_routes_stay_available(): + navigation = read("app/admin_ui/navigation.py") + communications = read("app/admin_ui/pages/communications.py") + operations = read("app/admin_ui/pages/operations.py") + assert '("communications", "/communications", "bi-envelope", "Comunicações")' not in navigation + assert '@router.get("/communications"' in communications + assert 'Chatwoot continua a ser a inbox' in communications or 'O Chatwoot é a inbox' in operations + + +def test_reopen_chatwoot_review_tasks_script_exists(): + script = read("scripts/reopen_chatwoot_review_tasks.py") + assert "REVIEW_MANUALLY" in script + assert "REMOVE_FROM_LIST" in script + assert "--dry-run" in script + assert "status = 'pending'" in script diff --git a/tests/test_v472_domain_routes.py b/tests/test_v472_domain_routes.py new file mode 100644 index 0000000..25a6f88 --- /dev/null +++ b/tests/test_v472_domain_routes.py @@ -0,0 +1,72 @@ +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def read(path: str) -> str: + return (ROOT / path).read_text(encoding="utf-8") + + +def test_v472_page_modules_exist(): + for module in [ + "dashboard", + "operations", + "opportunities", + "tasks", + "customers", + "products", + "orders", + "finance", + "integrations", + "communications", + "outbox", + "events", + "runs", + "queues", + "conversations", + "system", + ]: + path = ROOT / "app" / "admin_ui" / "pages" / f"{module}.py" + assert path.exists(), f"missing {path}" + content = path.read_text(encoding="utf-8") + assert "router = APIRouter()" in content + + +def test_v472_router_aggregates_domain_modules(): + router = read("app/admin_ui/router.py") + assert "route registration is split by domain" in router.lower() + for module in [ + "dashboard", "operations", "opportunities", "tasks", + "customers", "products", "orders", "finance", "integrations", + "communications", "outbox", "events", "runs", "queues", "system", + ]: + assert f"from app.admin_ui.pages import {module}" in router + assert f"router.include_router({module}.router)" in router + + +def test_v472_routes_are_no_longer_registered_in_admin_dashboard(): + dashboard = read("app/admin_dashboard.py") + assert "Route handlers moved to app.admin_ui.pages.* in v4.7.2" in dashboard + assert dashboard.count("@router.get(") <= 1 # /ui.css compatibility endpoint only + assert "@router.get(\"/operations\"" not in dashboard + assert "@router.get(\"/tasks\"" not in dashboard + assert "@router.get(\"/opportunities\"" not in dashboard + admin_lines = dashboard.count("\n") + 1 + assert admin_lines < 1800 + + +def test_v472_domain_modules_own_expected_routes(): + assert "@router.get(\"/operations\"" in read("app/admin_ui/pages/operations.py") + assert "@router.get(\"/operacoes\"" in read("app/admin_ui/pages/operations.py") + assert "@router.get(\"/opportunities\"" in read("app/admin_ui/pages/opportunities.py") + assert "@router.post(\"/opportunities/{opportunity_id}/stage\"" in read("app/admin_ui/pages/opportunities.py") + assert "@router.get(\"/tasks\"" in read("app/admin_ui/pages/tasks.py") + assert "@router.post(\"/tasks/{task_id}/complete\"" in read("app/admin_ui/pages/tasks.py") + assert "@router.get(\"/outbox\"" in read("app/admin_ui/pages/outbox.py") + assert "@router.get(\"/system/health\"" in read("app/admin_ui/pages/system.py") + + +def test_v472_documentation_exists_and_states_no_db_migration(): + doc = read("docs/CLIENTFLOW_V472_DOMAIN_ROUTES.md") + assert "Rotas por domínio" in doc + assert "Não requer migração de base de dados" in doc diff --git a/tests/test_v473_htmx_partials.py b/tests/test_v473_htmx_partials.py new file mode 100644 index 0000000..acc0c7b --- /dev/null +++ b/tests/test_v473_htmx_partials.py @@ -0,0 +1,54 @@ +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def read(path: str) -> str: + return (ROOT / path).read_text(encoding="utf-8") + + +def test_v473_htmx_helpers_and_labels_exist(): + assert (ROOT / "app/admin_ui/labels.py").exists() + assert (ROOT / "app/admin_ui/htmx.py").exists() + assert (ROOT / "app/admin_ui/view_models/operations.py").exists() + labels = read("app/admin_ui/labels.py") + assert "ACTION_LABELS" in labels + assert '"CONFIRM_PAYMENT": "Confirmar pagamento"' in labels + assert '"ASSOCIATE_OPPORTUNITY": "Associar oportunidade"' in labels + + +def test_v473_operations_has_htmx_work_items_partial(): + operations = read("app/admin_ui/pages/operations.py") + vm = read("app/admin_ui/view_models/operations.py") + assert '@router.get("/operations/partials/work-items"' in operations + assert 'id="operations-work-items"' in operations + assert 'hx-get="{esc(partial_href)}"' in operations + assert 'hx-target="#operations-work-items"' in operations + assert 'hx-push-url' in operations + assert "build_operations_view_model" in vm + assert "operation_item_matches_scope" in vm + + +def test_v473_tasks_has_htmx_list_partial(): + tasks = read("app/admin_ui/pages/tasks.py") + assert '@router.get("/tasks/partials/list"' in tasks + assert 'id="tasks-list"' in tasks + assert 'hx-get="/tasks/partials/list' in tasks + assert 'hx-target="#tasks-list"' in tasks + assert 'hx-push-url' in tasks + + +def test_v473_existing_opportunity_partials_remain_available(): + opportunities = read("app/admin_ui/pages/opportunities.py") + assert '@router.get("/opportunities/{opportunity_id}/partials/jasmin-documents"' in opportunities + assert '@router.get("/opportunities/{opportunity_id}/partials/products"' in opportunities + assert 'hx-target="#jasmin-documents-panel"' in read("app/admin_dashboard.py") + assert 'hx-target="#opportunity-products-panel"' in read("app/admin_dashboard.py") + + +def test_v473_documentation_states_no_db_or_workflow_change(): + doc = read("docs/CLIENTFLOW_V473_HTMX_PARTIALS.md") + assert "Não altera regras de negócio" in doc + assert "Não requer migração" not in doc or "base de dados" in doc + assert "/operations/partials/work-items" in doc + assert "/tasks/partials/list" in doc diff --git a/tests/test_v474_htmx_completion.py b/tests/test_v474_htmx_completion.py new file mode 100644 index 0000000..f3fe1b7 --- /dev/null +++ b/tests/test_v474_htmx_completion.py @@ -0,0 +1,51 @@ +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def read(path: str) -> str: + return (ROOT / path).read_text(encoding="utf-8") + + +def test_v474_opportunities_board_partial_exists_and_is_htmx_targeted(): + opportunities = read("app/admin_ui/pages/opportunities.py") + assert '@router.get("/opportunities/partials/board"' in opportunities + assert 'id="opportunities-board"' in opportunities + assert 'hx-get="/opportunities/partials/board"' in opportunities + assert 'hx-target="#opportunities-board"' in opportunities + assert 'hx-swap="outerHTML"' in opportunities + assert 'hx-push-url' in opportunities + for label in ["Novas", "Orçamento enviado", "Pró-forma enviada", "Pagamento pendente", "Bloqueadas"]: + assert label in opportunities + + +def test_v474_task_detail_partial_and_actions_are_htmx_targeted(): + tasks = read("app/admin_ui/pages/tasks.py") + assert '@router.get("/tasks/{task_id}/partials/detail"' in tasks + assert 'id="task-detail-panel"' in tasks + assert 'hx-target="#task-detail-panel"' in tasks + assert 'hx-swap="outerHTML"' in tasks + assert 'hx-post="/tasks/{esc(task_id)}/complete-with-note"' in tasks + assert 'hx-post="/tasks/{esc(task_id)}/skip"' in tasks + assert 'render_task_detail_partial(task_id, notice="Tarefa concluída.")' in tasks + + +def test_v474_outbox_table_partial_and_actions_are_htmx_targeted(): + outbox = read("app/admin_ui/pages/outbox.py") + assert '@router.get("/outbox/partials/table"' in outbox + assert 'id="outbox-table"' in outbox + assert 'hx-get="{_outbox_partial_url' in outbox + assert 'hx-target="#outbox-table"' in outbox + assert 'hx-swap="outerHTML"' in outbox + assert 'hx-post="/outbox/{oid}/pending"' in outbox + assert 'hx-post="/outbox/{oid}/ignored"' in outbox + + +def test_v474_documentation_states_no_db_or_workflow_change(): + doc = read("docs/CLIENTFLOW_V474_HTMX_COMPLETION.md") + assert "HTMX Completion" in doc + assert "/opportunities/partials/board" in doc + assert "/tasks/{task_id}/partials/detail" in doc + assert "/outbox/partials/table" in doc + assert "Não requer migração de base de dados" in doc + assert "Não altera auto-complete" in doc diff --git a/tests/test_v47_admin_ui_refactor.py b/tests/test_v47_admin_ui_refactor.py new file mode 100644 index 0000000..50d6a64 --- /dev/null +++ b/tests/test_v47_admin_ui_refactor.py @@ -0,0 +1,63 @@ +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def read(path: str) -> str: + return (ROOT / path).read_text(encoding="utf-8") + + +def test_v47_admin_ui_modules_exist_and_dashboard_is_smaller(): + assert (ROOT / "app/admin_ui/styles.py").exists() + assert (ROOT / "app/admin_ui/navigation.py").exists() + assert (ROOT / "app/admin_ui/layout.py").exists() + admin_lines = read("app/admin_dashboard.py").count("\n") + 1 + assert admin_lines < 4500 + assert "ADMIN_UI_CSS moved to app.admin_ui.styles" in read("app/admin_dashboard.py") + + +def test_v47_primary_navigation_is_operator_first_and_admin_is_separate(): + navigation = read("app/admin_ui/navigation.py") + primary = navigation[navigation.index("PRIMARY_NAV_ITEMS"):navigation.index("ADMIN_NAV_ITEMS")] + admin = navigation[navigation.index("ADMIN_NAV_ITEMS"):] + for label in ["Dashboard", "Centro de trabalho", "Oportunidades", "Clientes", "Produtos", "Encomendas", "Financeiro", "Integrações"]: + assert label in primary + for technical in ["Tasks", "Comunicações", "Outbox", "Eventos", "Runs", "Filas", "System health"]: + assert technical not in primary + assert technical in admin + assert "v4.7 Operator Workbench" in navigation + + +def test_v47_layout_uses_shared_navigation_and_css_endpoint_keeps_compatibility(): + layout = read("app/admin_ui/layout.py") + dashboard = read("app/admin_dashboard.py") + styles = read("app/admin_ui/styles.py") + assert "from app.admin_ui.navigation import nav" in layout + assert "from app.admin_ui.styles import ADMIN_UI_V451_CSS" in layout + assert "return Response(ADMIN_UI_V451_CSS" in dashboard + assert "v4.7 admin navigation" in styles + + +def test_v47_opportunities_board_has_direct_pending_task_cta(): + opportunities = read("app/admin_ui/pages/opportunities.py") + assert 'cta_label = "Concluir tarefa pendente" if pending else "Ver oportunidade"' in opportunities + assert "btn-primary" in opportunities + assert "Pipeline comercial com foco na próxima ação" in opportunities + + +def test_v47_documentation_describes_operator_model_and_no_db_migration(): + doc = read("docs/CLIENTFLOW_V47_ADMIN_UI_REFACTOR.md") + assert "Chatwoot" in doc and "Operations" in doc + assert "Admin" in doc and "Outbox" in doc + assert "não requer migração de base de dados" in doc + + +def test_v471_sidebar_admin_menu_can_scroll_when_open_on_short_screens(): + styles = read("app/admin_ui/styles.py") + assert "v4.7.1 admin navigation overflow fix" in styles + assert ".cf-sidebar" in styles + assert "overflow-y:auto" in styles + assert "max-height:100vh" in styles + assert "overscroll-behavior:contain" in styles + assert "scrollbar-gutter:stable" in styles + assert "overflow-y:hidden" in styles # mobile horizontal nav keeps old behavior diff --git a/tests/test_v4801_admin_menu_collapsed.py b/tests/test_v4801_admin_menu_collapsed.py new file mode 100644 index 0000000..d64b57d --- /dev/null +++ b/tests/test_v4801_admin_menu_collapsed.py @@ -0,0 +1,13 @@ +from app.admin_ui.navigation import nav + + +def test_v4801_admin_menu_is_collapsed_by_default_on_primary_pages(): + html = nav("operations") + assert '
    ' in html + assert '
    ' not in html + + +def test_v4801_admin_menu_opens_when_current_page_is_admin_area(): + html = nav("outbox") + assert '
    ' in html + assert '' in html diff --git a/tests/test_v480_operational_automation_audit_recovery.py b/tests/test_v480_operational_automation_audit_recovery.py new file mode 100644 index 0000000..78424bc --- /dev/null +++ b/tests/test_v480_operational_automation_audit_recovery.py @@ -0,0 +1,68 @@ +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def read(path: str) -> str: + return (ROOT / path).read_text(encoding="utf-8") + + +def test_v480_outbox_stale_recovery_service_and_worker_are_present(): + service = read("app/integration_outbox_service.py") + worker = read("scripts/process_outbox.py") + script = read("scripts/recover_stale_outbox.py") + assert "def recover_stale_processing_outbox" in service + assert "OUTBOX_STALE_PROCESSING_MINUTES" in service + assert "OUTBOX_STALE_RECOVERY_MODE" in service + assert "manual_only" in service + assert "retry_pending" in service + assert "mark_failed" in service + assert "recover_stale_processing_outbox" in worker + assert "OUTBOX_RECOVER_STALE_BEFORE_PROCESS" in worker + assert "recover_stale_processing_outbox" in script + + +def test_v480_operator_audit_service_and_sensitive_actions_are_audited(): + audit = read("app/operator_audit_service.py") + tasks = read("app/task_service.py") + outbox_page = read("app/admin_ui/pages/outbox.py") + assert "def record_operator_action" in audit + assert "event_type = 'operator_action'" in audit + assert "task_completed" in tasks + assert "task_skipped" in tasks + assert "task_reclassified" in tasks + assert "outbox_reprocess_requested" in outbox_page + assert "outbox_marked_failed" in outbox_page + assert "outbox_ignored" in outbox_page + + +def test_v480_sensitive_actions_have_htmx_confirmation(): + tasks_page = read("app/admin_ui/pages/tasks.py") + outbox_page = read("app/admin_ui/pages/outbox.py") + assert "hx-confirm=\"Confirmar conclusão desta tarefa?\"" in tasks_page + assert "hx-confirm=\"Ignorar esta tarefa?" in tasks_page + assert "hx-confirm=\"Guardar reclassificação manual desta tarefa?\"" in tasks_page + assert "hx-confirm=\"Reprocessar este item da outbox?\"" in outbox_page + assert "hx-confirm=\"Ignorar este item da outbox?\"" in outbox_page + assert "hx-confirm=\"Marcar este item como failed?\"" in outbox_page + + +def test_v480_operational_health_and_operations_filters_are_expanded(): + ops = read("app/admin_ui/view_models/operations.py") + service = read("app/operations_service.py") + system = read("app/admin_ui/pages/system.py") + assert "Ambíguas" in ops + assert "Atrasadas" in ops + assert "is_overdue" in ops + assert "operator_actions_24h" in service + assert "outbox_stale" in service + assert "Outbox stale" in system + assert "Ações operador 24h" in system + + +def test_v480_documentation_exists(): + doc = read("docs/CLIENTFLOW_V480_OPERATIONAL_AUTOMATION_AUDIT_RECOVERY.md") + assert "Operational Automation, Audit & Recovery" in doc + assert "OUTBOX_STALE_PROCESSING_MINUTES" in doc + assert "operator_action" in doc + assert "hx-confirm" in doc diff --git a/tests/test_v482_production_stabilization.py b/tests/test_v482_production_stabilization.py new file mode 100644 index 0000000..fe44d48 --- /dev/null +++ b/tests/test_v482_production_stabilization.py @@ -0,0 +1,58 @@ +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def read(path: str) -> str: + return (ROOT / path).read_text(encoding="utf-8") + + +def test_v482_guided_operations_helpers_exist(): + guidance = read("app/admin_ui/guidance.py") + assert "def fiscal_contact_panel_html" in guidance + assert "Cliente fiscal e contacto Chatwoot" in guidance + assert "def readiness_checklist_html" in guidance + assert "Prontidão" in guidance + assert "def outbox_operator_message" in guidance + assert "Motivo provável" in read("app/admin_ui/pages/outbox.py") + + +def test_v482_operations_and_opportunities_show_fiscal_context_and_blockers(): + ops = read("app/admin_ui/pages/operations.py") + opps = read("app/admin_ui/pages/opportunities.py") + vm = read("app/admin_ui/view_models/operations.py") + assert "work_item_blockers" in ops + assert "Associações por confirmar" in vm + assert "Bloqueios atuais" in opps + assert "Prontidão para documentos" in opps + assert "Prontidão para envio" in opps + assert "Associação fiscal" in opps + + +def test_v482_task_detail_separates_fiscal_customer_and_contact(): + tasks = read("app/admin_ui/pages/tasks.py") + assert "fiscal_contact_panel_html" in tasks + assert "Prontidão fiscal da tarefa" in tasks + assert "Prontidão mínima antes de documento/envio" in tasks + assert "Corrigir estes dados antes de emitir documento" in tasks + + +def test_v482_system_health_has_production_state_and_data_quality_metrics(): + service = read("app/operations_service.py") + system = read("app/admin_ui/pages/system.py") + assert "open_opportunities_without_fiscal_customer" in service + assert "active_incomplete_fiscal_customers" in service + assert "products_missing_external_code" in service + assert "seconds_since_last_chatwoot_webhook" in service + assert "Estado para operação" in system + assert "Oportunidades sem cliente fiscal" in system + assert "Clientes fiscais incompletos" in system + assert "Último webhook Chatwoot" in system + + +def test_v482_documentation_exists(): + doc = read("docs/CLIENTFLOW_V482_PRODUCTION_STABILIZATION.md") + assert "Production Stabilization" in doc + assert "Cliente fiscal" in doc + assert "Outbox em linguagem operacional" in doc + assert "Sem migração" in doc diff --git a/tests/test_v483_operations_guardrails.py b/tests/test_v483_operations_guardrails.py new file mode 100644 index 0000000..7af74e0 --- /dev/null +++ b/tests/test_v483_operations_guardrails.py @@ -0,0 +1,63 @@ +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def read(path: str) -> str: + return (ROOT / path).read_text(encoding="utf-8") + + +def test_v483_opportunity_creation_guardrails_exist(): + service = read("app/opportunity_service.py") + assert "OPPORTUNITY_CREATE_ACTION_CODES" in service + assert "NEVER_CREATE_OPPORTUNITY_ACTION_CODES" in service + assert "SYSTEM_SENDER_PATTERNS" in service + assert "Mail Delivery" not in service # patterns are normalized lower-case + assert "mail delivery subsystem" in service + assert "returned mail" in service + assert "def can_create_new_opportunity_for_task" in service + assert "system_or_bounce_message" in service + assert "send_info_without_clear_commercial_intent" in service + + +def test_v483_operations_card_is_compact_and_does_not_use_contact_as_fiscal_customer(): + ops_page = read("app/admin_ui/pages/operations.py") + ops_service = read("app/operations_service.py") + vm = read("app/admin_ui/view_models/operations.py") + guidance = read("app/admin_ui/guidance.py") + assert "operation_card_title" in ops_page + assert "operation_card_subtitle" in ops_page + assert "Ver detalhes" not in ops_page + assert "work_item_blockers" in ops_page + assert "Cliente fiscal" in guidance and "show_fiscal" in guidance + assert "COALESCE(cu.name, t.customer_id, t.contact_id, '') AS customer_name" not in ops_service + assert "fiscal_customer_name" in ops_service + assert "contact_display_name" in ops_service + assert "_is_numeric_reference" in vm + assert "Contacto Chatwoot #" in vm + + +def test_v483_blockers_are_journey_aware(): + guidance = read("app/admin_ui/guidance.py") + assert "DOCUMENT_ACTION_CODES" in guidance + assert "stage_requires_fiscal_customer" in guidance + assert "Missing fiscal data is not automatically a blocker at the first contact" in guidance + assert "def work_item_blockers" in guidance + assert "item_requires_fiscal_customer" in guidance + + +def test_v483_priority_demotes_review_marketing_noise(): + vm = read("app/admin_ui/view_models/operations.py") + assert "is_classification_failed" in vm + assert "REMOVE_FROM_LIST" in vm + assert "REVIEW_MANUALLY" in vm + assert "queue == \"marketing\"" in vm + assert "sem oportunidade comercial" in vm + + +def test_v483_cleanup_script_for_bad_opportunities_exists(): + script = read("scripts/cleanup_non_commercial_opportunities.py") + assert "system_or_bounce_created_by_mistake" in script + assert "--apply" in script + assert "Dry-run" in script + assert "mail delivery subsystem" in script diff --git a/tests/test_v484_fiscal_link_consistency.py b/tests/test_v484_fiscal_link_consistency.py new file mode 100644 index 0000000..4868e3b --- /dev/null +++ b/tests/test_v484_fiscal_link_consistency.py @@ -0,0 +1,44 @@ +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def read(path: str) -> str: + return (ROOT / path).read_text(encoding="utf-8") + + +def test_v484_operations_uses_opportunity_fiscal_customer_before_task_customer_id(): + service = read("app/operations_service.py") + assert "LEFT JOIN customers cu_opp ON cu_opp.id = o.local_customer_id" in service + assert "LEFT JOIN customers cu_task ON cu_task.id::text = t.customer_id" in service + assert "COALESCE(cu_opp.name, cu_task.name, '') AS fiscal_customer_name" in service + assert "COALESCE(NULLIF(re.payload->'sender'->>'name',''), NULLIF(re.payload->'sender'->>'email',''), NULLIF(t.contact_id,''), '') AS customer_name" in service + assert "cu.name AS fiscal_customer_name" not in service.split("UNION ALL", 1)[0] + + +def test_v484_operations_blockers_use_actual_fiscal_missing_fields_not_only_association(): + guidance = read("app/admin_ui/guidance.py") + assert "def work_item_fiscal_customer" in guidance + assert "The work queue must not infer a fiscal customer from Chatwoot contact_id" in guidance + assert "for missing in fiscal_customer_missing_fields(customer)" in guidance + assert "Cliente fiscal sem {missing}" in guidance + + +def test_v484_task_detail_does_not_say_no_critical_data_when_fiscal_readiness_has_warnings(): + service = read("app/task_service.py") + tasks = read("app/admin_ui/pages/tasks.py") + assert "cu.street_name AS linked_customer_street_name" in service + assert "cu.postal_zone AS linked_customer_postal_zone" in service + assert "cu.city_name AS linked_customer_city_name" in service + assert "fiscal_missing_labels = fiscal_customer_missing_fields" in tasks + assert "Cliente fiscal: {label}" in tasks + assert "missing_items = list(prep_vm.get(\"missing_fields\") or [])" in tasks + + +def test_v484_opportunity_queries_carry_full_fiscal_customer_fields(): + opp_service = read("app/opportunity_service.py") + guidance = read("app/admin_ui/guidance.py") + assert "c.street_name AS linked_customer_street_name" in opp_service + assert "c.postal_zone AS linked_customer_postal_zone" in opp_service + assert "c.city_name AS linked_customer_city_name" in opp_service + assert "opportunity.get(\"linked_customer_street_name\")" in guidance diff --git a/tests/test_v485_bounce_ignore_card_ux.py b/tests/test_v485_bounce_ignore_card_ux.py new file mode 100644 index 0000000..d78cd14 --- /dev/null +++ b/tests/test_v485_bounce_ignore_card_ux.py @@ -0,0 +1,50 @@ +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def read(path: str) -> str: + return (ROOT / path).read_text(encoding="utf-8") + + +def test_v485_bounce_ndr_is_ignored_without_task_or_opportunity(): + catalog = read("app/action_catalog.py") + decider = read("app/action_decider.py") + analyzer = read("app/analyzer.py") + task_service = read("app/task_service.py") + opportunity = read("app/opportunity_service.py") + + assert "IGNORE_BOUNCE" in catalog + assert "Your message" in decider or "couldn" in decider + assert "Recipient" in decider or "recipient" in decider + assert "Office 365" in decider or "office 365" in decider + assert 'action_result.action_code or "").upper() == "IGNORE_BOUNCE"' in analyzer + assert 'return None' in task_service + assert '"IGNORE_BOUNCE"' in opportunity + assert "unknown to address" in opportunity + + +def test_v485_operations_identity_is_operator_friendly_and_details_are_conditional(): + vm = read("app/admin_ui/view_models/operations.py") + ops = read("app/admin_ui/pages/operations.py") + + assert "operation_has_useful_details" in vm + assert "Contacto sem identificação" in vm + assert "_looks_like_generic_label" in vm + assert "Contacto Chatwoot #" in vm # legacy comment only, not the returned title + assert "operation_has_useful_details(item)" not in ops + assert 'status_chip_html = "" if status_text == "normal"' in ops + + +def test_v485_opportunity_cards_use_specific_cta_and_hide_low_value_fields(): + opportunities = read("app/admin_ui/pages/opportunities.py") + + assert "_opportunity_card_identity" in opportunities + assert "_opportunity_card_next_action" in opportunities + assert "primary_action_label" in opportunities + assert "Ver tarefa pendente" in opportunities + assert "cf-opportunity-card-compact" in opportunities + compact_card = opportunities.split('def _render_opportunity_card', 1)[1].split('def render_opportunities_board_partial', 1)[0] + assert "money_html(opp.get('value_amount'))" not in compact_card + assert "fmt_dt(opp.get('updated_at'))" not in compact_card + assert "pending_chip" not in compact_card diff --git a/tests/test_v486_opportunity_board_layout_cleanup.py b/tests/test_v486_opportunity_board_layout_cleanup.py new file mode 100644 index 0000000..617014c --- /dev/null +++ b/tests/test_v486_opportunity_board_layout_cleanup.py @@ -0,0 +1,36 @@ +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def read(path: str) -> str: + return (ROOT / path).read_text(encoding="utf-8") + + +def test_v486_opportunity_board_uses_clean_non_overlapping_layout(): + opportunities = read("app/admin_ui/pages/opportunities.py") + styles = read("app/admin_ui/styles.py") + + assert "cf-opportunities-board-scroll" in opportunities + assert "cf-opportunities-board-grid" in opportunities + assert "cf-opportunity-stage" in opportunities + assert "cf-opportunity-card-clean" in opportunities + assert "cf-opportunity-next-action" in opportunities + + # Cards no longer repeat stage/priority chips inside each opportunity card. + card_fn = opportunities.split("def _render_opportunity_card", 1)[1].split("def render_opportunities_board_partial", 1)[0] + assert "opportunity_stage_badge" not in card_fn + assert "opportunity_priority_chip" not in card_fn + assert "d-flex justify-content-between" not in card_fn + + # Columns have a minimum width instead of being squeezed until badges overlap. + assert "grid-template-columns: repeat(4, minmax(300px, 1fr))" in styles + assert "overflow-x: auto" in styles + assert "min-width: 1220px" in styles + + +def test_v486_opportunity_copy_describes_relevant_fields_only(): + opportunities = read("app/admin_ui/pages/opportunities.py") + + assert "identificação clara, assunto, próxima ação e bloqueios relevantes" in opportunities + assert "valor e pendências" not in opportunities diff --git a/tests/test_v488_operations_classic_cards.py b/tests/test_v488_operations_classic_cards.py new file mode 100644 index 0000000..34ac719 --- /dev/null +++ b/tests/test_v488_operations_classic_cards.py @@ -0,0 +1,52 @@ +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def read(path: str) -> str: + return (ROOT / path).read_text(encoding="utf-8") + + +def test_v488_operations_returns_to_classic_card_list_layout(): + operations = read("app/admin_ui/pages/operations.py") + styles = read("app/admin_ui/styles.py") + + assert 'id="operations-work-items" class="cf-live-panel"' in operations + assert 'cf-work-card' in operations + assert 'cf-work-section-title' in operations + assert 'cf-work-section' in operations + assert 'cf-workbench-layout' not in operations + assert 'operations-detail-panel' not in operations + assert '@router.get("/operations/partials/item-detail"' not in operations + + assert 'grid-template-columns:minmax(360px,440px) minmax(0,1fr)' not in styles + assert '.cf-work-detail-panel{position:sticky' not in styles + assert '.cf-work-row{' not in styles + + +def test_v488_operations_keeps_simplified_cards_without_inline_details(): + operations = read("app/admin_ui/pages/operations.py") + + assert 'operation_card_title(item)' in operations + assert 'operation_card_subtitle(item)' in operations + assert 'operation_has_useful_details(item)' not in operations + assert 'Cliente fiscal' not in operations.split('def _render_work_item', 1)[1].split('details_html = ""', 1)[0] + assert 'Ver detalhes' not in operations + assert 'operation_origin_label(item)' not in operations + + +def test_v488_operations_keeps_htmx_filter_partial_only(): + operations = read("app/admin_ui/pages/operations.py") + + assert '@router.get("/operations/partials/work-items"' in operations + assert 'hx-target="#operations-work-items"' in operations + assert 'hx-swap="outerHTML"' in operations + assert 'hx-push-url' in operations + + +def test_v488_documentation_states_classic_cards_replaced_workbench(): + doc = read("docs/CLIENTFLOW_V488_OPERATIONS_CLASSIC_CARDS.md") + assert "Operations Classic Cards" in doc + assert "sem painel lateral fixo" in doc + assert "Não requer migração de base de dados" in doc + assert "Não altera regras de negócio" in doc diff --git a/tests/test_v489_operations_card_polish.py b/tests/test_v489_operations_card_polish.py new file mode 100644 index 0000000..978127b --- /dev/null +++ b/tests/test_v489_operations_card_polish.py @@ -0,0 +1,36 @@ +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def read(path: str) -> str: + return (ROOT / path).read_text(encoding="utf-8") + + +def test_v489_operations_cards_remove_details_completely(): + operations = read("app/admin_ui/pages/operations.py") + styles = read("app/admin_ui/styles.py") + + assert "Ver detalhes" not in operations + assert "cf-work-details" not in operations + assert "operation_has_useful_details(item)" not in operations + assert "operation_origin_label(item)" not in operations + assert "cf-work-details" not in styles + + +def test_v489_operations_cards_have_primary_and_secondary_action_hierarchy(): + operations = read("app/admin_ui/pages/operations.py") + styles = read("app/admin_ui/styles.py") + + assert "cf-work-primary-action" in operations + assert "cf-work-secondary-actions" in operations + assert ".cf-work-primary-action" in styles + assert ".cf-work-secondary-actions" in styles + assert "border-top:1px solid #eef2f7" in styles + + +def test_v489_documentation_exists(): + doc = read("docs/CLIENTFLOW_V489_OPERATIONS_CARD_POLISH.md") + assert "Operations Card Polish" in doc + assert "Ver detalhes" in doc + assert "Não requer migração de base de dados" in doc diff --git a/tests/test_v490_operations_noise_cleanup.py b/tests/test_v490_operations_noise_cleanup.py new file mode 100644 index 0000000..a382605 --- /dev/null +++ b/tests/test_v490_operations_noise_cleanup.py @@ -0,0 +1,50 @@ +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def read(path: str) -> str: + return (ROOT / path).read_text(encoding="utf-8") + + +def test_v490_has_shared_noise_detector_and_filters_operations_queue(): + noise = read("app/operation_noise.py") + ops = read("app/operations_service.py") + + assert "SYSTEM_SENDER_PATTERNS" in noise + assert "BOUNCE_NDR_PATTERNS" in noise + assert "Mail Delivery System".casefold() in noise.casefold() + assert "undeliverable" in noise + assert "recipient" in noise + assert "is_noise_operation_item" in ops + assert "is_low_value_no_opportunity_item" in ops + assert "not in the daily work queue" in ops + + +def test_v490_generic_review_and_marketing_are_not_high_priority_by_default(): + task_service = read("app/task_service.py") + vm = read("app/admin_ui/view_models/operations.py") + + assert 'code in {"REVIEW_MANUALLY", "REMOVE_FROM_LIST", "MARK_NO_INTEREST", "IGNORE_SPAM", "NO_ACTION", "IGNORE_BOUNCE"}' in task_service + assert 'return "baixa"' in task_service + assert 'action_code in {"REMOVE_FROM_LIST", "IGNORE_SPAM", "NO_ACTION", "REVIEW_MANUALLY"}' in vm + assert "return False" in vm.split('def is_high_priority', 1)[1].split('def is_blocked', 1)[0] + + +def test_v490_cleanup_script_marks_noise_tasks_skipped_dry_run_by_default(): + script = read("scripts/cleanup_operations_noise.py") + + assert "Dry-run only" in script + assert "--apply" in script + assert "task_skipped_noise_cleanup" in script + assert "cleanup_operations_noise" in script + assert "Marked" in script + assert "Mail Delivery System".casefold() in script.casefold() + + +def test_v490_documentation_exists(): + doc = read("docs/CLIENTFLOW_V490_OPERATIONS_NOISE_CLEANUP.md") + assert "Operations Noise Cleanup" in doc + assert "postmaster" in doc + assert "Mail Delivery" in doc + assert "Não requer migração de base de dados" in doc diff --git a/tests/test_v4911_odoo_name_match_reconciliation.py b/tests/test_v4911_odoo_name_match_reconciliation.py new file mode 100644 index 0000000..ec67bca --- /dev/null +++ b/tests/test_v4911_odoo_name_match_reconciliation.py @@ -0,0 +1,89 @@ +"""v4.9.11 regression tests for Odoo name-only reconciliation. + +Odoo customers may not have VAT/NIF filled in. Reconciliation should still be +able to suggest/link by normalized company name and should not abort the whole +process candidate list when an individual suggestion lookup fails. +""" + +import sys +import types + +if "sqlalchemy" not in sys.modules: + sqlalchemy = types.ModuleType("sqlalchemy") + sqlalchemy.text = lambda sql: sql + sys.modules["sqlalchemy"] = sqlalchemy + +if "app.db" not in sys.modules: + app_db = types.ModuleType("app.db") + app_db.engine = object() + sys.modules["app.db"] = app_db + +if "app.action_catalog" not in sys.modules: + action_catalog = types.ModuleType("app.action_catalog") + action_catalog.get_action_config = lambda code: {"route": "rever", "action": code, "action_required": True, "safe_to_post": False} + sys.modules["app.action_catalog"] = action_catalog + +from app import reconciliation_service as svc + + +def test_company_name_normalization_matches_odoo_without_nif(): + score = svc._name_match_score( + "ELEGANTLEGACY, LDA VALONGO Portugal", + "Elegantlegacy Lda", + "Oportunidade · elegantlegacy", + ) + assert score >= 35 + + +def test_single_odoo_sale_without_nif_becomes_process_candidate(monkeypatch): + monkeypatch.setattr( + svc, + "list_reconciliation_items", + lambda *, status, limit, days: [ + { + "id": "odoo-sale-274", + "source_system": "odoo", + "external_type": "odoo_sale_order", + "title": "Venda Odoo sem oportunidade · S00274", + "customer_name": "ELEGANTLEGACY, LDA", + "customer_tax_id": "", + "customer_email": "", + "document_number": "S00274", + "document_date": "2026-06-02", + "amount": "537.00", + "currency": "EUR", + "operation_suggestions": [], + } + ], + ) + + candidates = svc.list_reconciliation_process_candidates(status="open", days=7, limit=10) + + assert len(candidates) == 1 + candidate = candidates[0] + assert candidate["match_key"] == "name" + assert candidate["confidence"] == "alta" + assert candidate["identity_reason"] == "nome fiscal" + assert candidate["customer_name"] == "ELEGANTLEGACY, LDA" + assert candidate["suggested_stage"] == "ODOO_ORDER_CREATED" + assert candidate["suggested_action"] == "SEND_INVOICE" + assert candidate["item_ids"] == ["odoo-sale-274"] + + +def test_list_process_candidates_is_resilient_when_item_query_fails(monkeypatch): + def fail_listing(*, status, limit, days): + raise RuntimeError("simulated db lookup failure") + + monkeypatch.setattr(svc, "list_reconciliation_items", fail_listing) + + assert svc.list_reconciliation_process_candidates(status="open", days=7, limit=10) == [] + + +def test_person_name_only_is_not_high_confidence_identity(): + confidence, reason = svc._process_group_identity_label( + "name", + [{"customer_name": "Bruno Oliveira", "customer_tax_id": "", "source_system": "odoo"}], + ) + + assert confidence == "média" + assert reason == "nome" diff --git a/tests/test_v4912_odoo_fulfilment_reconciliation.py b/tests/test_v4912_odoo_fulfilment_reconciliation.py new file mode 100644 index 0000000..393779c --- /dev/null +++ b/tests/test_v4912_odoo_fulfilment_reconciliation.py @@ -0,0 +1,129 @@ +"""v4.9.12 tests for fiscal-customer Odoo fulfilment reconstruction. + +The Odoo connector may only provide a reliable company name. Once a sale order +is staged, ClientFlow should still import order fulfilment evidence and rebuild +a useful process timeline: sale -> delivery done -> invoice to issue. +""" + +import sys +import types + +if "sqlalchemy" not in sys.modules: + sqlalchemy = types.ModuleType("sqlalchemy") + sqlalchemy.text = lambda sql: sql + sys.modules["sqlalchemy"] = sqlalchemy + +if "app.db" not in sys.modules: + app_db = types.ModuleType("app.db") + app_db.engine = object() + sys.modules["app.db"] = app_db + +if "app.config" not in sys.modules: + app_config = types.ModuleType("app.config") + app_config.settings = types.SimpleNamespace(odoo_enabled=True) + sys.modules["app.config"] = app_config + +if "app.action_catalog" not in sys.modules: + action_catalog = types.ModuleType("app.action_catalog") + action_catalog.get_action_config = lambda code: {"route": "rever", "action": code, "action_required": True, "safe_to_post": False} + sys.modules["app.action_catalog"] = action_catalog + +from app import external_reconciliation_sync as sync +from app import reconciliation_service as svc + + +class FakeOdooClient: + def search_read(self, model, domain=None, fields=None, *, limit=500, offset=0, order="id asc", context=None): + if model == "sale.order": + return [{ + "id": 274, + "name": "S00274", + "state": "sale", + "date_order": "2026-06-02 11:25:49", + "partner_id": [288, "ELEGANTLEGACY, LDA"], + "amount_total": 537.0, + "currency_id": [125, "EUR"], + "invoice_status": "to invoice", + "order_line": [1001, 1002], + }] + if model == "res.partner": + return [{"id": 288, "name": "ELEGANTLEGACY, LDA", "vat": "", "email": ""}] + if model == "sale.order.line": + return [ + {"id": 1001, "order_id": [274, "S00274"], "product_id": [11, "Wallbox 7.4KW"], "name": "Wallbox 7.4KW", "product_uom_qty": 1.0, "qty_delivered": 1.0, "qty_invoiced": 0.0, "price_unit": 179.0, "price_total": 179.0}, + {"id": 1002, "order_id": [274, "S00274"], "product_id": [12, "Wallbox 11KW"], "name": "Wallbox 11KW", "product_uom_qty": 1.0, "qty_delivered": 1.0, "qty_invoiced": 0.0, "price_unit": 179.0, "price_total": 179.0}, + ] + if model == "stock.picking": + return [{"id": 295, "name": "WH/OUT/00295", "state": "done", "origin": "S00274", "picking_type_id": [1, "Delivery Orders"], "scheduled_date": "2026-06-02 12:25:00", "date_done": "2026-06-02 13:12:00"}] + if model == "mrp.production": + return [] + return [] + + +def test_odoo_sync_enriches_sale_with_lines_pickings_and_fulfilment(monkeypatch): + monkeypatch.setattr(sync.settings, "odoo_enabled", True, raising=False) + + odoo_module = types.ModuleType("app.odoo_client") + odoo_module.OdooClient = lambda: FakeOdooClient() + monkeypatch.setitem(sys.modules, "app.odoo_client", odoo_module) + + staged = [] + monkeypatch.setattr(sync, "upsert_reconciliation_item", lambda **kw: staged.append(kw) or kw) + + result = sync.sync_odoo_reconciliation_candidates(days=7, limit=10) + + assert result["seen"] == 1 + candidate = staged[0] + record = candidate["payload"]["record"] + assert record["partner_external_id"] == 288 + assert record["order_lines"][0]["product_name"] == "Wallbox 7.4KW" + assert record["pickings"][0]["name"] == "WH/OUT/00295" + assert record["fulfilment"]["delivery_done"] is True + assert record["fulfilment"]["invoice_pending"] is True + assert candidate["suggested_action"] == "SEND_INVOICE" + + +def test_process_timeline_includes_delivery_done_and_invoice_pending(): + item = { + "id": "odoo-sale-274", + "source_system": "odoo", + "external_type": "odoo_sale_order", + "document_number": "S00274", + "document_date": "2026-06-02", + "amount": "537.00", + "currency": "EUR", + "payload": { + "record": { + "name": "S00274", + "invoice_status": "to invoice", + "order_lines": [{"product_name": "Wallbox 7.4KW", "qty_delivered": 1.0, "qty_invoiced": 0.0}], + "fulfilment": { + "delivery_done": True, + "invoice_pending": True, + "physical_status": "shipped", + "outgoing_pickings": [{"name": "WH/OUT/00295", "state": "done", "date_done": "2026-06-02 13:12:00"}], + "lines": [{"product_name": "Wallbox 7.4KW", "qty_delivered": 1.0, "qty_invoiced": 0.0}], + }, + } + }, + } + + state = svc.infer_reconciliation_process_state([item]) + labels = [step["label"] for step in state["steps"]] + types = [step["external_type"] for step in state["steps"]] + + assert "Venda/encomenda encontrada no Odoo" in labels + assert "Entrega Odoo concluída" in labels + assert "Fatura por emitir" in labels + assert "odoo_delivery" in types + assert "odoo_invoice_pending" in types + assert state["action_code"] == "SEND_INVOICE" + assert state["stage"] == "SHIPMENT_CREATED" + + +def test_static_hooks_for_partner_mapping_and_window_cleanup_reopen(): + service_source = open("app/reconciliation_service.py", encoding="utf-8").read() + assert "external_customer_mappings" in service_source + assert "mapeamento Odoo confirmado" in service_source + assert "payload - 'window_cleanup'" in service_source + assert "_upsert_odoo_operation_links_from_item" in service_source diff --git a/tests/test_v4913_apply_reconstructed_process.py b/tests/test_v4913_apply_reconstructed_process.py new file mode 100644 index 0000000..b6bb41e --- /dev/null +++ b/tests/test_v4913_apply_reconstructed_process.py @@ -0,0 +1,109 @@ +"""v4.9.13 regression tests for applying reconstructed processes. + +The reconstruction is useful only if confirming a process updates the +opportunity pipeline, Odoo links and commercial lines instead of leaving a loose +reconciliation reference. +""" + +import sys +import types + +if "sqlalchemy" not in sys.modules: + sqlalchemy = types.ModuleType("sqlalchemy") + sqlalchemy.text = lambda sql: sql + sys.modules["sqlalchemy"] = sqlalchemy + +if "app.db" not in sys.modules: + app_db = types.ModuleType("app.db") + app_db.engine = object() + sys.modules["app.db"] = app_db + +if "app.action_catalog" not in sys.modules: + action_catalog = types.ModuleType("app.action_catalog") + action_catalog.get_action_config = lambda code: { + "route": "financeiro" if code == "SEND_INVOICE" else "rever", + "action": "Enviar fatura ao cliente" if code == "SEND_INVOICE" else code, + "action_required": True, + "safe_to_post": False, + } + sys.modules["app.action_catalog"] = action_catalog + +from app import reconciliation_service as svc + + +def _odoo_item_with_delivery(): + return { + "id": "odoo-sale-274", + "source_system": "odoo", + "external_type": "odoo_sale_order", + "external_id": "274", + "document_number": "S00274", + "document_date": "2026-06-02", + "amount": "537.00", + "currency": "EUR", + "payload": { + "record": { + "id": 274, + "name": "S00274", + "partner_external_id": 288, + "partner_name": "ELEGANTLEGACY, LDA", + "invoice_status": "to invoice", + "fulfilment": { + "physical_status": "shipped", + "delivery_done": True, + "invoice_pending": True, + "lines": [ + {"id": 643, "product_name": "Wallbox 7.4KW", "qty_ordered": 1.0, "qty_delivered": 1.0, "qty_invoiced": 0.0, "price_unit": 160.0, "price_total": 160.0}, + {"id": 644, "product_name": "Wallbox 11KW", "qty_ordered": 1.0, "qty_delivered": 1.0, "qty_invoiced": 0.0, "price_unit": 178.0, "price_total": 178.0}, + {"id": 645, "product_name": "Wallbox 22KW", "qty_ordered": 1.0, "qty_delivered": 1.0, "qty_invoiced": 0.0, "price_unit": 199.0, "price_total": 199.0}, + {"id": 646, "product_name": "[Delivery_007] Standard delivery", "qty_ordered": 1.0, "qty_delivered": 0.0, "qty_invoiced": 0.0, "price_unit": 0.0, "price_total": 0.0}, + ], + "outgoing_pickings": [{"name": "WH/OUT/00295", "state": "done", "date_done": "2026-06-02 12:12:10"}], + "productions": [{"name": "WH/MO/00477", "state": "done", "product_name": "Wallbox 7.4KW"}], + }, + } + }, + } + + +def test_odoo_importable_lines_skip_zero_delivery_carrier_line(): + lines = svc._odoo_importable_lines(_odoo_item_with_delivery()) + + assert [line["product_name"] for line in lines] == [ + "Wallbox 7.4KW", + "Wallbox 11KW", + "Wallbox 22KW", + ] + assert sum(float(line["price_total"]) for line in lines) == 537.0 + + +def test_process_state_for_delivered_odoo_sale_points_to_invoice(): + state = svc.infer_reconciliation_process_state([_odoo_item_with_delivery()]) + + assert state["stage"] == "SHIPMENT_CREATED" + assert state["action_code"] == "SEND_INVOICE" + assert [step["external_type"] for step in state["steps"]] == [ + "odoo_sale_order", + "odoo_sale_lines", + "odoo_delivery", + "odoo_invoice_pending", + ] + + +def test_apply_reconstructed_process_hooks_are_present(): + source = open("app/reconciliation_service.py", encoding="utf-8").read() + + assert "def _apply_reconstructed_process_to_opportunity" in source + assert "_apply_reconstructed_process_to_opportunity(conn, items, opportunity_id" in source + assert "def _upsert_opportunity_items_from_odoo_item" in source + assert "metadata->>'source_line_id'" in source + assert "system, external_type)" in source and "physical_validation" in source + assert "Venda Odoo" not in source or "odoo_invoice_pending" in source + + +def test_reconciliation_ui_supports_active_days_window(): + ui = open("app/admin_ui/pages/reconciliation.py", encoding="utf-8").read() + + assert "days: Optional[int] = 3" in ui + assert "Hoje" in ui and "7 dias" in ui and "30 dias" in ui + assert "sync_odoo_reconciliation_candidates(limit=50, days=days)" in ui diff --git a/tests/test_v4914_reconciliation_link_hotfix.py b/tests/test_v4914_reconciliation_link_hotfix.py new file mode 100644 index 0000000..ab8d50b --- /dev/null +++ b/tests/test_v4914_reconciliation_link_hotfix.py @@ -0,0 +1,19 @@ +from pathlib import Path + + +def test_v4914_schema_guards_for_process_link_columns(): + source = Path("app/reconciliation_service.py").read_text() + for fragment in [ + "ALTER TABLE operation_links ADD COLUMN IF NOT EXISTS last_synced_at", + "ALTER TABLE opportunity_events ADD COLUMN IF NOT EXISTS action_code", + "ALTER TABLE opportunity_events ADD COLUMN IF NOT EXISTS to_stage", + "ALTER TABLE opportunity_items ADD COLUMN IF NOT EXISTS total_price", + "ALTER TABLE opportunities ADD COLUMN IF NOT EXISTS metadata", + ]: + assert fragment in source + + +def test_v4914_reconciliation_link_route_is_guarded(): + source = Path("app/admin_ui/pages/reconciliation.py").read_text() + assert 'logger.exception("failed to link reconstructed reconciliation process' in source + assert 'Erro%20ao%20ligar%20processo' in source diff --git a/tests/test_v4915_reconciliation_amount_cast.py b/tests/test_v4915_reconciliation_amount_cast.py new file mode 100644 index 0000000..0850e8e --- /dev/null +++ b/tests/test_v4915_reconciliation_amount_cast.py @@ -0,0 +1,8 @@ +from pathlib import Path + + +def test_v4915_reconstruction_amount_update_casts_numeric_parameter(): + source = Path("app/reconciliation_service.py").read_text() + assert "WHEN CAST(:amount AS NUMERIC) IS NOT NULL" in source + assert "THEN CAST(:amount AS NUMERIC)" in source + assert "WHEN :amount IS NOT NULL" not in source diff --git a/tests/test_v491_operational_reconciliation.py b/tests/test_v491_operational_reconciliation.py new file mode 100644 index 0000000..61e8f02 --- /dev/null +++ b/tests/test_v491_operational_reconciliation.py @@ -0,0 +1,75 @@ +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def read(path: str) -> str: + return (ROOT / path).read_text(encoding="utf-8") + + +def test_v491_reconciliation_service_adds_staging_schema_and_payment_proofs(): + service = read("app/reconciliation_service.py") + db = read("app/db.py") + + assert "CREATE TABLE IF NOT EXISTS reconciliation_items" in service + assert "CREATE TABLE IF NOT EXISTS payment_proofs" in service + assert "source_system TEXT NOT NULL" in service + assert "external_type TEXT NOT NULL" in service + assert "ux_reconciliation_items_source_external" in service + assert "ensure_reconciliation_schema" in db + + +def test_v491_reconciliation_page_and_navigation_exist(): + page = read("app/admin_ui/pages/reconciliation.py") + router = read("app/admin_ui/router.py") + nav = read("app/admin_ui/navigation.py") + + assert '@router.get("/reconciliation"' in page + assert '@router.get("/reconciliacao"' in page + assert "Registar pedido externo" in page + assert "Adicionar comprovativo" in page + assert "Criar oportunidade" in page + assert "Reconciliação" in nav + assert "router.include_router(reconciliation.router)" in router + + +def test_v491_external_intake_creates_opportunity_without_fake_chatwoot_ids(): + service = read("app/reconciliation_service.py") + + assert "def create_external_request" in service + assert "manual_external_intake" in service + assert "source_channel" in service + assert "WhatsApp" in read("app/admin_ui/pages/reconciliation.py") + assert "conversation_id" not in service.split("def create_external_request", 1)[1].split("def create_payment_proof", 1)[0] + + +def test_v491_payment_proof_never_confirms_payment_automatically(): + service = read("app/reconciliation_service.py") + page = read("app/admin_ui/pages/reconciliation.py") + doc = read("docs/CLIENTFLOW_V491_OPERATIONAL_RECONCILIATION.md") + + assert "pending_validation" in service + assert "Comprovativo recebido não confirma pagamento" in page + assert "CONFIRM_PAYMENT" in service + assert "não confirma pagamento" in doc + assert "payment_confirmed" not in service + + +def test_v491_sync_script_creates_reconciliation_candidates_idempotently(): + script = read("scripts/sync_reconciliation_candidates.py") + service = read("app/reconciliation_service.py") + + assert "sync_local_documents_without_opportunity" in script + assert "commercial_documents" in service + assert "opportunity_id IS NULL" in service + assert "reconcile:commercial_document" in service + assert "created_or_updated" in script + + +def test_v491_documentation_describes_operator_confirmation_model(): + doc = read("docs/CLIENTFLOW_V491_OPERATIONAL_RECONCILIATION.md") + + assert "Operational Reconciliation" in doc + assert "Operador confirma" in doc + assert "Não confirma pagamentos automaticamente" in doc + assert "Não usa LLM" in doc diff --git a/tests/test_v4920_reconciliation_multiple_orders.py b/tests/test_v4920_reconciliation_multiple_orders.py new file mode 100644 index 0000000..8b33c9f --- /dev/null +++ b/tests/test_v4920_reconciliation_multiple_orders.py @@ -0,0 +1,104 @@ +"""v4.9.20: one fiscal customer can have multiple simultaneous orders. + +Customer identity (NIF/fiscal name) must not be confused with commercial +operation identity. Two Odoo sale orders for the same company should be shown +as two process candidates, not collapsed into one card. +""" + +import sys +import types + +if "sqlalchemy" not in sys.modules: + sqlalchemy = types.ModuleType("sqlalchemy") + sqlalchemy.text = lambda sql: sql + sys.modules["sqlalchemy"] = sqlalchemy + +if "app.db" not in sys.modules: + app_db = types.ModuleType("app.db") + app_db.engine = object() + sys.modules["app.db"] = app_db + +if "app.action_catalog" not in sys.modules: + action_catalog = types.ModuleType("app.action_catalog") + action_catalog.get_action_config = lambda code: {"route": "rever", "action": code, "action_required": True, "safe_to_post": False} + sys.modules["app.action_catalog"] = action_catalog + +from app import reconciliation_service as svc + + +def _odoo_sale(id_, number, amount, product): + return { + "id": id_, + "source_system": "odoo", + "external_type": "odoo_sale_order", + "title": f"Venda Odoo sem oportunidade · {number}", + "customer_name": "ACZCO BRAGA ENERGY, LDA", + "customer_tax_id": "517249200", + "document_number": number, + "document_date": "2026-06-05", + "amount": amount, + "currency": "EUR", + "payload": { + "record": { + "name": number, + "invoice_status": "to invoice", + "fulfilment": { + "invoice_pending": True, + "lines": [{"id": f"line-{number}", "product_name": product, "qty_ordered": 1, "price_total": amount}], + }, + } + }, + "operation_suggestions": [], + } + + +def test_same_company_two_odoo_orders_become_two_process_candidates(monkeypatch): + monkeypatch.setattr( + svc, + "list_reconciliation_items", + lambda *, status, limit, days: [ + _odoo_sale("sale-279", "S00279", "638.60", "Wallbox 7.4KW"), + _odoo_sale("sale-280", "S00280", "190.00", "Cabo"), + ], + ) + + candidates = svc.list_reconciliation_process_candidates(status="open", days=7, limit=10) + + assert len(candidates) == 2 + process_keys = {candidate["process_key"] for candidate in candidates} + assert any("S00279" in key for key in process_keys) + assert any("S00280" in key for key in process_keys) + assert {tuple(candidate["item_ids"]) for candidate in candidates} == {("sale-279",), ("sale-280",)} + assert all(candidate["match_key"] == "nif" for candidate in candidates) + assert all(candidate["suggested_action"] == "SEND_INVOICE" for candidate in candidates) + + +def test_loose_quote_assigns_to_matching_order_by_amount(monkeypatch): + monkeypatch.setattr( + svc, + "list_reconciliation_items", + lambda *, status, limit, days: [ + { + "id": "quote-279", + "source_system": "jasmin", + "external_type": "jasmin_quotation", + "title": "Orçamento Jasmin sem oportunidade · ORC.ORC2026.154", + "customer_name": "ACZCO BRAGA ENERGY, LDA", + "customer_tax_id": "517249200", + "document_number": "ORC.ORC2026.154", + "document_date": "2026-06-05", + "amount": "638.60", + "currency": "EUR", + "operation_suggestions": [], + }, + _odoo_sale("sale-279", "S00279", "638.60", "Wallbox 7.4KW"), + _odoo_sale("sale-280", "S00280", "190.00", "Cabo"), + ], + ) + + candidates = svc.list_reconciliation_process_candidates(status="open", days=7, limit=10) + + assert len(candidates) == 2 + grouped = {candidate["operation_key"]: set(candidate["item_ids"]) for candidate in candidates} + assert grouped["odoo_sale_order:S00279"] == {"quote-279", "sale-279"} + assert grouped["odoo_sale_order:S00280"] == {"sale-280"} diff --git a/tests/test_v4920_reconciliation_multiple_purchases.py b/tests/test_v4920_reconciliation_multiple_purchases.py new file mode 100644 index 0000000..a92cc32 --- /dev/null +++ b/tests/test_v4920_reconciliation_multiple_purchases.py @@ -0,0 +1,112 @@ +"""v4.9.20: fiscal customer identity is not purchase identity.""" + +import sys +import types + +if "sqlalchemy" not in sys.modules: + sqlalchemy = types.ModuleType("sqlalchemy") + sqlalchemy.text = lambda sql: sql + sys.modules["sqlalchemy"] = sqlalchemy + +if "app.db" not in sys.modules: + app_db = types.ModuleType("app.db") + app_db.engine = object() + sys.modules["app.db"] = app_db + +if "app.action_catalog" not in sys.modules: + action_catalog = types.ModuleType("app.action_catalog") + action_catalog.get_action_config = lambda code: {"route": "rever", "action": code, "action_required": True, "safe_to_post": False} + sys.modules["app.action_catalog"] = action_catalog + +from app import reconciliation_service as svc + + +def _jasmin_doc(id_, external_type, number, date, amount): + return { + "id": id_, + "source_system": "jasmin", + "external_type": external_type, + "title": f"Documento Jasmin · {number}", + "customer_name": "ACZCO BRAGA ENERGY, LDA", + "customer_tax_id": "517249200", + "document_number": number, + "document_date": date, + "amount": amount, + "currency": "EUR", + "operation_suggestions": [], + } + + +def test_same_company_two_jasmin_quotes_stay_as_two_purchase_candidates(monkeypatch): + monkeypatch.setattr( + svc, + "list_reconciliation_items", + lambda *, status, limit, days: [ + _jasmin_doc("quote-154", "jasmin_quotation", "ORC.ORC2026.154", "2026-06-03", "638.60"), + _jasmin_doc("quote-160", "jasmin_quotation", "ORC.ORC2026.160", "2026-06-06", "190.00"), + ], + ) + + candidates = svc.list_reconciliation_process_candidates(status="open", days=7, limit=10) + + assert len(candidates) == 2 + keys = {candidate["operation_key"] for candidate in candidates} + assert "jasmin_quotation:ORC.ORC2026.154" in keys + assert "jasmin_quotation:ORC.ORC2026.160" in keys + assert {tuple(candidate["item_ids"]) for candidate in candidates} == {("quote-154",), ("quote-160",)} + + +def test_quote_and_invoice_with_same_amount_and_near_date_can_form_one_purchase(monkeypatch): + monkeypatch.setattr( + svc, + "list_reconciliation_items", + lambda *, status, limit, days: [ + _jasmin_doc("quote-154", "jasmin_quotation", "ORC.ORC2026.154", "2026-06-03", "638.60"), + _jasmin_doc("invoice-80", "jasmin_invoice", "FA2026.80", "2026-06-05", "638.60"), + _jasmin_doc("quote-160", "jasmin_quotation", "ORC.ORC2026.160", "2026-06-20", "190.00"), + ], + ) + + candidates = svc.list_reconciliation_process_candidates(status="open", days=30, limit=10) + + grouped = {candidate["operation_key"]: set(candidate["item_ids"]) for candidate in candidates} + assert grouped["jasmin_quotation:ORC.ORC2026.154"] == {"quote-154", "invoice-80"} + assert grouped["jasmin_quotation:ORC.ORC2026.160"] == {"quote-160"} + + +def test_payment_proof_assigns_only_when_order_match_is_unambiguous(monkeypatch): + order_a = { + "id": "sale-279", + "source_system": "odoo", + "external_type": "odoo_sale_order", + "title": "Venda Odoo sem oportunidade · S00279", + "customer_name": "ACZCO BRAGA ENERGY, LDA", + "customer_tax_id": "517249200", + "document_number": "S00279", + "document_date": "2026-06-05", + "amount": "638.60", + "currency": "EUR", + "payload": {"record": {"name": "S00279", "invoice_status": "to invoice"}}, + "operation_suggestions": [], + } + order_b = {**order_a, "id": "sale-280", "document_number": "S00280", "amount": "190.00", "payload": {"record": {"name": "S00280", "invoice_status": "to invoice"}}} + proof = { + "id": "proof-279", + "source_system": "email", + "external_type": "payment_proof", + "title": "Comprovativo recebido", + "customer_name": "ACZCO BRAGA ENERGY, LDA", + "customer_tax_id": "517249200", + "document_number": "comprovativo-279", + "document_date": "2026-06-06", + "amount": "638.60", + "currency": "EUR", + "operation_suggestions": [], + } + monkeypatch.setattr(svc, "list_reconciliation_items", lambda *, status, limit, days: [order_a, order_b, proof]) + + candidates = svc.list_reconciliation_process_candidates(status="open", days=7, limit=10) + + grouped = {candidate["operation_key"]: set(candidate["item_ids"]) for candidate in candidates} + assert grouped["odoo_sale_order:S00279"] == {"sale-279", "proof-279"} + assert grouped["odoo_sale_order:S00280"] == {"sale-280"} diff --git a/tests/test_v4922_reconciliation_rebuild_button.py b/tests/test_v4922_reconciliation_rebuild_button.py new file mode 100644 index 0000000..62b2383 --- /dev/null +++ b/tests/test_v4922_reconciliation_rebuild_button.py @@ -0,0 +1,29 @@ +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def read(path: str) -> str: + return (ROOT / path).read_text(encoding="utf-8") + + +def test_reconciliation_page_has_safe_rebuild_button(): + page = read("app/admin_ui/pages/reconciliation.py") + service = read("app/reconciliation_service.py") + + assert '/reconciliation/rebuild' in page + assert 'Apagar e correr novamente' in page + assert 'Não apaga clientes, oportunidades nem documentos reais' in page + assert 'reset_generated_reconciliation_items' in service + assert 'opportunity_id IS NULL' in service + assert 'CREATE TABLE {backup_table} AS SELECT * FROM reconciliation_items' in service + + +def test_rebuild_endpoint_resets_then_syncs_external_candidates(): + page = read("app/admin_ui/pages/reconciliation.py") + + rebuild_block = page.split('@router.post("/reconciliation/rebuild")', 1)[1].split('@router.post("/reconciliation/cleanup-window")', 1)[0] + assert 'reset_generated_reconciliation_items' in rebuild_block + assert 'apply=True' in rebuild_block + assert 'sync_all_external_reconciliation_candidates' in rebuild_block + assert 'limit=200' in rebuild_block diff --git a/tests/test_v4923_reconciliation_customer_first_pipeline.py b/tests/test_v4923_reconciliation_customer_first_pipeline.py new file mode 100644 index 0000000..1ed6e99 --- /dev/null +++ b/tests/test_v4923_reconciliation_customer_first_pipeline.py @@ -0,0 +1,49 @@ +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def read(path: str) -> str: + return (ROOT / path).read_text(encoding="utf-8") + + +def test_v4923_reconciliation_sync_runs_customer_seed_before_documents(): + sync = read("app/external_reconciliation_sync.py") + block = sync.split("async def sync_all_external_reconciliation_candidates", 1)[1] + + assert "sync_external_fiscal_customers_for_reconciliation" in sync + assert "sync_jasmin_fiscal_customers_for_reconciliation" in sync + assert "sync_odoo_fiscal_customers_for_reconciliation" in sync + assert block.index("customer_result = await sync_external_fiscal_customers_for_reconciliation") < block.index("jasmin_result = await sync_jasmin_reconciliation_candidates") + assert block.index("jasmin_result = await sync_jasmin_reconciliation_candidates") < block.index("odoo_result = await asyncio.to_thread(sync_odoo_reconciliation_candidates") + assert "customer_seen" in block + assert "customers_created_or_updated" in block + + +def test_v4923_candidates_are_linked_to_seeded_fiscal_customer(): + sync = read("app/external_reconciliation_sync.py") + + assert "_upsert_fiscal_customer_from_external_record(record, source_system=\"jasmin\")" in sync + assert "_upsert_fiscal_customer_from_external_record(record, source_system=\"odoo\")" in sync + assert "\"customer_id\": customer_id" in sync + assert "source_system == \"odoo\" and not (tax_id or is_company or _looks_like_company_name(name))" in sync + + +def test_v4923_customer_upsert_uses_fiscal_name_when_nif_missing(): + service = read("app/commercial_service.py") + + assert "def normalize_fiscal_name" in service + assert "No domínio ClientFlow o nome fiscal de empresa é tratado como chave" in service + assert "normalized_name = normalize_fiscal_name" in service + assert "regexp_replace(lower(name)" in service + assert "metadata = COALESCE(metadata, '{}'::jsonb) || CAST(:metadata AS JSONB)" in service + + +def test_v4923_ui_explains_sequential_rebuild_pipeline(): + page = read("app/admin_ui/pages/reconciliation.py") + script = read("scripts/sync_external_reconciliation.py") + + assert "clientes fiscais → documentos Jasmin → vendas Odoo" in page + assert "clientes fiscais analisados" in page + assert "clientes criados/atualizados" in page + assert "--customers" in script diff --git a/tests/test_v4924_reconciliation_process_review.py b/tests/test_v4924_reconciliation_process_review.py new file mode 100644 index 0000000..f04e9a1 --- /dev/null +++ b/tests/test_v4924_reconciliation_process_review.py @@ -0,0 +1,96 @@ +"""v4.9.24: reconciliation process review must be explainable and actionable.""" + +import sys +import types +from pathlib import Path + +if "sqlalchemy" not in sys.modules: + sqlalchemy = types.ModuleType("sqlalchemy") + sqlalchemy.text = lambda sql: sql + sys.modules["sqlalchemy"] = sqlalchemy + +if "app.db" not in sys.modules: + app_db = types.ModuleType("app.db") + app_db.engine = object() + sys.modules["app.db"] = app_db + +if "app.action_catalog" not in sys.modules: + action_catalog = types.ModuleType("app.action_catalog") + action_catalog.get_action_config = lambda code: {"route": "rever", "action": code, "action_required": True, "safe_to_post": False} + sys.modules["app.action_catalog"] = action_catalog + +from app import reconciliation_service as svc + +ROOT = Path(__file__).resolve().parents[1] + + +def read(path: str) -> str: + return (ROOT / path).read_text(encoding="utf-8") + + +def _item(id_, external_type, number, date, amount): + return { + "id": id_, + "source_system": "jasmin" if external_type.startswith("jasmin") else "odoo", + "external_type": external_type, + "title": f"{external_type} · {number}", + "customer_name": "ACZCO BRAGA ENERGY, LDA", + "customer_tax_id": "517249200", + "document_number": number, + "document_date": date, + "amount": amount, + "currency": "EUR", + "payload": {"record": {"name": number, "invoice_status": "to invoice"}} if external_type == "odoo_sale_order" else {}, + "operation_suggestions": [], + } + + +def test_process_candidates_explain_reasons_and_risks(monkeypatch): + monkeypatch.setattr( + svc, + "list_reconciliation_items", + lambda *, status, limit, days: [ + _item("quote-154", "jasmin_quotation", "ORC.ORC2026.154", "2026-06-03", "638.60"), + _item("sale-279", "odoo_sale_order", "S00279", "2026-06-05", "638.60"), + ], + ) + + candidates = svc.list_reconciliation_process_candidates(status="open", days=7, limit=10) + + assert len(candidates) == 1 + candidate = candidates[0] + assert candidate["review_status"] == "needs_review" + assert "cliente fiscal por NIF exato" in candidate["reasons"] + assert "valor igual entre documentos" in candidate["reasons"] + assert "datas próximas" in candidate["reasons"] + assert "venda Odoo sem fatura Jasmin associada" in candidate["risks"] + + +def test_process_candidates_flag_conflict_when_amounts_differ(monkeypatch): + monkeypatch.setattr( + svc, + "list_reconciliation_items", + lambda *, status, limit, days: [ + _item("quote-154", "jasmin_quotation", "ORC.ORC2026.154", "2026-06-03", "638.60"), + _item("invoice-80", "jasmin_invoice", "FA2026.80", "2026-06-05", "190.00"), + ], + ) + + candidates = svc.list_reconciliation_process_candidates(status="open", days=7, limit=10) + + assert len(candidates) == 2 # different amounts keep separate purchase anchors + assert all("reasons" in candidate and "risks" in candidate for candidate in candidates) + + +def test_v4924_schema_and_ui_have_decisions_review_and_history_actions(): + service = read("app/reconciliation_service.py") + page = read("app/admin_ui/pages/reconciliation.py") + styles = read("app/admin_ui/styles.py") + + assert "CREATE TABLE IF NOT EXISTS reconciliation_decisions" in service + assert "def _candidate_reasons_and_risks" in service + assert "reasons" in service and "risks" in service and "review_status" in service + assert "/reconciliation/processes/needs-review" in page + assert "/reconciliation/processes/historical" in page + assert "Motivos" in page and "Riscos" in page + assert "cf-reconcile-explain" in styles diff --git a/tests/test_v4925_2_odoo_customer_field_compat.py b/tests/test_v4925_2_odoo_customer_field_compat.py new file mode 100644 index 0000000..f5faaee --- /dev/null +++ b/tests/test_v4925_2_odoo_customer_field_compat.py @@ -0,0 +1,24 @@ +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def read(path: str) -> str: + return (ROOT / path).read_text(encoding="utf-8") + + +def test_v4925_2_odoo_customer_sync_discovers_schema_before_search_read(): + sync = read("app/external_reconciliation_sync.py") + assert "def _odoo_model_field_names" in sync + assert "fields_get" in sync + assert "def _odoo_partner_fields_for_available_schema" in sync + assert "def _odoo_partner_domains_for_available_schema" in sync + assert "ignored_optional_fields" in sync + + +def test_v4925_2_mobile_is_optional_for_odoo_partner_sync(): + sync = read("app/external_reconciliation_sync.py") + helper = sync.split("def _odoo_partner_fields_for_available_schema", 1)[1].split("def _odoo_partner_domains_for_available_schema", 1)[0] + assert '"mobile"' in helper + assert 'if field != "mobile"' in helper + assert "Invalid field 'mobile'" in sync diff --git a/tests/test_v4925_3_odoo_customer_identity_guard.py b/tests/test_v4925_3_odoo_customer_identity_guard.py new file mode 100644 index 0000000..61e048c --- /dev/null +++ b/tests/test_v4925_3_odoo_customer_identity_guard.py @@ -0,0 +1,26 @@ +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def read(path: str) -> str: + return (ROOT / path).read_text(encoding="utf-8") + + +def test_v4925_3_odoo_sale_order_reference_never_becomes_customer_name(): + sync = read("app/external_reconciliation_sync.py") + assert "def _looks_like_odoo_sale_order_reference" in sync + assert "S\\d{4,}" in sync + assert "def _odoo_fiscal_customer_name" in sync + seed = sync.split("def _customer_seed_data_from_record", 1)[1].split("def _upsert_fiscal_customer_from_external_record", 1)[0] + assert 'source_system == "odoo"' in seed + assert "_odoo_fiscal_customer_name(record)" in seed + assert "Never create/update a fiscal customer with an Odoo sale order reference" in seed + + +def test_v4925_3_includes_repair_script_for_existing_s00_customers(): + script = read("scripts/repair_odoo_sale_order_customer_names.py") + assert "Dry-run by default" in script + assert "metadata->>'source_system' = 'odoo'" in script + assert "partner_name" in script + assert "--apply" in script diff --git a/tests/test_v4925_fiscal_enrichment_worker.py b/tests/test_v4925_fiscal_enrichment_worker.py new file mode 100644 index 0000000..21a1fcd --- /dev/null +++ b/tests/test_v4925_fiscal_enrichment_worker.py @@ -0,0 +1,57 @@ +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def read(path: str) -> str: + return (ROOT / path).read_text(encoding="utf-8") + + +def test_v4925_has_autonomous_fiscal_enrichment_service_and_schema(): + service = read("app/fiscal_enrichment_service.py") + assert "def ensure_fiscal_enrichment_schema" in service + assert "external_company_cache" in service + assert "fiscal_customer_suggestions" in service + assert "fiscal_enrichment_runs" in service + assert "def enrich_open_opportunities" in service + assert "def enrich_opportunity" in service + assert "contactos/search" in service + assert "empresas/domain" in service + assert "empresas/nif" in service + assert "POST" in service or "_http_json(\"POST\"" in service or "batch" in service + + +def test_v4925_pipeline_runs_enrichment_before_reconciliation_sync(): + sync = read("app/external_reconciliation_sync.py") + block = sync.split("async def sync_all_external_reconciliation_candidates", 1)[1] + assert "enrich_open_opportunities" in block + assert block.index("enrich_open_opportunities") < block.index("sync_external_fiscal_customers_for_reconciliation") + assert "enrichment_auto_applied" in block + assert "enrichment_suggested" in block + + +def test_v4925_opportunity_creation_triggers_best_effort_enrichment(): + service = read("app/opportunity_service.py") + block = service.split("def upsert_opportunity_for_task", 1)[1].split("def advance_opportunity_after_task_done", 1)[0] + assert "from app.fiscal_enrichment_service import enrich_opportunity" in block + assert "enrich_opportunity(opportunity_id, apply_safe=True)" in block + assert "except Exception" in block + + +def test_v4925_ui_exposes_enrichment_actions(): + opp = read("app/admin_ui/pages/opportunities.py") + recon = read("app/admin_ui/pages/reconciliation.py") + assert "/opportunities/{opportunity_id}/fiscal-enrich" in opp + assert "/fiscal-suggestions/{suggestion_id}/accept" in opp + assert "Sugestões fiscais" in opp + assert "/reconciliation/enrich-fiscal" in recon + assert "Enriquecer oportunidades" in recon + + +def test_v4925_scripts_and_settings_exist(): + assert (ROOT / "scripts/enrich_fiscal_customers.py").exists() + assert (ROOT / "scripts/run_reconciliation_pipeline.py").exists() + config = read("app/config.py") + assert "external_company_lookup_enabled" in config + assert "external_company_lookup_base_url" in config + assert "external_company_lookup_api_key" in config diff --git a/tests/test_v4926_1_email_identity_quality.py b/tests/test_v4926_1_email_identity_quality.py new file mode 100644 index 0000000..d87e4c6 --- /dev/null +++ b/tests/test_v4926_1_email_identity_quality.py @@ -0,0 +1,124 @@ +from app.email_identity_extraction_service import extract_email_identity + + +DIETIMPORT_EMAIL = """ +Boa tarde, + +Conseguiria enviar-me a fatura em formato digital, referente à nossa última encomenda? + +Obrigado. + +Com os melhores cumprimentos, + +Filipe Delgado + +Engenheiro Civil + +Responsável +Departamento de Manutenção + +Rua Dr. Costa Sacadura N.º4 1800-176 Lisboa +Telem: +351 96 873 88 56 (Chamada para Rede Móvel Nacional) +Tel: +351 21 159 01 95 (Chamada para Rede Fixa Nacional) + +AVISO +Esta mensagem (incluindo quaisquer anexos) pode conter informação confidencial para uso exclusivo do destinatário. Se não for o destinatário pretendido não deverá usar, distribuir ou copiar este e-mail. Se recebeu esta mensagem por engano por favor informe o emissor e elimine-a imediatamente. As opiniões emitidas nesta mensagem não são necessariamente as da Dietimport S.A.. + +Obrigado. +""" + + +def test_regex_identity_ignores_valediction_as_person_name(): + data = extract_email_identity( + DIETIMPORT_EMAIL, + email="filipedelgado@dietimport.pt", + use_llm=False, + ) + assert data["person_name"] == "Filipe Delgado" + assert data["person_name"] != "Com os melhores cumprimentos," + + +def test_regex_identity_keeps_company_name_not_full_disclaimer(): + data = extract_email_identity( + DIETIMPORT_EMAIL, + email="filipedelgado@dietimport.pt", + use_llm=False, + ) + assert data["company_mentions"] == ["Dietimport S.A"] + assert all("informação confidencial" not in company for company in data["company_mentions"]) + assert all(len(company) < 80 for company in data["company_mentions"]) + + +def test_regex_identity_address_stops_before_phone_labels(): + data = extract_email_identity( + DIETIMPORT_EMAIL, + email="filipedelgado@dietimport.pt", + use_llm=False, + ) + assert data["address"] == "Rua Dr. Costa Sacadura N.º4 1800-176 Lisboa" + assert "Telem" not in data["address"] + assert "+351 96 873 88 56" in data["phones"] + assert data["domain"] == "dietimport.pt" + + +def test_identity_cleanup_removes_legal_paragraph_mentions(monkeypatch): + from app.email_identity_extraction_service import merge_identity + + regex_data = { + "person_name": "", + "company_mentions": [ + "dos nesta comunicação estão incorporados num ficheiro cujo responsável é a SOPAC, S.A", + "obter a confirmação sobre se a SOPAC, S.A", + ], + "address": "", + "phones": [], + "websites": [], + "email": "info@adp-fertilizantes.pt", + "domain": "adp-fertilizantes.pt", + "confidence": 0.9, + "evidence": [], + "method": "regex", + } + data = merge_identity(regex_data, None) + assert data["company_mentions"] == ["SOPAC, S.A"] + assert all("ficheiro" not in company for company in data["company_mentions"]) + + +def test_identity_cleanup_removes_cjk_noise_from_person_name(): + from app.email_identity_extraction_service import merge_identity + + regex_data = { + "person_name": "Luis Roch游戏副本a", + "company_mentions": [], + "address": "", + "phones": [], + "websites": [], + "email": "luis.rocha@coimpack.pt", + "domain": "coimpack.pt", + "confidence": 0.85, + "evidence": [], + "method": "regex", + } + data = merge_identity(regex_data, None) + assert data["person_name"] == "Luis Rocha" + assert data["confidence"] <= 0.45 + + +def test_identity_cleanup_rejects_mobile_footer_as_person_name(): + from app.email_identity_extraction_service import merge_identity + + regex_data = { + "person_name": "Enviado do meu Galaxy", + "company_mentions": [], + "address": "", + "phones": [], + "websites": [], + "email": "paulo.campelo@sapo.pt", + "domain": "sapo.pt", + "confidence": 0.7, + "evidence": [], + "method": "regex", + } + data = merge_identity(regex_data, None) + assert data["person_name"] == "" + assert data["confidence"] <= 0.45 diff --git a/tests/test_v4926_5_manual_opportunity_static.py b/tests/test_v4926_5_manual_opportunity_static.py new file mode 100644 index 0000000..f208143 --- /dev/null +++ b/tests/test_v4926_5_manual_opportunity_static.py @@ -0,0 +1,31 @@ +from pathlib import Path + + +def test_assisted_identity_skips_duplicate_current_customer_suggestion(): + source = Path("app/fiscal_enrichment_service.py").read_text() + assert "email_identity_matches_current_fiscal_customer" in source + assert "suggested_customer_id == linked_customer_id" in source + assert '"suggested": 0' in source + + +def test_customer_detail_has_create_opportunity_flow(): + source = Path("app/admin_ui/pages/customers.py").read_text() + assert "/customers/{customer_id}/opportunities/create" in source + assert "create_manual_opportunity_from_customer" in source + assert "Criar oportunidade" in source + + +def test_manual_opportunity_command_links_customer_and_optional_task(): + source = Path("app/opportunity_service.py").read_text() + assert "def create_manual_opportunity_from_customer" in source + assert "local_customer_id" in source + assert "created_from_customer_page" in source + assert "manual_opportunity_created" in source + assert "created_from_manual_opportunity" in source + + +def test_duplicate_identity_metadata_jsonb_parameters_are_cast_to_text(): + source = Path("app/fiscal_enrichment_service.py").read_text() + assert "CAST(:customer_id AS TEXT)" in source + assert "CAST(:customer_name AS TEXT)" in source + assert "CAST(:customer_tax_id AS TEXT)" in source diff --git a/tests/test_v4926_6_2_jasmin_money_extraction_static.py b/tests/test_v4926_6_2_jasmin_money_extraction_static.py new file mode 100644 index 0000000..8486cc9 --- /dev/null +++ b/tests/test_v4926_6_2_jasmin_money_extraction_static.py @@ -0,0 +1,39 @@ +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def read(path: str) -> str: + return (ROOT / path).read_text() + + +def test_jasmin_money_object_helper_exists(): + source = read("app/reconciliation_service.py") + assert "def _jasmin_money_value" in source + assert '"amount", "baseAmount", "reportingAmount", "value"' in source + assert "value = _jasmin_money_value(value)" in source + + +def test_jasmin_document_totals_read_real_amount_fields(): + source = read("app/reconciliation_service.py") + assert "def _jasmin_document_totals" in source + assert '"grossValueAmount"' in source + assert '"taxExclusiveAmountAmount"' in source + assert '"taxTotalAmount"' in source + assert '"payableAmountAmount"' in source + assert '"payableAmount"' in source + + +def test_jasmin_line_prices_read_nested_and_scalar_fields(): + source = read("app/reconciliation_service.py") + assert '"unitPriceAmount"' in source + assert '"unitPrice"' in source + assert '"lineExtensionAmountAmount"' in source + assert '"lineExtensionAmount"' in source + assert '"grossValueAmount"' in source + + +def test_backfill_script_accepts_money_objects_from_detail_fetch(): + source = read("scripts/backfill_jasmin_opportunity_details.py") + assert "def _money_value" in source + assert "value = _money_value(value)" in source diff --git a/tests/test_v4926_6_jasmin_reconciliation_static.py b/tests/test_v4926_6_jasmin_reconciliation_static.py new file mode 100644 index 0000000..b05e799 --- /dev/null +++ b/tests/test_v4926_6_jasmin_reconciliation_static.py @@ -0,0 +1,31 @@ +from pathlib import Path + + +def test_jasmin_reconciliation_imports_documents_and_lines(): + source = Path("app/reconciliation_service.py").read_text() + assert "def _upsert_jasmin_document_from_item" in source + assert "commercial_documents" in source + assert "commercial_document_lines" in source + assert "def _jasmin_document_lines_from_item" in source + + +def test_jasmin_reconciliation_imports_opportunity_items_and_value(): + source = Path("app/reconciliation_service.py").read_text() + assert "def _upsert_opportunity_items_from_jasmin_item" in source + assert "metadata->>'source_system' = 'jasmin'" in source + assert "_apply_jasmin_documents_to_opportunity" in source + assert "jasmin_reconciliation_imported" in source + + +def test_create_and_link_process_apply_jasmin_import(): + source = Path("app/reconciliation_service.py").read_text() + assert "_apply_jasmin_documents_to_opportunity(conn, items, opportunity_id, actor=actor)" in source + assert "_apply_jasmin_documents_to_opportunity(conn, [item], opportunity_id, actor=actor)" in source + + +def test_jasmin_reimport_updates_existing_opportunity_items(): + source = Path("app/reconciliation_service.py").read_text() + assert "UPDATE opportunity_items" in source + assert "unit_price = CAST(:unit_price AS NUMERIC)" in source + assert "total_price = CAST(:total_price AS NUMERIC)" in source + assert "price_source" in source diff --git a/tests/test_v4926_7_identity_and_customer_guards.py b/tests/test_v4926_7_identity_and_customer_guards.py new file mode 100644 index 0000000..dca2cc8 --- /dev/null +++ b/tests/test_v4926_7_identity_and_customer_guards.py @@ -0,0 +1,20 @@ +from app.email_identity_extraction_service import _sanitize_company_mention, is_plausible_company_mention +from app.fiscal_enrichment_service import _company_name_matches_mention + + +def test_rejects_tld_and_domain_fragments_as_company_mentions(): + for value in ["pt", "com", "www", "sementenergias.pt", "http", "email"]: + assert _sanitize_company_mention(value) == "" + assert not is_plausible_company_mention(value) + + +def test_keeps_valid_portuguese_company_mentions(): + assert _sanitize_company_mention("SEMENTENERGIAS, UNIPESSOAL, LDA") + assert is_plausible_company_mention("Dietimport S.A") + + +def test_short_mentions_do_not_match_inside_company_names(): + assert not _company_name_matches_mention("pt", "planetoption lda") + assert not _company_name_matches_mention("pt", "sementenergias unipessoal lda") + assert _company_name_matches_mention("sementenergias unipessoal lda", "sementenergias unipessoal lda") + assert _company_name_matches_mention("dietimport s a", "dietimport s a") diff --git a/tests/test_v4928_decisional_ui_static.py b/tests/test_v4928_decisional_ui_static.py new file mode 100644 index 0000000..507383a --- /dev/null +++ b/tests/test_v4928_decisional_ui_static.py @@ -0,0 +1,33 @@ +from pathlib import Path + + +def test_reconciliation_decision_service_exists_and_is_read_only(): + text = Path("app/reconciliation_decision_service.py").read_text() + assert "classify_reconciliation_item" in text + assert "classify_reconciliation_process" in text + assert "reconciliation_decision_summary" in text + assert "CREATE UNIQUE INDEX" not in text + + +def test_reconciliation_page_exposes_decision_lanes_and_actions(): + text = Path("app/admin_ui/pages/reconciliation.py").read_text() + assert "cf-reconcile-decision-grid" in text + assert "Ação recomendada" in text + assert "/reconciliation/{item_id}/needs-review" in text + assert "/reconciliation/processes/ignore" in text + assert "sort_items_for_operator" in text + + +def test_opportunity_page_has_operational_map(): + text = Path("app/admin_ui/pages/opportunities.py").read_text() + assert "Mapa operacional" in text + assert "Cliente fiscal" in text + assert "Documento principal" in text + assert "Ações avançadas" in text + + +def test_jasmin_actions_avoid_duplicate_linking_when_current_doc_exists(): + text = Path("app/admin_dashboard.py").read_text() + assert "Já existe documento atual. A associação direta fica bloqueada" in text + assert "invoice_source_exists" in text + assert "É necessário um orçamento ou pró-forma atual para converter" in text diff --git a/tests/test_v492_external_api_sync.py b/tests/test_v492_external_api_sync.py new file mode 100644 index 0000000..b467dce --- /dev/null +++ b/tests/test_v492_external_api_sync.py @@ -0,0 +1,63 @@ +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def read(path: str) -> str: + return (ROOT / path).read_text(encoding="utf-8") + + +def test_v492_external_sync_service_stages_candidates_not_opportunities(): + service = read("app/external_reconciliation_sync.py") + + assert "sync_jasmin_reconciliation_candidates" in service + assert "sync_odoo_reconciliation_candidates" in service + assert "sync_packlink_reconciliation_candidates" in service + assert "upsert_reconciliation_item" in service + assert "create_opportunity" not in service + assert "CONFIRM_PAYMENT" in service + assert "não" not in service.lower() or "confirm" in service.lower() + + +def test_v492_jasmin_odoo_packlink_types_are_supported(): + service = read("app/external_reconciliation_sync.py") + reconciliation = read("app/reconciliation_service.py") + page = read("app/admin_ui/pages/reconciliation.py") + + assert "jasmin_quotation" in service + assert "jasmin_invoice" in service + assert "odoo_sale_order" in service + assert "packlink_shipment" in service + assert "packlink_shipment" in reconciliation + assert "Envio Packlink" in page + + +def test_v492_reconciliation_page_has_external_sync_actions(): + page = read("app/admin_ui/pages/reconciliation.py") + + assert 'action="/reconciliation/sync-jasmin"' in page + assert 'action="/reconciliation/sync-odoo"' in page + assert 'action="/reconciliation/sync-packlink"' in page + assert 'action="/reconciliation/sync-external"' in page + assert "Sincronizar APIs externas" in page + + +def test_v492_sync_script_exists_and_reports_json_summary(): + script = read("scripts/sync_external_reconciliation.py") + + assert "--all" in script + assert "--jasmin" in script + assert "--odoo" in script + assert "--packlink" in script + assert "created_or_updated" in script + assert "json.dumps" in script + + +def test_v492_documentation_describes_operator_confirmation(): + doc = read("docs/CLIENTFLOW_V492_EXTERNAL_API_SYNC.md") + + assert "External API Sync" in doc + assert "não cria oportunidades automaticamente" in doc.lower() + assert "não confirma pagamentos" in doc.lower() + assert "operador decide" in doc.lower() + assert "systemd" in doc diff --git a/tests/test_v493_jasmin_recent_sync.py b/tests/test_v493_jasmin_recent_sync.py new file mode 100644 index 0000000..3a455ca --- /dev/null +++ b/tests/test_v493_jasmin_recent_sync.py @@ -0,0 +1,38 @@ +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def read(path: str) -> str: + return (ROOT / path).read_text(encoding="utf-8") + + +def test_v493_jasmin_sync_respects_days_and_global_limit(): + service = read("app/external_reconciliation_sync.py") + script = read("scripts/sync_external_reconciliation.py") + + assert "sync_jasmin_reconciliation_candidates(*, limit: int = 100, days: int = 3)" in service + assert "since =" in service + assert "_within_days(record, days=days)" in service + assert "staged.sort" in service + assert "staged[:limit]" in service + assert "sync_jasmin_reconciliation_candidates(limit=args.limit, days=args.days)" in script + + +def test_v493_jasmin_client_supports_odata_filter_and_orderby(): + client = read("app/jasmin_client.py") + + assert "filter: Optional[str] = None" in client + assert "orderby: Optional[str] = None" in client + assert 'params["$filter"] = filter' in client + assert 'params["$orderby"] = orderby' in client + + +def test_v493_stale_reconciliation_cleanup_script_exists(): + script = read("scripts/cleanup_stale_reconciliation_items.py") + + assert "--jasmin" in script + assert "--days" in script + assert "--apply" in script + assert "cleanup_reconciliation_outside_window" in script + assert "Dry-run" in script diff --git a/tests/test_v494_reconciliation_recent_window.py b/tests/test_v494_reconciliation_recent_window.py new file mode 100644 index 0000000..14de7e9 --- /dev/null +++ b/tests/test_v494_reconciliation_recent_window.py @@ -0,0 +1,27 @@ +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def read(path: str) -> str: + return (ROOT / path).read_text(encoding="utf-8") + + +def test_v494_external_sync_defaults_to_three_day_window(): + script = read("scripts/sync_external_reconciliation.py") + service = read("app/external_reconciliation_sync.py") + + assert 'parser.add_argument("--days", type=int, default=3' in script + assert "sync_all_external_reconciliation_candidates(*, limit: int = 100, days: int = 3)" in service + assert "sync_odoo_reconciliation_candidates(*, limit: int = 100, days: int = 3)" in service + assert "sync_packlink_reconciliation_candidates(*, limit: int = 100, days: int = 3)" in service + + +def test_v494_reconciliation_ui_has_cleanup_window_action(): + ui = read("app/admin_ui/pages/reconciliation.py") + service = read("app/reconciliation_service.py") + + assert "/reconciliation/cleanup-window" in ui + assert "Limpar fora dos 3 dias" in ui + assert "cleanup_reconciliation_outside_window" in service + assert "fora da janela operacional" in service diff --git a/tests/test_v495_reconciliation_operation_suggestions.py b/tests/test_v495_reconciliation_operation_suggestions.py new file mode 100644 index 0000000..9073744 --- /dev/null +++ b/tests/test_v495_reconciliation_operation_suggestions.py @@ -0,0 +1,35 @@ +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def read(path: str) -> str: + return (ROOT / path).read_text(encoding="utf-8") + + +def test_v495_reconciliation_service_suggests_open_operations(): + service = read("app/reconciliation_service.py") + + assert "find_open_operation_suggestions_for_reconciliation" in service + assert "operation_suggestions" in service + assert "operação aberta" in service + assert "match_score" in service + assert "status = 'pending'" in service + assert "o.status = 'open'" in service + + +def test_v495_reconciliation_ui_prefills_link_actions_from_suggestions(): + page = read("app/admin_ui/pages/reconciliation.py") + + assert "Sugestão encontrada em Operations" in page + assert "Ligar a esta operação" in page + assert "name=\"opportunity_id\"" in page + assert "Escolher outra oportunidade" in page + assert "Criar oportunidade" in page + + +def test_v495_reconciliation_suggestion_styles_exist(): + styles = read("app/admin_ui/styles.py") + + assert ".cf-reconcile-suggestion" in styles + assert "v4.9.5 reconciliation operation suggestions" in styles diff --git a/tests/test_v496_reconciliation_nif_suggestions.py b/tests/test_v496_reconciliation_nif_suggestions.py new file mode 100644 index 0000000..91d87c8 --- /dev/null +++ b/tests/test_v496_reconciliation_nif_suggestions.py @@ -0,0 +1,42 @@ +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def read(path: str) -> str: + return (ROOT / path).read_text(encoding="utf-8") + + +def test_v496_reconciliation_schema_stores_customer_tax_id(): + service = read("app/reconciliation_service.py") + + assert "customer_tax_id TEXT" in service + assert "idx_reconciliation_customer_tax_id" in service + assert "customer_tax_id: Optional[str]" in service + assert "customer_tax_id = _normalize_tax_id(customer_tax_id)" in service + + +def test_v496_nif_exact_match_is_primary_suggestion_signal(): + service = read("app/reconciliation_service.py") + + assert "NIF exato" in service + assert "COALESCE(lc.tax_id, '') = :customer_tax_id" in service + assert "THEN 90 ELSE 0" in service + assert "if not any([customer_id, customer_tax_id, customer_email, customer_name, amount_value])" in service + + +def test_v496_external_sync_extracts_tax_id_from_jasmin_and_odoo(): + sync = read("app/external_reconciliation_sync.py") + + assert "def _customer_tax_id" in sync + assert "companyTaxID" in sync + assert "buyerCustomerPartyTaxID" in sync + assert '"customer_tax_id": _customer_tax_id(record)' in sync + assert 'client.search_read(\n "res.partner"' in sync + assert 'record["partner_vat"]' in sync + + +def test_v496_reconciliation_ui_shows_customer_nif_when_available(): + page = read("app/admin_ui/pages/reconciliation.py") + + assert "NIF {esc(item.get(\"customer_tax_id\"))}" in page diff --git a/tests/test_v498_reconciliation_matching_board_cleanup.py b/tests/test_v498_reconciliation_matching_board_cleanup.py new file mode 100644 index 0000000..8b65428 --- /dev/null +++ b/tests/test_v498_reconciliation_matching_board_cleanup.py @@ -0,0 +1,44 @@ +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def read(path: str) -> str: + return (ROOT / path).read_text(encoding="utf-8") + + +def test_v498_recent_window_uses_human_last_n_days(): + sync = read("app/external_reconciliation_sync.py") + service = read("app/reconciliation_service.py") + + assert "_recent_window_start" in sync + assert "days=days - 1" in sync + assert "recent_window_start" in service + assert "days=days - 1" in service + + +def test_v498_reconciliation_page_recent_window_and_search_action(): + page = read("app/admin_ui/pages/reconciliation.py") + + assert "list_reconciliation_items(status=status, external_type=item_type, limit=100, days=recent_days)" in page + assert "reconciliation_summary(days=recent_days)" in page + assert "Pesquisar oportunidade" in page + assert "Escolher outra oportunidade" in page + + +def test_v498_odoo_sync_enriches_partner_for_matching(): + sync = read("app/external_reconciliation_sync.py") + + assert 'fields=["id", "vat", "email", "name"]' in sync + assert 'record["partner_vat"]' in sync + assert 'record["partner_email"]' in sync + assert '"customer_email": _customer_email(record)' in sync + + +def test_v498_opportunity_board_hides_noise_and_uses_action_column(): + page = read("app/admin_ui/pages/opportunities.py") + + assert "is_noise_operation_item" in page + assert "_is_noise_opportunity" in page + assert "_opportunity_board_column_for_opportunity" in page + assert "SEND_INVOICE" in page and "payment" in page diff --git a/tests/test_v499_process_timeline_reconstruction.py b/tests/test_v499_process_timeline_reconstruction.py new file mode 100644 index 0000000..7ba9ccd --- /dev/null +++ b/tests/test_v499_process_timeline_reconstruction.py @@ -0,0 +1,39 @@ +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def read(path: str) -> str: + return (ROOT / path).read_text(encoding="utf-8") + + +def test_v499_reconciliation_service_groups_process_candidates(): + service = read("app/reconciliation_service.py") + + assert "PROCESS_STEP_BY_EXTERNAL_TYPE" in service + assert "list_reconciliation_process_candidates" in service + assert "infer_reconciliation_process_state" in service + assert "create_opportunity_from_reconciliation_process" in service + assert "link_reconciliation_process_to_opportunity" in service + assert "opportunity_reconstructed_from_reconciliation" in service + assert "reconciliation_evidence_imported" in service + + +def test_v499_reconciliation_page_renders_process_cards_and_actions(): + page = read("app/admin_ui/pages/reconciliation.py") + + assert "Processos candidatos" in page + assert "_render_process_candidates" in page + assert "Criar oportunidade reconstruída" in page + assert "Ligar processo à sugestão" in page + assert 'action="/reconciliation/processes/create-opportunity"' in page + assert 'action="/reconciliation/processes/link"' in page + + +def test_v499_reconciliation_styles_and_docs_exist(): + styles = read("app/admin_ui/styles.py") + docs = read("docs/CLIENTFLOW_V499_PROCESS_TIMELINE_RECONSTRUCTION.md") + + assert "v4.9.9 process timeline reconstruction" in styles + assert "cf-reconcile-process-card" in styles + assert "Process Timeline Reconstruction" in docs diff --git a/tests/test_v499_process_timeline_reconstruction_behavior.py b/tests/test_v499_process_timeline_reconstruction_behavior.py new file mode 100644 index 0000000..d3dd12f --- /dev/null +++ b/tests/test_v499_process_timeline_reconstruction_behavior.py @@ -0,0 +1,308 @@ +"""Behavior tests for v4.9.9 process timeline reconstruction. + +These tests validate the core reconstruction rules without touching the real +production database: grouping is based on strong identity signals and the +suggested stage/action is inferred from the most advanced external evidence. +""" + +import sys +import types + +# The project test suite is intentionally lightweight and mostly static. This +# file exercises pure reconstruction helpers without requiring a live DB or +# SQLAlchemy in the test environment. +if "sqlalchemy" not in sys.modules: + sqlalchemy = types.ModuleType("sqlalchemy") + sqlalchemy.text = lambda sql: sql + sys.modules["sqlalchemy"] = sqlalchemy + +if "app.db" not in sys.modules: + app_db = types.ModuleType("app.db") + app_db.engine = object() + sys.modules["app.db"] = app_db + +if "app.action_catalog" not in sys.modules: + action_catalog = types.ModuleType("app.action_catalog") + action_catalog.get_action_config = lambda code: {"route": "rever", "action": code, "action_required": True, "safe_to_post": False} + sys.modules["app.action_catalog"] = action_catalog + +from app import reconciliation_service as svc + + +def test_process_state_reconstructs_quote_then_odoo_sale_as_invoice_check(): + state = svc.infer_reconciliation_process_state([ + { + "id": "quote-1", + "source_system": "jasmin", + "external_type": "jasmin_quotation", + "document_number": "ORC.ORC2026.157", + "document_date": "2026-06-05", + "amount": "320.00", + }, + { + "id": "sale-1", + "source_system": "odoo", + "external_type": "odoo_sale_order", + "document_number": "S00275", + "document_date": "2026-06-05", + "amount": "320.00", + }, + ]) + + assert state["stage"] == "ODOO_ORDER_CREATED" + assert state["action_code"] == "SEND_INVOICE" + assert [step["external_type"] for step in state["steps"]] == [ + "jasmin_quotation", + "odoo_sale_order", + ] + + +def test_process_state_payment_proof_requires_manual_payment_confirmation(): + state = svc.infer_reconciliation_process_state([ + { + "id": "invoice-1", + "source_system": "jasmin", + "external_type": "jasmin_invoice", + "document_number": "FA.FA2026.209", + "document_date": "2026-06-05", + "amount": "537.00", + }, + { + "id": "proof-1", + "source_system": "manual", + "external_type": "payment_proof", + "document_number": "comprovativo.jpg", + "document_date": "2026-06-05", + "amount": "537.00", + }, + ]) + + assert state["stage"] == "INVOICE_SENT" + assert state["action_code"] == "CONFIRM_PAYMENT" + assert any(step["external_type"] == "payment_proof" for step in state["steps"]) + + +def test_process_candidates_group_by_exact_nif_and_ignore_unrelated_lone_items(monkeypatch): + monkeypatch.setattr( + svc, + "list_reconciliation_items", + lambda *, status, limit, days: [ + { + "id": "quote-1", + "source_system": "jasmin", + "external_type": "jasmin_quotation", + "title": "Orçamento Jasmin sem oportunidade · ORC.ORC2026.157", + "customer_name": "DOCTOR SOLUTION CONSTRUÇÕES, UNIPESSOAL, LDA", + "customer_tax_id": "PT515708690", + "document_number": "ORC.ORC2026.157", + "document_date": "2026-06-05", + "amount": "320.00", + "currency": "EUR", + }, + { + "id": "sale-1", + "source_system": "odoo", + "external_type": "odoo_sale_order", + "title": "Venda Odoo sem oportunidade · S00275", + "customer_name": "DOCTOR SOLUTION CONSTRUÇÕES, UNIPESSOAL, LDA", + "customer_tax_id": "515708690", + "document_number": "S00275", + "document_date": "2026-06-05", + "amount": "320.00", + "currency": "EUR", + }, + { + "id": "lone-1", + "source_system": "jasmin", + "external_type": "jasmin_quotation", + "title": "Orçamento isolado", + "customer_name": "CLIENTE SEM SEGUNDA EVIDÊNCIA", + "customer_tax_id": "599999999", + "document_number": "ORC.ISOLATED", + "document_date": "2026-06-05", + "amount": "10.00", + "currency": "EUR", + }, + ], + ) + + candidates = svc.list_reconciliation_process_candidates(status="open", days=3, limit=10) + + assert len(candidates) == 1 + candidate = candidates[0] + assert candidate["match_key"] == "nif" + assert candidate["match_value"] == "515708690" + assert candidate["confidence"] == "alta" + assert candidate["item_ids"] == ["quote-1", "sale-1"] + assert candidate["suggested_stage"] == "ODOO_ORDER_CREATED" + assert candidate["suggested_action"] == "SEND_INVOICE" + + +def test_single_item_with_open_operation_suggestion_becomes_process_candidate(monkeypatch): + opportunity_id = "11111111-1111-1111-1111-111111111111" + monkeypatch.setattr( + svc, + "list_reconciliation_items", + lambda *, status, limit, days: [ + { + "id": "invoice-1", + "source_system": "jasmin", + "external_type": "jasmin_invoice", + "title": "Fatura Jasmin sem oportunidade · FA.FA2026.209", + "customer_name": "ELEGANTLEGACY, LDA", + "customer_tax_id": "515510165", + "document_number": "FA.FA2026.209", + "document_date": "2026-06-05", + "amount": "537.00", + "currency": "EUR", + "operation_suggestions": [ + { + "opportunity_id": opportunity_id, + "title": "Elegantlegacy Lda · Emitir fatura", + "reason": "NIF exato", + } + ], + } + ], + ) + + candidates = svc.list_reconciliation_process_candidates(status="open", days=3, limit=10) + + assert len(candidates) == 1 + assert candidates[0]["item_ids"] == ["invoice-1"] + assert candidates[0]["suggestions"][0]["opportunity_id"] == opportunity_id + assert candidates[0]["suggested_action"] == "CONFIRM_PAYMENT" + + +def test_process_candidates_merge_jasmin_nif_with_odoo_name_only_evidence(monkeypatch): + """Jasmin may provide NIF while Odoo only provides the fiscal name. + + The real operational process should still be shown as one candidate, not as + an isolated Jasmin quote with an Odoo sale merely suggested elsewhere. + """ + monkeypatch.setattr( + svc, + "list_reconciliation_items", + lambda *, status, limit, days: [ + { + "id": "quote-aczco", + "source_system": "jasmin", + "external_type": "jasmin_quotation", + "title": "Orçamento Jasmin sem oportunidade · ORC.ORC2026.154", + "customer_name": "ACZCO BRAGA ENERGY, LDA", + "customer_tax_id": "517249200", + "document_number": "ORC.ORC2026.154", + "document_date": "2026-06-03", + "amount": None, + "currency": "EUR", + "operation_suggestions": [], + }, + { + "id": "sale-aczco", + "source_system": "odoo", + "external_type": "odoo_sale_order", + "title": "Venda Odoo sem oportunidade · S00279", + "customer_name": "ACZCO BRAGA ENERGY, LDA", + "customer_tax_id": "", + "document_number": "S00279", + "document_date": "2026-06-05", + "amount": "638.60", + "currency": "EUR", + "payload": { + "record": { + "invoice_status": "to invoice", + "fulfilment": { + "invoice_pending": True, + "lines": [ + {"product_name": "Wallbox 7.4KW"}, + {"product_name": "RFID Reader"}, + ], + "outgoing_pickings": [{"name": "WH/OUT/00300", "state": "done", "date_done": "2026-06-05 15:12:00"}], + }, + } + }, + "operation_suggestions": [], + }, + ], + ) + + candidates = svc.list_reconciliation_process_candidates(status="open", days=7, limit=10) + + assert len(candidates) == 1 + candidate = candidates[0] + assert candidate["match_key"] == "nif" + assert candidate["match_value"] == "517249200" + assert candidate["confidence"] == "alta" + assert candidate["identity_reason"] == "NIF + nome fiscal" + assert set(candidate["item_ids"]) == {"quote-aczco", "sale-aczco"} + assert candidate["suggested_stage"] == "SHIPMENT_CREATED" + assert candidate["suggested_action"] == "SEND_INVOICE" + assert any(step["external_type"] == "jasmin_quotation" for step in candidate["timeline"]) + assert any(step["external_type"] == "odoo_sale_order" for step in candidate["timeline"]) + assert any(step["external_type"] == "odoo_delivery" for step in candidate["timeline"]) + + +def test_process_candidates_bridge_jasmin_nif_to_odoo_name_with_location_suffix(monkeypatch): + """Odoo may only expose a display name with address/location suffixes. + + Jasmin remains the fiscal identity source because it has the NIF. Odoo + evidence should still join the same reconstructed process when its company + name is a strong normalized name match. + """ + monkeypatch.setattr( + svc, + "list_reconciliation_items", + lambda *, status, limit, days: [ + { + "id": "quote-aczco-fiscal", + "source_system": "jasmin", + "external_type": "jasmin_quotation", + "title": "Orçamento Jasmin sem oportunidade · ORC.ORC2026.154", + "customer_name": "ACZCO BRAGA ENERGY, LDA", + "customer_tax_id": "517249200", + "document_number": "ORC.ORC2026.154", + "document_date": "2026-06-03", + "amount": None, + "currency": "EUR", + "operation_suggestions": [], + }, + { + "id": "sale-aczco-display-name", + "source_system": "odoo", + "external_type": "odoo_sale_order", + "title": "Venda Odoo sem oportunidade · S00279", + "customer_name": "ACZCO BRAGA ENERGY, LDA MERELIM SÃO PAIO Portugal", + "customer_tax_id": "", + "document_number": "S00279", + "document_date": "2026-06-05", + "amount": "638.60", + "currency": "EUR", + "payload": { + "record": { + "invoice_status": "to invoice", + "fulfilment": { + "invoice_pending": True, + "lines": [{"product_name": "Wallbox 11KW"}], + "outgoing_pickings": [{"name": "WH/OUT/00301", "state": "done", "date_done": "2026-06-05 15:12:00"}], + }, + } + }, + "operation_suggestions": [], + }, + ], + ) + + candidates = svc.list_reconciliation_process_candidates(status="open", days=7, limit=10) + + assert len(candidates) == 1 + candidate = candidates[0] + assert candidate["match_key"] == "nif" + assert candidate["match_value"] == "517249200" + assert candidate["confidence"] == "alta" + assert candidate["identity_reason"] == "NIF + nome fiscal" + assert set(candidate["item_ids"]) == {"quote-aczco-fiscal", "sale-aczco-display-name"} + assert candidate["suggested_stage"] == "SHIPMENT_CREATED" + assert candidate["suggested_action"] == "SEND_INVOICE" + assert any(step["external_type"] == "jasmin_quotation" for step in candidate["timeline"]) + assert any(step["external_type"] == "odoo_sale_order" for step in candidate["timeline"]) + assert any(step["external_type"] == "odoo_delivery" for step in candidate["timeline"])