Implement document reconciliation v2

scripts/preflight_document_reconciliation_v2.py

@@ -0,0 +1,107 @@
+#!/usr/bin/env python3
+"""Read-only preflight for migration 007; it never repairs production data."""
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+from typing import Any, Dict
+
+from sqlalchemy import text
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT))
+
+
+def run_preflight(conn: Any) -> Dict[str, Any]:
+    column_rows = list(conn.execute(text("""
+        SELECT table_name, column_name, data_type, udt_name FROM information_schema.columns
+        WHERE table_schema='public' AND table_name IN
+          ('commercial_document_lines','commercial_documents','opportunities')
+    """)))
+    columns = {row[1] for row in column_rows if row[0] == "commercial_document_lines"}
+    required = {"id", "document_id"}
+    counts: Dict[str, int] = {}
+    details: Dict[str, list[Dict[str, Any]]] = {}
+    blockers = []
+    missing = sorted(required - columns)
+    if missing:
+        blockers.append("commercial_document_lines missing columns: " + ", ".join(missing))
+        return {"counts": counts, "blockers": blockers, "columns": sorted(columns)}
+    types = {(row[0], row[1]): row[3] for row in column_rows}
+    for table_name, column_name in (("commercial_document_lines", "document_id"),
+                                    ("commercial_documents", "id"),
+                                    ("commercial_documents", "opportunity_id"),
+                                    ("opportunities", "id")):
+        actual = types.get((table_name, column_name))
+        if actual != "uuid":
+            blockers.append(f"unexpected type {table_name}.{column_name}: {actual or 'missing'} (expected uuid)")
+    counts["null_document_id"] = int(conn.execute(text(
+        "SELECT count(*) FROM commercial_document_lines WHERE document_id IS NULL"
+    )).scalar() or 0)
+    counts["orphan_document_id"] = int(conn.execute(text("""
+        SELECT count(*) FROM commercial_document_lines dl
+        LEFT JOIN commercial_documents d ON d.id=dl.document_id
+        WHERE dl.document_id IS NOT NULL AND d.id IS NULL
+    """)).scalar() or 0)
+    duplicate_documents = [dict(row) for row in conn.execute(text("""
+          SELECT system, external_id, document_kind, company,
+                 array_agg(id::text ORDER BY id) AS ids,
+                 'duplicate_external_document' AS reason
+          FROM commercial_documents
+          WHERE external_id IS NOT NULL AND btrim(external_id) <> ''
+          GROUP BY system, external_id, document_kind, company HAVING count(*)>1
+    """)).mappings().all()]
+    duplicate_associations = [dict(row) for row in conn.execute(text("""
+          SELECT opportunity_id::text, system, external_id, document_kind,
+                 array_agg(id::text ORDER BY id) AS ids,
+                 'duplicate_opportunity_document_association' AS reason
+          FROM commercial_documents
+          WHERE opportunity_id IS NOT NULL AND external_id IS NOT NULL AND btrim(external_id) <> ''
+          GROUP BY opportunity_id, system, external_id, document_kind HAVING count(*)>1
+    """)).mappings().all()]
+    cross_opportunity = [dict(row) for row in conn.execute(text("""
+          SELECT system, external_id, document_kind, company,
+                 array_agg(id::text ORDER BY id) AS ids,
+                 array_agg(DISTINCT opportunity_id::text) AS opportunity_ids,
+                 'external_document_linked_to_multiple_opportunities' AS reason
+          FROM commercial_documents
+          WHERE opportunity_id IS NOT NULL AND external_id IS NOT NULL AND btrim(external_id) <> ''
+          GROUP BY system, external_id, document_kind, company
+          HAVING count(DISTINCT opportunity_id)>1
+    """)).mappings().all()]
+    multiple_primaries = [dict(row) for row in conn.execute(text("""
+          SELECT opportunity_id::text, document_kind, array_agg(id::text ORDER BY id) AS ids,
+                 'multiple_legacy_primaries' AS reason
+          FROM commercial_documents
+          WHERE opportunity_id IS NOT NULL AND COALESCE(is_primary,FALSE)
+            AND COALESCE(role,'current') IN ('current','accepted')
+          GROUP BY opportunity_id, document_kind HAVING count(*)>1
+    """)).mappings().all()]
+    details.update(duplicate_documents=duplicate_documents,
+                   duplicate_associations=duplicate_associations,
+                   cross_opportunity_documents=cross_opportunity,
+                   multiple_legacy_primaries=multiple_primaries)
+    counts["duplicate_document_groups"] = len(duplicate_documents)
+    counts["duplicate_associations"] = len(duplicate_associations)
+    counts["cross_opportunity_documents"] = len(cross_opportunity)
+    counts["multiple_legacy_primaries"] = len(multiple_primaries)
+    for key in ("null_document_id", "orphan_document_id", "duplicate_document_groups",
+                "duplicate_associations", "cross_opportunity_documents"):
+        if counts[key]:
+            blockers.append(f"{key}: {counts[key]}")
+    if counts["multiple_legacy_primaries"]:
+        blockers.append(f"multiple_legacy_primaries: {counts['multiple_legacy_primaries']}")
+    return {"counts": counts, "blockers": blockers, "details": details, "columns": sorted(columns)}
+
+
+def main() -> int:
+    from app.db import engine
+    with engine.begin() as conn:
+        result = run_preflight(conn)
+    print(json.dumps(result, ensure_ascii=False, indent=2))
+    return 2 if result["blockers"] else 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())