Implement document reconciliation v2
This commit is contained in:
plx
@@ -39,7 +39,8 @@ def main() -> int:
|
||||
parser.add_argument("--dry-run", action="store_true", help="List pending migrations without applying them.")
|
||||
args = parser.parse_args()
|
||||
|
||||
files = sorted(MIGRATIONS_DIR.glob("*.sql"))
|
||||
# Down migrations are operator-invoked rollback artefacts, never pending ups.
|
||||
files = sorted(path for path in MIGRATIONS_DIR.glob("*.sql") if not path.stem.endswith("_down"))
|
||||
if not files:
|
||||
print("No migrations found.")
|
||||
return 0
|
||||
@@ -69,6 +70,14 @@ def main() -> int:
|
||||
return 0
|
||||
|
||||
for version, path in pending:
|
||||
if version == "007":
|
||||
from scripts.preflight_document_reconciliation_v2 import run_preflight
|
||||
diagnostics = run_preflight(conn)
|
||||
if diagnostics["blockers"]:
|
||||
print("Migration 007 preflight failed:")
|
||||
for blocker in diagnostics["blockers"]:
|
||||
print(f"- {blocker}")
|
||||
return 2
|
||||
sql = path.read_text()
|
||||
print(f"Applying {path.name}...")
|
||||
conn.execute(text(sql))
|
||||
|
||||
137
scripts/migrate_document_reconciliation_v2.py
Normal file
137
scripts/migrate_document_reconciliation_v2.py
Normal file
@@ -0,0 +1,137 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Classify legacy document links into Document Reconciliation v2.
|
||||
|
||||
Dry-run is the default. This script changes ClientFlow only and never calls
|
||||
Jasmin. Ambiguous evidence is explicitly classified REVIEW_REQUIRED.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from app.document_reconciliation_backfill import build_backfill_result
|
||||
|
||||
|
||||
def load_documents(
|
||||
opportunity_id: str | None = None,
|
||||
resume_from: str | None = None,
|
||||
batch_size: int = 500,
|
||||
) -> tuple[list[Dict[str, Any]], int, int]:
|
||||
from sqlalchemy import text
|
||||
|
||||
from app.db import engine
|
||||
|
||||
where, params = ["d.opportunity_id IS NOT NULL"], {"limit": batch_size}
|
||||
if opportunity_id:
|
||||
where.append("d.opportunity_id=CAST(:opportunity_id AS UUID)"); params["opportunity_id"] = opportunity_id
|
||||
if resume_from:
|
||||
try:
|
||||
resume_opportunity_id, resume_document_kind = resume_from.split("|", 1)
|
||||
except ValueError as exc:
|
||||
raise ValueError("--resume-from must be OPPORTUNITY_ID|DOCUMENT_KIND") from exc
|
||||
where.append("(d.opportunity_id::text, d.document_kind) > (:resume_opportunity_id, :resume_document_kind)")
|
||||
params.update(resume_opportunity_id=resume_opportunity_id, resume_document_kind=resume_document_kind)
|
||||
with engine.begin() as conn:
|
||||
rows = [dict(r) for r in conn.execute(text(f"""
|
||||
WITH selected_groups AS (
|
||||
SELECT d.opportunity_id, d.document_kind
|
||||
FROM commercial_documents d WHERE {' AND '.join(where)}
|
||||
GROUP BY d.opportunity_id, d.document_kind
|
||||
ORDER BY d.opportunity_id, d.document_kind
|
||||
LIMIT :limit
|
||||
)
|
||||
SELECT d.id::text, d.opportunity_id::text, d.document_kind, d.external_id,
|
||||
d.document_number, d.status, d.role, d.is_primary, d.is_active,
|
||||
d.total_amount, d.amount, d.payload
|
||||
FROM selected_groups g
|
||||
JOIN commercial_documents d ON d.opportunity_id=g.opportunity_id
|
||||
AND d.document_kind=g.document_kind
|
||||
ORDER BY d.opportunity_id, d.document_kind, d.created_at, d.id
|
||||
"""), params).mappings().all()]
|
||||
orphan_lines = int(conn.execute(text("""SELECT count(*) FROM commercial_document_lines dl
|
||||
LEFT JOIN commercial_documents d ON d.id=COALESCE(dl.commercial_document_id,dl.document_id)
|
||||
WHERE d.id IS NULL""")).scalar() or 0)
|
||||
multi = int(conn.execute(text("""SELECT count(*) FROM (SELECT COALESCE(external_id,document_number), count(DISTINCT opportunity_id)
|
||||
FROM commercial_documents WHERE opportunity_id IS NOT NULL AND COALESCE(external_id,document_number) IS NOT NULL
|
||||
GROUP BY 1 HAVING count(DISTINCT opportunity_id)>1) q""")).scalar() or 0)
|
||||
return rows, orphan_lines, multi
|
||||
|
||||
|
||||
def scan(opportunity_id: str | None = None, resume_from: str | None = None, batch_size: int = 500) -> Dict[str, Any]:
|
||||
rows, orphan_lines, multi = load_documents(opportunity_id, resume_from, batch_size)
|
||||
result = build_backfill_result(
|
||||
rows,
|
||||
lines_without_document=orphan_lines,
|
||||
documents_in_multiple_opportunities=multi,
|
||||
)
|
||||
groups = sorted({(str(row["opportunity_id"]), str(row["document_kind"])) for row in rows})
|
||||
result["summary"]["batch_groups"] = len(groups)
|
||||
result["summary"]["checkpoint"] = "|".join(groups[-1]) if groups else resume_from
|
||||
return result
|
||||
|
||||
|
||||
def actions_for_only_unambiguous(actions: list[Dict[str, Any]]) -> list[Dict[str, Any]]:
|
||||
"""Keep ambiguous documents represented; never partially migrate a group.
|
||||
|
||||
Option B of the rollout contract is used: REVIEW_REQUIRED links are written
|
||||
alongside unambiguous links so every selected group becomes v2-complete.
|
||||
"""
|
||||
return list(actions)
|
||||
|
||||
|
||||
def apply_actions(actions: list[Dict[str, Any]]) -> int:
|
||||
from app.db import engine
|
||||
from app.document_reconciliation_service import set_document_relationship
|
||||
groups: Dict[tuple[str, str], list[Dict[str, Any]]] = {}
|
||||
for action in actions:
|
||||
groups.setdefault((str(action["opportunity_id"]), str(action["document_kind"])), []).append(action)
|
||||
# A group is the rollout authority boundary. Links, events and legacy dual
|
||||
# writes commit together; any failure rolls the entire group back.
|
||||
for group in groups.values():
|
||||
with engine.begin() as conn:
|
||||
for action in group:
|
||||
set_document_relationship(action["opportunity_id"], action["id"], action["relationship"],
|
||||
actor="document_reconciliation_v2_backfill", reason=action["decision_reason"], is_manual=False,
|
||||
source="backfill_v2", idempotency_key=f"backfill-v2:{action['opportunity_id']}:{action['id']}",
|
||||
event_type="BACKFILLED", metadata={"legacy_role": action.get("role"), "legacy_is_primary": action.get("is_primary")},
|
||||
_conn=conn)
|
||||
return len(actions)
|
||||
|
||||
|
||||
def write_reports(result: Dict[str, Any], json_path: str | None, csv_path: str | None) -> None:
|
||||
if json_path: Path(json_path).write_text(json.dumps(result, ensure_ascii=False, indent=2, default=str) + "\n")
|
||||
if csv_path:
|
||||
fields = ["opportunity_id", "id", "document_kind", "document_number", "status", "relationship", "decision_reason"]
|
||||
with Path(csv_path).open("w", newline="", encoding="utf-8") as handle:
|
||||
writer = csv.DictWriter(handle, fieldnames=fields, extrasaction="ignore"); writer.writeheader(); writer.writerows(result["actions"])
|
||||
|
||||
|
||||
def main() -> int:
|
||||
os.chdir(ROOT)
|
||||
parser = argparse.ArgumentParser()
|
||||
mode = parser.add_mutually_exclusive_group(); mode.add_argument("--dry-run", action="store_true"); mode.add_argument("--apply", action="store_true")
|
||||
parser.add_argument("--only-unambiguous", action="store_true")
|
||||
parser.add_argument("--opportunity-id"); parser.add_argument("--batch-size", type=int, default=500)
|
||||
parser.add_argument("--resume-from"); parser.add_argument("--output-json"); parser.add_argument("--output-csv")
|
||||
args = parser.parse_args()
|
||||
result = scan(args.opportunity_id, args.resume_from, args.batch_size)
|
||||
actions = result["actions"]
|
||||
if args.only_unambiguous: actions = actions_for_only_unambiguous(actions)
|
||||
applied = 0
|
||||
if args.apply:
|
||||
applied = apply_actions(actions)
|
||||
result["summary"]["mode"] = "apply" if args.apply else "dry-run"; result["summary"]["applied"] = applied
|
||||
write_reports(result, args.output_json, args.output_csv)
|
||||
print(json.dumps(result["summary"], ensure_ascii=False, indent=2, default=str))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__": raise SystemExit(main())
|
||||
107
scripts/preflight_document_reconciliation_v2.py
Normal file
107
scripts/preflight_document_reconciliation_v2.py
Normal file
@@ -0,0 +1,107 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Read-only preflight for migration 007; it never repairs production data."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict
|
||||
|
||||
from sqlalchemy import text
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
|
||||
def run_preflight(conn: Any) -> Dict[str, Any]:
|
||||
column_rows = list(conn.execute(text("""
|
||||
SELECT table_name, column_name, data_type, udt_name FROM information_schema.columns
|
||||
WHERE table_schema='public' AND table_name IN
|
||||
('commercial_document_lines','commercial_documents','opportunities')
|
||||
""")))
|
||||
columns = {row[1] for row in column_rows if row[0] == "commercial_document_lines"}
|
||||
required = {"id", "document_id"}
|
||||
counts: Dict[str, int] = {}
|
||||
details: Dict[str, list[Dict[str, Any]]] = {}
|
||||
blockers = []
|
||||
missing = sorted(required - columns)
|
||||
if missing:
|
||||
blockers.append("commercial_document_lines missing columns: " + ", ".join(missing))
|
||||
return {"counts": counts, "blockers": blockers, "columns": sorted(columns)}
|
||||
types = {(row[0], row[1]): row[3] for row in column_rows}
|
||||
for table_name, column_name in (("commercial_document_lines", "document_id"),
|
||||
("commercial_documents", "id"),
|
||||
("commercial_documents", "opportunity_id"),
|
||||
("opportunities", "id")):
|
||||
actual = types.get((table_name, column_name))
|
||||
if actual != "uuid":
|
||||
blockers.append(f"unexpected type {table_name}.{column_name}: {actual or 'missing'} (expected uuid)")
|
||||
counts["null_document_id"] = int(conn.execute(text(
|
||||
"SELECT count(*) FROM commercial_document_lines WHERE document_id IS NULL"
|
||||
)).scalar() or 0)
|
||||
counts["orphan_document_id"] = int(conn.execute(text("""
|
||||
SELECT count(*) FROM commercial_document_lines dl
|
||||
LEFT JOIN commercial_documents d ON d.id=dl.document_id
|
||||
WHERE dl.document_id IS NOT NULL AND d.id IS NULL
|
||||
""")).scalar() or 0)
|
||||
duplicate_documents = [dict(row) for row in conn.execute(text("""
|
||||
SELECT system, external_id, document_kind, company,
|
||||
array_agg(id::text ORDER BY id) AS ids,
|
||||
'duplicate_external_document' AS reason
|
||||
FROM commercial_documents
|
||||
WHERE external_id IS NOT NULL AND btrim(external_id) <> ''
|
||||
GROUP BY system, external_id, document_kind, company HAVING count(*)>1
|
||||
""")).mappings().all()]
|
||||
duplicate_associations = [dict(row) for row in conn.execute(text("""
|
||||
SELECT opportunity_id::text, system, external_id, document_kind,
|
||||
array_agg(id::text ORDER BY id) AS ids,
|
||||
'duplicate_opportunity_document_association' AS reason
|
||||
FROM commercial_documents
|
||||
WHERE opportunity_id IS NOT NULL AND external_id IS NOT NULL AND btrim(external_id) <> ''
|
||||
GROUP BY opportunity_id, system, external_id, document_kind HAVING count(*)>1
|
||||
""")).mappings().all()]
|
||||
cross_opportunity = [dict(row) for row in conn.execute(text("""
|
||||
SELECT system, external_id, document_kind, company,
|
||||
array_agg(id::text ORDER BY id) AS ids,
|
||||
array_agg(DISTINCT opportunity_id::text) AS opportunity_ids,
|
||||
'external_document_linked_to_multiple_opportunities' AS reason
|
||||
FROM commercial_documents
|
||||
WHERE opportunity_id IS NOT NULL AND external_id IS NOT NULL AND btrim(external_id) <> ''
|
||||
GROUP BY system, external_id, document_kind, company
|
||||
HAVING count(DISTINCT opportunity_id)>1
|
||||
""")).mappings().all()]
|
||||
multiple_primaries = [dict(row) for row in conn.execute(text("""
|
||||
SELECT opportunity_id::text, document_kind, array_agg(id::text ORDER BY id) AS ids,
|
||||
'multiple_legacy_primaries' AS reason
|
||||
FROM commercial_documents
|
||||
WHERE opportunity_id IS NOT NULL AND COALESCE(is_primary,FALSE)
|
||||
AND COALESCE(role,'current') IN ('current','accepted')
|
||||
GROUP BY opportunity_id, document_kind HAVING count(*)>1
|
||||
""")).mappings().all()]
|
||||
details.update(duplicate_documents=duplicate_documents,
|
||||
duplicate_associations=duplicate_associations,
|
||||
cross_opportunity_documents=cross_opportunity,
|
||||
multiple_legacy_primaries=multiple_primaries)
|
||||
counts["duplicate_document_groups"] = len(duplicate_documents)
|
||||
counts["duplicate_associations"] = len(duplicate_associations)
|
||||
counts["cross_opportunity_documents"] = len(cross_opportunity)
|
||||
counts["multiple_legacy_primaries"] = len(multiple_primaries)
|
||||
for key in ("null_document_id", "orphan_document_id", "duplicate_document_groups",
|
||||
"duplicate_associations", "cross_opportunity_documents"):
|
||||
if counts[key]:
|
||||
blockers.append(f"{key}: {counts[key]}")
|
||||
if counts["multiple_legacy_primaries"]:
|
||||
blockers.append(f"multiple_legacy_primaries: {counts['multiple_legacy_primaries']}")
|
||||
return {"counts": counts, "blockers": blockers, "details": details, "columns": sorted(columns)}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
from app.db import engine
|
||||
with engine.begin() as conn:
|
||||
result = run_preflight(conn)
|
||||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||||
return 2 if result["blockers"] else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user