"""M1-LONG linkage unit-test stub — IDPREV/DATEPR rules (synthetic fixtures). Research-only. Encodes rules from docs/m1/WAVE2B_M1_LONG_PILOT_PLAN.md §2. No Michigan SCA microdata required yet; when CSV arrives, extend these tests against real extracts under data/raw/michigan_sca/microdata/. Hard rules encoded here: - Reject ID-only / CASEID-only linkage. - Primary 12-month link: destination (IDPREV2, DATEPR2) → origin (ID, YYYYMM). - Chain sanity: (IDPREV, DATEPR) → prior (ID, YYYYMM), then once more to origin; must agree with IDPREV2/DATEPR2 when both present. - Fresh rows have missing IDPREV/DATEPR. - Duplicate person keys after linkage = fail. - gap_months recorded; design target ≈ 12. Do not authorize M1 scores or touch L1 production. """ from __future__ import annotations from dataclasses import dataclass from typing import Iterable, Optional import pytest # --------------------------------------------------------------------------- # Synthetic fixture rows (no microdata file) # --------------------------------------------------------------------------- # SAMPLE codes from pilot plan §3.5 SAMPLE_FRESH = {1, 3, 6} SAMPLE_REINT2 = {2, 4, 7} SAMPLE_REINT3 = {5, 8} @dataclass(frozen=True) class ScaRow: caseid: str yyyymm: int id: int idprev: Optional[int] datepr: Optional[int] idprev2: Optional[int] datepr2: Optional[int] sample: int pexp: Optional[int] pago: Optional[int] def _synthetic_panel() -> list[ScaRow]: """Minimal 1→2→3 chain for one person + distractors. Person P: fresh 202001 id=10 PEXP=1 reint2 202007 id=20 (links via IDPREV/DATEPR) reint3 202101 id=30 PAGO=5 (IDPREV2/DATEPR2 → fresh) """ return [ ScaRow( caseid="F-202001-10", yyyymm=202001, id=10, idprev=None, datepr=None, idprev2=None, datepr2=None, sample=1, # fresh landline pexp=1, pago=3, ), ScaRow( caseid="R2-202007-20", yyyymm=202007, id=20, idprev=10, datepr=202001, idprev2=None, datepr2=None, sample=2, # reinterview 2 pexp=3, pago=3, ), ScaRow( caseid="R3-202101-30", yyyymm=202101, id=30, idprev=20, datepr=202007, idprev2=10, datepr2=202001, sample=5, # second reinterview pexp=None, pago=5, ), # Distractor: another fresh with same within-month id in a different month # (proves ID-only matching would be unsafe if months differ / collide). ScaRow( caseid="F-201901-10", yyyymm=201901, id=10, idprev=None, datepr=None, idprev2=None, datepr2=None, sample=3, pexp=5, pago=5, ), # Broken row: claims IDPREV2 that does not exist at DATEPR2 ScaRow( caseid="BAD-202102-99", yyyymm=202102, id=99, idprev=88, datepr=202008, idprev2=77, datepr2=202002, sample=8, pexp=None, pago=1, ), ] # --------------------------------------------------------------------------- # Linkage helpers (stub implementations for unit tests) # --------------------------------------------------------------------------- def yyyymm_gap_months(origin: int, dest: int) -> int: """Calendar-month gap YYYYMM_dest − YYYYMM_origin.""" oy, om = divmod(origin, 100) dy, dm = divmod(dest, 100) return (dy - oy) * 12 + (dm - om) def index_by_id_yyyymm(rows: Iterable[ScaRow]) -> dict[tuple[int, int], ScaRow]: idx: dict[tuple[int, int], ScaRow] = {} for r in rows: key = (r.id, r.yyyymm) if key in idx: raise ValueError(f"duplicate (ID, YYYYMM) key in panel: {key}") idx[key] = r return idx def link_id_only(dest: ScaRow, rows: Iterable[ScaRow]) -> list[ScaRow]: """FORBIDDEN heuristic: match on ID alone (ignores DATEPR). Exists only so tests can assert we reject it. """ return [r for r in rows if r.id == dest.idprev2 and r is not dest] def link_primary_idprev2(dest: ScaRow, by_key: dict[tuple[int, int], ScaRow]) -> Optional[ScaRow]: """Primary link: (IDPREV2, DATEPR2) → (ID, YYYYMM).""" if dest.idprev2 is None or dest.datepr2 is None: return None return by_key.get((dest.idprev2, dest.datepr2)) def link_chain_to_origin( dest: ScaRow, by_key: dict[tuple[int, int], ScaRow] ) -> Optional[ScaRow]: """Chain: dest --(IDPREV,DATEPR)→ mid --(IDPREV,DATEPR)→ origin.""" if dest.idprev is None or dest.datepr is None: return None mid = by_key.get((dest.idprev, dest.datepr)) if mid is None: return None if mid.idprev is None or mid.datepr is None: return None return by_key.get((mid.idprev, mid.datepr)) def build_primary_pairs(rows: list[ScaRow]) -> list[dict]: """Build interview-3 → interview-1 pairs under pilot inclusion rules.""" by_key = index_by_id_yyyymm(rows) pairs: list[dict] = [] seen_person_keys: set[tuple[int, int]] = set() for dest in rows: if dest.sample not in SAMPLE_REINT3 and not ( dest.idprev2 is not None and dest.datepr2 is not None ): continue if dest.pago not in (1, 3, 5): continue origin = link_primary_idprev2(dest, by_key) if origin is None: continue if origin.sample not in SAMPLE_FRESH: continue if origin.pexp not in (1, 3, 5): continue chained = link_chain_to_origin(dest, by_key) if chained is not None and ( chained.id != origin.id or chained.yyyymm != origin.yyyymm ): raise AssertionError( "IDPREV2 path disagrees with chained IDPREV path: " f"primary={origin.caseid} chain={chained.caseid}" ) person_key = (origin.id, origin.yyyymm) if person_key in seen_person_keys: raise AssertionError(f"duplicate person key after linkage: {person_key}") seen_person_keys.add(person_key) pairs.append( { "origin_caseid": origin.caseid, "dest_caseid": dest.caseid, "origin_id": origin.id, "origin_yyyymm": origin.yyyymm, "dest_id": dest.id, "dest_yyyymm": dest.yyyymm, "pexp": origin.pexp, "pago": dest.pago, "gap_months": yyyymm_gap_months(origin.yyyymm, dest.yyyymm), } ) return pairs # --------------------------------------------------------------------------- # Tests # --------------------------------------------------------------------------- def test_reject_id_only_linkage_ambiguous(): """ID-only match hits multiple rows with the same ID across months.""" rows = _synthetic_panel() dest = next(r for r in rows if r.caseid == "R3-202101-30") id_only_hits = link_id_only(dest, rows) # Fresh 202001 id=10 AND distractor fresh 201901 id=10 assert len(id_only_hits) >= 2 # Correct primary link is unique via (IDPREV2, DATEPR2) by_key = index_by_id_yyyymm(rows) primary = link_primary_idprev2(dest, by_key) assert primary is not None assert primary.caseid == "F-202001-10" assert primary.yyyymm == dest.datepr2 def test_primary_idprev2_datepr2_link(): rows = _synthetic_panel() by_key = index_by_id_yyyymm(rows) dest = next(r for r in rows if r.caseid == "R3-202101-30") origin = link_primary_idprev2(dest, by_key) assert origin is not None assert (origin.id, origin.yyyymm) == (dest.idprev2, dest.datepr2) assert origin.pexp == 1 assert dest.pago == 5 def test_chain_idprev_agrees_with_idprev2(): rows = _synthetic_panel() by_key = index_by_id_yyyymm(rows) dest = next(r for r in rows if r.caseid == "R3-202101-30") primary = link_primary_idprev2(dest, by_key) chained = link_chain_to_origin(dest, by_key) assert primary is not None and chained is not None assert (primary.id, primary.yyyymm) == (chained.id, chained.yyyymm) def test_fresh_rows_missing_idprev_datepr(): rows = _synthetic_panel() fresh = [r for r in rows if r.sample in SAMPLE_FRESH] assert fresh, "fixture must include fresh rows" for r in fresh: assert r.idprev is None and r.datepr is None assert r.idprev2 is None and r.datepr2 is None def test_gap_months_near_twelve(): assert yyyymm_gap_months(202001, 202101) == 12 assert yyyymm_gap_months(202001, 202012) == 11 assert yyyymm_gap_months(202001, 202102) == 13 pairs = build_primary_pairs(_synthetic_panel()) assert len(pairs) == 1 assert pairs[0]["gap_months"] == 12 assert pairs[0]["gap_months"] in range(11, 14) def test_broken_idprev2_does_not_pair(): rows = _synthetic_panel() pairs = build_primary_pairs(rows) assert all(p["dest_caseid"] != "BAD-202102-99" for p in pairs) def test_duplicate_person_key_fails(): """Two destinations claiming the same origin key must fail loudly.""" rows = list(_synthetic_panel()) # Clone a second interview-3 pointing at the same origin rows.append( ScaRow( caseid="R3-DUP-31", yyyymm=202101, id=31, idprev=20, datepr=202007, idprev2=10, datepr2=202001, sample=5, pexp=None, pago=1, ) ) with pytest.raises(AssertionError, match="duplicate person key"): build_primary_pairs(rows) def test_chain_disagreement_fails(): """If IDPREV chain resolves to a different origin than IDPREV2, fail.""" rows = [ ScaRow("F1", 202001, 10, None, None, None, None, 1, 1, 3), ScaRow("F2", 202002, 11, None, None, None, None, 1, 1, 3), ScaRow("M", 202007, 20, 11, 202002, None, None, 2, 3, 3), # chain mid→F2 ScaRow( "D", 202101, 30, 20, 202007, 10, # IDPREV2 claims F1 202001, 5, None, 5, ), ] with pytest.raises(AssertionError, match="disagrees"): build_primary_pairs(rows) def test_no_microdata_dependency(): """Stub must run without any on-disk SCA microdata extract.""" from pathlib import Path root = Path(__file__).resolve().parents[2] micro = root / "data" / "raw" / "michigan_sca" / "microdata" # Presence is fine later; absence must not break this module's tests. _ = micro # documented expected future path assert build_primary_pairs(_synthetic_panel()) def test_m1_still_unauthorized(): from srp.constants import WAVE1_AUTHORIZATION assert WAVE1_AUTHORIZATION["M1"]["score_authorized"] is False assert WAVE1_AUTHORIZATION["L1"]["score_authorized"] is True assert WAVE1_AUTHORIZATION["L1"]["rule_id"] == "L1-US-v0.1"