LiquidAI
/

LFM2.5-Encoder-350M-PII-Detector

+"""Context-cued ID detection layer (standalone, multilingual).
+The #1 failure of the PII token classifier is the family of *arbitrary
+alphanumeric IDs* (passport, tax_id, national_id, drivers_license, medical_record,
+health_plan_id, bank_account, case_number, login_credentials, password, username,
+device_id). These have NO learnable shape — a passport number, a chart number and a
+purchase-order number are byte-for-byte indistinguishable — so the model gets
+~0 recall on them. But in real documents they almost always appear immediately
+after a *field label / cue* ("Passport No.:", "Reisepass:", "MRN:", "Case No.",
+"Versichertennummer:", "n° de compte", "护照号", ...).
+This module detects the lead-in CUE, then captures the following VALUE token(s) as
+that type. The cue GATES the match -> high precision: it will not fire on a bare
+`PO#778231` / `JIRA-4821` / `ISBN ...` because those cues are not PII field labels.
+Design:
+  * Per type, a list of multilingual cue phrases (regex-escaped, ws-flexible),
+    covering the 33 eval languages (en, de, fr, es, it, nl, pt, pl, ru, zh, ja, ko,
+    ar, hi, id, vi, th, sv, fi, da, el, ro, hu, cs, sk, bg, lt, lv, et, ga, mt).
+  * After a cue, allow <=3 chars of separators (colon/hash/dot/dash/space and a few
+    i18n colons),
+    then a VALUE: an alphanumeric token allowing internal spaces/hyphens/slashes/dots,
+    ``[A-Z0-9][A-Z0-9 ./-]{2,24}[A-Z0-9]``, trimmed, longest plausible.
+  * Robust to value being on the same line as the cue, and (for credentials) to an
+    inline secondary cue inside the value (e.g. "User: sa Pass: Master#01").
+Public API:
+  context_cued_spans(text) -> [{start,end,type,text}, ...]
+  hybrid_with_context(text, model_spans) -> hybrid_spans + the cued tier (authoritative)
+Order in the full decode: AUTH (shape) -> CONTEXT (these IDs, cue-gated) ->
+SNAP (phone/date/amount/postal) -> model spans for the rest.
+"""
+from __future__ import annotations
+import re
+# --------------------------------------------------------------------------- #
+# VALUE: an arbitrary alphanumeric ID token. Must start & end on an alnum char,
+# may carry internal spaces / hyphens / slashes / dots (so "1 90 12 33 123 456 78",
+# "2:20-cv-09876-ABC", "12.345.678/0001-95", "20-00-00 acc: 55779911" all survive).
+# Letters incl. accented Latin so "53219876S", "FK123456", "C01X00T47" survive; we
+# stay Latin-only inside the value (CJK IDs are digit/Latin in this eval).
+_VAL = r"[A-Za-z0-9@.#][A-Za-z0-9 ./_:@()#\-]{2,40}[A-Za-z0-9)]"
+# Credential VALUE: passwords/secret tokens carry special chars (&!#$%@*?) and no
+# spaces. Used ONLY for password / login-credential cues so the wider char set never
+# leaks into the high-precision ID matching. Single token, no internal whitespace.
+_CRED_VAL = re.compile(r"""["']?(?!@)([^\s"'@][^\s"']{3,59})["']?""")
+# separators allowed between cue and value (incl. i18n colons / no.-abbreviations /
+# quotes for `password = "..."`). Slightly longer cap to span `   = "`.
+_SEP = r"""[\s:#=.№　：\-"']{0,6}"""
+# secrets embedded in a connection string: scheme://user:PASSWORD(@host | end). The
+# password must NOT be all-digits (that's a :PORT, e.g. redis://host:6379) and stops
+# before '@host'.
+_CONN_PW = re.compile(r"(?i)\b[a-z][a-z0-9+.\-]*://[^\s:@/]+:(?!\d+(?:[@\s\"']|$))([^\s:@/]{3,})(?=@|[\s\"']|$)")
+# A trailing value must contain at least one DIGIT *or* be a credential-style token.
+# IDs in this schema are numeric or alphanumeric-with-digits; pure-word tails after a
+# cue ("Passport: Required") would otherwise false-fire. Credentials/usernames are
+# exempt (passwords/usernames can be all-letters: "hunter2"? has a digit; "password"
+# does not -> we exempt them, gated tightly by the credential cues).
+_HAS_DIGIT = re.compile(r"\d")
+_HAS_ALNUM = re.compile(r"[A-Za-z0-9]")
+# --------------------------------------------------------------------------- #
+# Cue phrases per type. Authored as raw alternation fragments; matched
+# case-insensitively, with flexible internal whitespace. Keep them SPECIFIC to PII
+# field labels so the precision traps (PO#, WO-, PROJ-, JIRA-, TICKET-, RMA-, INC,
+# CHG, EPIC-, BUG-, GH-, ISBN, DOI, PMID, ICD-10, Rule/Section/Chapter/Title,
+# port/commit/tag/Flight/Gate/Lane/Building/Channel/tracking/Receipt/invoice/INV-/
+# ORD-/REF/PURCHASE) do NOT match.
+_CUES: dict[str, list[str]] = {
+    "identity.passport": [
+        r"passport(?:\s*(?:no|number|num|nr|#))?",
+        r"reisepass(?:nr|nummer)?", r"reisepass\s*-?\s*nr",
+        r"n[°ºo]\.?\s*(?:de\s*)?passeport", r"passeport\s*n[°ºo]?",
+        r"n[.°ºo]*\s*(?:de\s*)?pasaporte", r"pasaporte\s*n[°ºo]?",
+        r"passaporto", r"n[°ºo]\.?\s*(?:de\s*)?passaporte", r"paspoort",
+        r"numer\s*paszportu", r"paszport",
+        r"パスポート(?:番号)?",  # パスポート(番号)
+        r"여권(?:번호)?",                    # 여권(번호)
+        r"护照号?",                              # 护照(号)
+        r"رقم\s*جواز\s*السفر", r"جواز\s*سفر(?:\s*رقم)?",  # رقم جواز السفر / جواز سفر رقم
+        r"पासपोर्ट",  # पासपोर्ट (hi)
+        r"паспорт",        # паспорт (ru/bg)
+        r"диабатирио",  # (filler-safe)
+        r"διαβατήριο",  # διαβατήριο (el)
+        r"pasul?(?:uri)?",  # ro passport-ish (guarded by digit value)
+        r"h[oó]\s*chi[ếe]u", r"so\s*h[oó]\s*chi[ếe]u",        # hộ chiếu (vi)
+        r"หนังสือเดินทาง",  # หนังสือเดินทาง (th)
+        r"pase", r"pas\s*nr", r"reisedokument", r"uütlevee",  # da/sv/et-ish
+    ],
+    "identity.tax_id": [
+        r"tax\s*(?:id|identification|no|number|#)?", r"\bTIN\b", r"\bEIN\b",
+        r"vat\s*(?:id|no|number|reg|registration|#)?", r"\bVAT\b",
+        r"steuer\s*-?\s*id", r"steuernummer", r"steuer\s*-?\s*nr", r"ust\s*-?\s*idnr",
+        r"umsatzsteuer", r"\bNIF\b", r"\bCIF\b", r"\bCPF\b", r"\bCNPJ\b", r"\bRFC\b",
+        r"\bNIP\b", r"\bPAN\b", r"पैन", r"\bΑΦΜ\b", r"αφμ", r"codice\s*fiscale", r"partita\s*iva",
+        r"num[ée]ro\s*(?:fiscal|de\s*tva)", r"identifiant\s*fiscal",
+        r"momsnr", r"momsregistrerings", r"btw\s*-?\s*nr", r"\bDPH\b",
+        r"税号", r"纳税人识别号",  # 税号 / 纳税人识别号
+        r"расчётный\s*номер",  # ru tax-ish
+        r"ИНН", r"инн",            # ИНН (ru tax id)
+        r"マイナンバー",                # マイナンバー (ja)
+        r"税務", r"세금", r"사업자등록번호",
+        r"رقم\s*ضريبي",  # رقم ضريبي (ar)
+        r"adoazonos[ií]t[oó]", r"ad[oó]sz[aá]m",                # hu tax id
+        r"daňov[eé]\s*č[ií]slo", r"ičo\s*dph",
+        r"mok[ėe]t[oų]jo\s*kodas", r"pvm",                      # lt vat
+    ],
+    "identity.national_id": [
+        r"national\s*(?:id|identity|insurance)\s*(?:no|number|#)?",
+        r"\bNIN\b", r"\bNINO\b", r"\bDNI\b", r"\bNIE\b", r"\bNIF\b",
+        r"personalausweis(?:nr|nummer)?", r"ausweis\s*-?\s*nr",
+        r"identit[ée]\s*nationale", r"carte\s*nationale", r"num[ée]ro\s*national",
+        r"s[ée]curit[ée]\s*sociale", r"num[ée]ro\s*de\s*s[ée]curit[ée]\s*sociale",
+        r"n[°ºo]\.?\s*(?:de\s*)?s[ée]curit[ée]", r"insee",
+        r"documento\s*nacional", r"documento\s*de\s*identidad", r"c[ée]dula",
+        r"carta\s*d['’]?identit[aà]", r"codice\s*identit[aà]",
+        r"\bBSN\b", r"burgerservicenummer", r"\bPESEL\b", r"\brodn[eé]\s*č[ií]slo\b",
+        r"personnummer", r"henkil[öo]tunnus", r"cpr\s*-?\s*nr", r"\bCPR\b",
+        r"isikukood", r"personas\s*kods", r"asmens\s*kodas",
+        r"身份证(?:号|号码)?",        # 身份证(号)
+        r"身份證(?:字號)?",               # 身份證(字號)
+        r"주민(?:등록)?번호",         # 주민(등록)번호
+        r"マイナンバー",              # マイナンバー
+        r"рациональный",  # национальный
+        r"номер\s*паспорта",
+        r"आधार(?:\s*संख्या|\s*नंबर)?",  # आधार (संख्या) (hi)
+        r"رقم\s*وطني", r"الرقم\s*الوطني",  # رقم وطني
+        r"บัตรประชาชน",  # บัตรประชาชน (th)
+        r"ΑΔΤ", r"αριθμ[όο]ς\s*ταυτ[όο]τητας",  # ΑΔΤ
+        r"cnp", r"cod\s*numeric\s*personal",                  # ro
+        r"szem[eé]lyi\s*(?:azonos[ií]t[oó]|igazolv[aá]ny)",   # hu
+        r"so\s*cmnd", r"can\s*cu[oơ]c", r"cmnd", r"cccd",      # vi
+        r"เลขบัตร",        # th id
+        r"\bNRIC\b", r"\bKTP\b", r"nomor\s*induk\s*kependudukan", r"\bNIK\b",  # id
+    ],
+    "identity.drivers_license": [
+        r"driver'?s?\s*licen[cs]e\s*(?:no|number|#)?", r"\bDL\b\s*#?",
+        r"driving\s*licen[cs]e", r"f[üu]hrerschein(?:nr|nummer)?",
+        r"permis\s*de\s*conduire", r"permis\s*conduire",
+        r"permiso\s*de\s*conducir", r"carnet\s*de\s*conducir", r"licencia\s*de\s*conducir",
+        r"patente\s*(?:di\s*guida|nr)?", r"rijbewijs",
+        r"prawo\s*jazdy", r"k[öo]rkort", r"ajokortti", r"f[øo]rerbevis", r"k[øo]rekort",
+        r"运转驾驶证", r"驾驶证",      # 驾驶证 (zh)
+        r"運転免許", r"免許",                  # 運転免許 (ja)
+        r"운전면허(?:증)?",                       # 운전면허(증)
+        r"рукавительское",  # filler
+        r"водительское\s*удостоверение",  # ru
+        r"رخصة\s*(?:ال)?قيادة",  # رخصة القيادة
+        r"permis\s*de\s*conducere", r"vezet[őo]i\s*enged[eé]ly",       # ro/hu
+        r"ใบขับขี่",            # ใบขับขี่ (th)
+        r"gi[aâ]y\s*ph[eé]p\s*l[aá]i\s*xe", r"b[aă]ng\s*l[aá]i",        # vi
+    ],
+    "healthcare.medical_record": [
+        r"\bMRN\b", r"medical\s*record\s*(?:no|number|#)?", r"med\.?\s*rec\.?\s*(?:no|#)?",
+        r"chart\s*(?:no|number|#)?", r"patient\s*(?:id|no|number|#)",
+        r"\bUR\s*(?:number|no|#)?\b", r"\bNHS\s*(?:no|number)?\b", r"health\s*record",
+        r"aktenzeichen", r"patientennummer", r"fallnummer", r"patienten\s*-?\s*id",
+        r"n[°ºo]\.?\s*(?:de\s*)?dossier(?:\s*m[ée]dical)?", r"dossier\s*m[ée]dical",
+        r"dossiernummer", r"nr\.?\s*dosar(?:\s*medical)?", r"dosar\s*medical",
+        r"nr\.?\s*karty", r"medicininės\s*kortelės\s*nr", r"kortelės\s*nr",
+        r"n[ºo]\.?\s*(?:de\s*)?historia\s*cl[ií]nica", r"historia\s*cl[ií]nica",
+        r"numero\s*de\s*historia", r"n[uú]mero\s*de\s*historia",
+        r"cartella\s*clinica", r"numero\s*cartella",
+        r"prontu[áa]rio", r"pacientennummer", r"pati[ëe]ntnummer",
+        r"病历号", r"病歷號", r"医疗记录",  # 病历号
+        r"カルテ番号", r"患者番号",            # カルテ番号 (ja)
+        r"차트번호", r"환자번호",                 # 차트번호 (ko)
+        r"регистрационный", # filler
+        r"номер\s*медицинской\s*карты",  # ru
+        r"رقم\s*الملف\s*الطبي",  # رقم الملف الطبي
+        r"เวชระเบียน", # th medical record
+    ],
+    "healthcare.health_plan_id": [
+        r"health\s*plan\s*(?:id|no|number|#)?(?:\s*is)?", r"member\s*(?:id|no|number|#)",
+        r"id\s*plan\s*de\s*s[aă]n[aă]tate", r"plan\s*de\s*s[aă]n[aă]tate",
+        r"convenio", r"conv[êe]nio", r"स्वास्थ्य\s*योजना(?:\s*संख्या)?",
+        r"policy\s*(?:no|number|#|id)", r"insurance\s*(?:id|no|number|#)",
+        r"plan\s*id", r"subscriber\s*(?:id|no|#)", r"group\s*(?:no|number|#)\s*id",
+        r"versichertennummer", r"versicherten\s*-?\s*nr", r"krankenversicherung",
+        r"versicherungsnummer", r"\bAOK\b\s*versichert",
+        r"n[°ºo]\.?\s*(?:de\s*)?mutuelle", r"num[ée]ro\s*d['’]?assur[ée]", r"\bCPAM\b",
+        r"n[ºo]\.?\s*(?:de\s*)?(?:p[óo]liza|seguro)", r"n[uú]mero\s*de\s*afiliaci[óo]n",
+        r"tessera\s*sanitaria", r"polizza", r"numero\s*assicurato",
+        r"zorgverzekering", r"polisnummer",
+        r"保险号", r"医保号", r"医疗保险",  # 保险号
+        r"保険証番号", r"被保険者番号",  # 保険証番号 (ja)
+        r"보험증번호", r"건강보험",            # 보험증번호 (ko)
+        r"номер\s*полиса", r"ОМС",  # номер полиса / ОМС
+        r"رقم\s*(?:الت[أا])?مين",  # رقم التأمين
+        r"szem[eé]lyi\s*biztos[ií]t",  # hu insurance-ish
+    ],
+    "financial.bank_account": [
+        r"bank\s*account\s*(?:no|number|#)?", r"\baccount\s*(?:no|number|#)",
+        r"\bacct\b\.?\s*(?:no|#)?", r"\bacc\b\.?\s*(?:no|#)?", r"\ba/?c\b\s*(?:no|#)?",
+        r"checking\s*(?:account|no|#)?", r"savings\s*(?:account|no|#)?",
+        r"sort\s*code", r"routing\s*(?:no|number|#)?", r"\bABA\b", r"transit\s*(?:no|#)?",
+        r"konto(?:nummer|nr)?", r"konto\s*-?\s*nr", r"bankverbindung",
+        r"n[°ºo]\.?\s*(?:de\s*)?compte", r"compte\s*bancaire", r"\bRIB\b",
+        r"n[ºo]\.?\s*(?:de\s*)?cuenta", r"cuenta\s*bancaria", r"numero\s*de\s*cuenta",
+        r"conto\s*(?:corrente|bancario)?", r"numero\s*di\s*conto",
+        r"conta\s*(?:banc[áa]ria|corrente)?", r"n[uú]mero\s*da\s*conta",
+        r"rekeningnummer", r"bankrekening",
+        r"numer\s*konta", r"nr\s*konta", r"kontonr", r"bankkonto",
+        r"kontonummer", r"tilinumero", r"pankkitili", r"konto\s*nr",
+        r"银行账号", r"账号", r"帐号", r"銀行口座",  # 银行账号
+        r"口座番号", r"銀行口座",                  # 口座番号 (ja)
+        r"계좌번호", r"은행계좌",                 # 계좌번호 (ko)
+        r"номер\s*счета", r"расчетный\s*счет",  # номер счета
+        r"рахм\s*алхисаб",  # رقم الحساب-ish
+        r"رقم\s*الحساب",             # رقم الحساب (ar)
+        r"cont\s*bancar", r"num[aă]r\s*de\s*cont",                                # ro
+        r"banksz[aá]mla", r"sz[aá]mlasz[aá]m",                                    # hu
+        r"so\s*t[aà]i\s*kho[aả]n", r"t[aà]i\s*kho[aả]n",                          # vi
+        r"เลขที่บัญชี",    # เลขที่บัญชี (th)
+    ],
+    "legal.case_number": [
+        r"case\s*(?:no|number|#)", r"docket\s*(?:no|number|#)?",
+        r"cause\s*(?:no|number)", r"indictment\s*(?:no|number)", r"file\s*(?:no|number)",
+        r"aktenzeichen", r"gesch[äa]ftsnummer", r"\bAz\.?\s*:",
+        r"n[°ºo]\.?\s*(?:de\s*)?(?:r[ôo]le|dossier|affaire)", r"r[ée]f[ée]rence\s*affaire",
+        r"\bR\.?\s*G\.?\s*n", r"numero\s*di\s*ruolo", r"procedimento\s*n",
+        r"n[ºo]\.?\s*(?:de\s*)?(?:expediente|procedimiento|causa)", r"autos\s*n",
+        r"sygnatura(?:\s*akt)?", r"\bsygn\.?\s*akt\b",
+        r"zaaknummer", r"rolnummer", r"m[ åa]lnummer", r"sagsnr", r"asianumero",
+        r"案件号", r"案号", r"案件编号",       # 案件号 (zh)
+        r"事件��号", r"裁判番号",                 # 事件番号 (ja)
+        r"사건번호",                                             # 사건번호 (ko)
+        r"номер\s*дела", r"дело\s*№",  # номер дела
+        r"رقم\s*القضية",             # رقم القضية (ar)
+        r"num[aă]r\s*(?:dosar|de\s*[ií]nregistrare)", r"dosar\s*nr",              # ro
+        r"[üu]gysz[aá]m", r"\bb[ií]r[oó]s[aá]gi\b",                               # hu
+        r"so\s*v[uụ]\s*[aá]n", r"so\s*h[oồ]\s*s[oơ]",                             # vi
+        r"αριθμ[όο]ς\s*υπ[οό]θεσης",  # el
+        r"spr[aá]vne\s*č[ií]slo", r"č[ií]slo\s*jednac[ií]",            # cs
+    ],
+    "developer.login_credentials": [
+        r"login\s*credentials?", r"credentials?", r"login\s*=?", r"logon",
+        r"oauth_token", r"oauth_secret", r"access[_\s]*token", r"auth\s*token",
+        r"anmeldedaten", r"zugangsdaten", r"identifiants?\s*de\s*connexion",
+        r"credenciales", r"credenziali", r"inloggegevens", r"dane\s*logowania",
+        r"登录凭据", r"ログイン情報",
+        r"인증정보", r"бианиевые",
+        r"учётные\s*данные",  # учётные данные (ru)
+    ],
+    "credential.password": [
+        r"password", r"passwd", r"\bpwd\b", r"pass\b", r"passphrase",
+        r"passwort", r"kennwort", r"mot\s*de\s*passe", r"contrase[ñn]a",
+        r"senha", r"wachtwoord", r"has[łl]o", r"heslo", r"l[öo]senord", r"salasana", r"adgangskode",
+        r"密码", r"パスワード", r"비밀번호",
+        r"пароль",                                  # пароль (ru/bg)
+        r"كلمة\s*(?:ال)?سر",              # كلمة السر (ar)
+        r"รหัสผ่าน",                      # รหัสผ่าน (th)
+        r"m[aậ]t\s*kh[aẩ]u", r"jelsz[oó]", r"parol[aă]", r"sl[aā]žvārds",
+    ],
+    "online.username": [
+        r"username", r"user\s*name", r"\buser\b", r"\buserid\b", r"user\s*id",
+        r"handle", r"account\s*name", r"\bacct\s*name\b", r"screen\s*name",
+        r"login\s*name", r"nick(?:name)?", r"\bid\s*utilisateur\b",
+        r"benutzername", r"benutzer\b", r"nom\s*d['’]?utilisateur", r"identifiant",
+        r"nombre\s*de\s*usuario", r"usuario", r"nome\s*utente", r"nome\s*de\s*usu[áa]rio",
+        r"gebruikersnaam", r"nazwa\s*u[żz]ytkownika", r"u[żz]ytkownik",
+        r"anv[äa]ndarnamn", r"k[äa]ytt[äa]j[äa]tunnus", r"vartotojas", r"brugernavn",
+        r"用户名", r"ユーザー名", r"사용자명", r"아이디",  # 用户名 / ユーザー名
+        r"имя\s*пользователя", r"пользователь", r"логин",  # имя пользователя / пользователь / логин
+        r"اسم\s*المستخدم",  # اسم المستخدم
+        r"ชื่อผู้ใช้",          # ชื่อผู้ใช้ (th)
+        r"t[eê]n\s*(?:đăng\s*nh[aậ]p|ng[uư][oờ]i\s*d[uù]ng)",          # vi
+        r"felhaszn[aá]l[oó]n[eé]v",  # hu
+    ],
+    "developer.device_id": [
+        r"device\s*(?:id|asset\s*tag|serial)?", r"device\s*asset\s*tag", r"asset\s*tag",
+        r"\bIMEI\b", r"\bUDID\b", r"\bESN\b", r"\bMEID\b", r"\bSN\b\s*:?",
+        r"serial\s*(?:no|number|#)?", r"seriennummer", r"ger[äa]te\s*-?\s*id",
+        r"asset\s*-?\s*nr", r"asset\s*-?\s*nummer", r"ger[äa]tenummer",
+        r"num[ée]ro\s*de\s*s[ée]rie", r"identifiant\s*(?:de\s*l['’]?)?appareil",
+        r"n[uú]mero\s*de\s*serie", r"identificador\s*de\s*dispositivo",
+        r"numero\s*di\s*serie", r"id\s*dispositivo", r"apparaat\s*id",
+        r"设备号", r"设备标识", r"序列号",  # 设备号
+        r"デバイス番号", r"シリアル番号",  # デバイス番号
+        r"장치아이디", r"일련번호",            # 장치 아이디 (ko)
+        r"номер\s*устройства",  # номер устройства
+        r"رقم\s*الجهاز",             # رقم الجهاز (ar)
+    ],
+}
+# Inline secondary cues for credential pairs: inside a login_credentials VALUE the
+# user/pass tokens carry their own micro-cues. We keep the whole "User: x Pass: y"
+# region as one login_credentials span (matches the gold which does the same).
+_LOGIN_PAIR = re.compile(
+    r"(?i)\b(?:user|usuario|benutzer|utilisateur|u|login|name)\b\s*[:=]?\s*\S+"
+    r".{0,8}?\b(?:pass(?:word)?|pwd|passwort|mot\s*de\s*passe|p|token|secret)\b\s*[:=]?\s*\S+"
+)
+# Compile: (type, compiled cue regex). Longer/more-specific cues first within a type
+# so the alternation prefers the most specific label.
+def _compile(cues: list[str]) -> re.Pattern:
+    # sort by descending raw length so e.g. "national insurance no" beats "id"
+    ordered = sorted(cues, key=len, reverse=True)
+    return re.compile(r"(?i)(?:" + r"|".join(ordered) + r")")
+_CUE_RX = [(t, _compile(cs)) for t, cs in _CUES.items()]
+_VAL_RX = re.compile(_VAL)
+# types whose value may be all-letters (no digit required)
+_LETTER_OK = {"credential.password", "online.username", "developer.login_credentials"}
+# a username/handle: single token (handles may carry @ . _ -), NOT a capitalized prose
+# word ("Holder", "Statement", "Bank"). Rejects multi-word values and Title-case words.
+_PROSE_WORD = re.compile(r"^[A-ZÀ-Þ][a-zà-ÿ]+$")
+def _username_ok(val: str, sep: str) -> bool:
+    if " " in val:                       # usernames don't contain spaces
+        return False
+    if _PROSE_WORD.match(val):           # 'Holder', 'Statement', 'Bank', 'Sort'
+        return False
+    # require a real label delimiter (':' '=') OR an '@'-handle: a bare "User cannot"
+    # (cue + space + word) is prose, not a labelled field.
+    if not (":" in sep or "=" in sep or val.startswith("@")):
+        return False
+    return True
+# login_credentials standalone value must look credential-ish (has digit/symbol or '='),
+# not a bare prose word ('attempts', 'success', 'cannot').
+def _login_ok(val: str) -> bool:
+    if _PROSE_WORD.match(val) or (val.isalpha() and val.islower()):
+        return False
+    return True
+# password-context English words that follow the cue 'password' in prose
+# ("password authentication failed", "password expired") -> not a password value.
+_PW_STOP = {"authentication", "expired", "required", "reset", "change", "changed",
+            "incorrect", "invalid", "failed", "failure", "policy", "rotation",
+            "manager", "protected", "field", "must", "should", "cannot", "and", "for",
+            "the", "is", "was", "has", "not", "verification", "recovery", "strength"}
+def _password_ok(val: str) -> bool:
+    if " " in val:
+        return False
+    return val.lower() not in _PW_STOP
+# A short stop-list of cue *contexts* that are traps even though a sub-cue matched.
+# e.g. "ISBN", "DOI", "PMID", "PO#", "JIRA-" never become our types because their
+# cue strings are simply not in _CUES — so no extra guard needed there. But a few
+# generic English words ("user", "pass", "acc", "id", "SN") can appear in non-PII
+# contexts; we gate them by requiring a plausible value right after.
+# value characters that, if the value is ENTIRELY one of these shapes, indicate a
+# version/path/non-PII tail we should reject even after a cue (rare; cue already gates).
+_VERSIONISH = re.compile(r"^v?\d+(?:\.\d+){2,}$")  # 1.29.2, v2.0.0
+def _trim(text: str, s: int, e: int) -> tuple[int, int]:
+    while s < e and not _HAS_ALNUM.match(text[s]):
+        s += 1
+    while e > s and not _HAS_ALNUM.match(text[e - 1]):
+        e -= 1
+    return s, e
+# country/jurisdiction & abbreviation connector tokens that legitimately sit INSIDE an
+# id value ("DL: WA: SMITHJ123AB", "sort: 20-00-00 acc: 55779911", "No. 1 90 12 ...").
+_ID_CONNECTORS = {"no", "no.", "nr", "nr.", "acc", "acc:", "acct", "sort", "bsb",
+                  "transit", "routing", "checking", "savings", "de", "fr", "id",
+                  "uid", "udid", "sn", "esn", "imei", "tva", "vat", "nip", "iva"}
+_LOWER_WORD = re.compile(r"^[a-zà-öø-ÿ]+$")
+def _id_like_chunk(c: str) -> bool:
+    """A space-separated chunk that plausibly continues an arbitrary-ID value:
+    contains a digit, is all-caps, or is a known connector/jurisdiction token. A plain
+    lowercase word ('email', 'phone', 'oder', 'etwas', 'next') is prose -> ends value."""
+    if not c:
+        return False
+    if _HAS_DIGIT.search(c):
+        return True
+    cc = c.rstrip(".:#-/")
+    if not cc:
+        return False
+    if cc.lower() in _ID_CONNECTORS:
+        return True
+    if _LOWER_WORD.match(cc):              # pure lowercase word -> prose, stop
+        return False
+    if cc.isupper():                       # DNI, CPAM, BCBS, WA, NHS, SMITHJ ...
+        return True
+    return False                            # mixed-case word w/o digit -> stop
+def _bound_value(text: str, vs: int, ve: int) -> int:
+    """Stop a multi-token value at the first non-ID-like (prose) chunk, so a value on
+    the same line as following prose ('C01X00T47 email [email protected]') is not over-captured.
+    The first chunk after the cue is always kept (it IS the id head)."""
+    seg = text[vs:ve]
+    if " " not in seg:
+        return ve
+    parts = seg.split(" ")
+    keep = 1
+    for c in parts[1:]:
+        if _id_like_chunk(c):
+            keep += 1
+        else:
+            break
+    if keep == len(parts):
+        return ve
+    new = vs + len(" ".join(parts[:keep]))
+    return new
+def context_cued_spans(text: str) -> list[dict]:
+    """Detect cue -> value ID spans. Returns [{start,end,type,text}], non-overlapping,
+    cue-gated (high precision)."""
+    out = []
+    claimed = [False] * len(text)
+    # Within a type, find every cue occurrence and grab the following value.
+    # Process types in a priority order so that when two cues overlap (e.g. "user"
+    # for username vs "User:" inside a login pair) the more specific wins. We let
+    # login_credentials pairs be detected first (they subsume user/pass micro-cues).
+    candidates = []  # (start, end, type, specificity)
+    sep_rx = re.compile(_SEP)
+    for typ, rx in _CUE_RX:
+        # credential secrets are single tokens with special chars -> use _CRED_VAL
+        cred = typ in ("credential.password", "developer.login_credentials")
+        for m in rx.finditer(text):
+            cue_end = m.end()
+            # capture value starting within _SEP chars after the cue
+            sep = sep_rx.match(text, cue_end)
+            val_start = sep.end() if sep else cue_end
+            if cred:
+                vm = _CRED_VAL.match(text, val_start)
+                if not vm:
+                    continue
+                vs, ve = vm.start(1), vm.end(1)
+            else:
+                vm = _VAL_RX.match(text, val_start)
+                if not vm:
+                    continue
+                vs, ve = _trim(text, vm.start(), vm.end())
+                ve = _bound_value(text, vs, ve)
+                vs, ve = _trim(text, vs, ve)
+            if ve - vs < 3:
+                continue
+            val = text[vs:ve]
+            # gating
+            if typ not in _LETTER_OK and not _HAS_DIGIT.search(val):
+                continue
+            if _VERSIONISH.match(val.replace(" ", "")):
+                continue
+            if typ == "online.username" and not _username_ok(val, text[cue_end:val_start]):
+                continue
+            if typ == "developer.login_credentials" and not _login_ok(val):
+                continue
+            if typ == "credential.password" and not _password_ok(val):
+                continue
+            # specificity = length of matched cue (longer cue = more specific label)
+            spec = m.end() - m.start()
+            candidates.append((vs, ve, typ, spec, val))
+    # Login-credential pairs: capture the whole user/pass region as one span.
+    for m in _LOGIN_PAIR.finditer(text):
+        s, e = _trim(text, m.start(), m.end())
+        if e - s >= 3:
+            candidates.append((s, e, "developer.login_credentials", 9999, text[s:e]))
+    # Connection-string embedded password: scheme://user:PASSWORD@host
+    for m in _CONN_PW.finditer(text):
+        vs, ve = m.start(1), m.end(1)
+        if ve - vs >= 3:
+            candidates.append((vs, ve, "credential.password", 5000, text[vs:ve]))
+    # Resolve overlaps: prefer higher specificity, then longer span.
+    candidates.sort(key=lambda c: (-c[3], -(c[1] - c[0])))
+    for vs, ve, typ, spec, val in candidates:
+        if any(claimed[vs:ve]):
+            continue
+        for i in range(vs, ve):
+            claimed[i] = True
+        out.append({"start": vs, "end": ve, "type": typ, "text": text[vs:ve]})
+    out.sort(key=lambda d: (d["start"], d["end"]))
+    return out
+# --------------------------------------------------------------------------- #
+# Hybrid variant: AUTH -> CONTEXT (these IDs, authoritative) -> SNAP -> model.
+# We reuse the shipped self-contained decode (hybrid_spans + _AUTH_TYPES) by loading
+# it from the v8 model dir, so AUTH/SNAP logic is never re-implemented here.
+_BASE_DECODE = None
+def _load_base_decode():
+    global _BASE_DECODE
+    if _BASE_DECODE is None:
+        import importlib.util
+        path = "/lambdafs/simon/models/pii-detect-v8/pii_hybrid_decode.py"
+        spec = importlib.util.spec_from_file_location("pii_hybrid_decode_v8", path)
+        mod = importlib.util.module_from_spec(spec)
+        spec.loader.exec_module(mod)
+        _BASE_DECODE = (mod.hybrid_spans, mod._AUTH_TYPES)
+    return _BASE_DECODE
+def hybrid_with_context(text: str, model_spans: list[dict]) -> list[dict]:
+    """Run the existing hybrid decode, then layer the cue-gated CONTEXT tier on top.
+    CONTEXT spans are authoritative for their (Group-A) types: they REPLACE any model
+    span of the same type that they overlap, and own their boundaries. AUTH still wins
+    over CONTEXT on overlap (shape-bearing formats take priority)."""
+    hybrid_spans, _AUTH_TYPES = _load_base_decode()
+    base = hybrid_spans(text, model_spans)
+    cued = context_cued_spans(text)
+    if not cued:
+        return base
+    cued_types = set(_CUES)
+    # mark char ranges owned by AUTH spans (AUTH > CONTEXT)
+    auth_claim = [False] * len(text)
+    for sp in base:
+        if sp["type"] in _AUTH_TYPES:
+            for i in range(sp["start"], sp["end"]):
+                auth_claim[i] = True
+    kept_cued = []
+    for c in cued:
+        if any(auth_claim[c["start"]:c["end"]]):
+            continue  # AUTH owns this region
+        kept_cued.append(c)
+    cued_ranges = [(c["start"], c["end"]) for c in kept_cued]
+    def _overlaps_cued(sp):
+        for s, e in cued_ranges:
+            if min(e, sp["end"]) > max(s, sp["start"]):
+                return True
+        return False
+    out = []
+    for sp in base:
+        # drop model/SNAP spans of a CONTEXT type that overlap a cued span
+        if sp["type"] in cued_types and _overlaps_cued(sp):
+            continue
+        out.append(sp)
+    out.extend(kept_cued)
+    seen, uniq = set(), []
+    for sp in sorted(out, key=lambda s: (s["start"], s["end"])):
+        k = (sp["start"], sp["end"], sp["type"])
+        if k in seen:
+            continue
+        if len(text[sp["start"]:sp["end"]].strip()) < 3:
+            continue
+        seen.add(k)
+        uniq.append(sp)
+    return uniq
+CONTEXT_TYPES = set(_CUES)  # the 12 Group-A types this tier owns