{ "schema_version": "1.0", "artifact_scope": "public_summary", "disclosure": "This public artifact intentionally excludes dev-environment runtime details and working-tree provenance. Full benchmark artifact with runtime + provenance metadata remains available in the internal reviewer bundle.", "generated_at": "2026-06-13T16:31:54Z", "timestamp": "2026-06-13T16:31:54Z", "git": { "head": "7a1a4e1a" }, "runtime": { "python": "3.12.12" }, "benchmark": { "type": "dated_in_repo_precision_recall_snapshot", "notes": [ "This is a dated in-repo benchmark snapshot, not an external audit.", "The adversarial false-positive slices use case-level zero-false-positive precision because their gold labels are intentionally empty.", "The Saudi-name slices measure curated PERSON recall only and do not claim production-wide coverage.", "Report-only hardening slices are observed in the artifact but do not affect the existing release gates until an operator explicitly promotes them.", "The Wojood news-register slice is generated locally from an enclave copy and is intentionally not committed to git." ], "corpora": [ { "id": "policy_code", "kind": "false_positive", "gate_metric": "precision", "gate_min": 0.95, "case_count": 30, "report_only": false }, { "id": "operational_text", "kind": "false_positive", "gate_metric": "precision", "gate_min": 0.95, "case_count": 30, "report_only": false }, { "id": "structured_noise", "kind": "false_positive", "gate_metric": "precision", "gate_min": 0.95, "case_count": 30, "report_only": false }, { "id": "document_bodies", "kind": "false_positive", "gate_metric": "precision", "gate_min": 0.95, "case_count": 40, "report_only": false }, { "id": "bilingual_boilerplate", "kind": "false_positive", "gate_metric": "precision", "gate_min": 0.95, "case_count": 30, "report_only": false }, { "id": "numeric_ambiguity", "kind": "false_positive", "gate_metric": "precision", "gate_min": 0.95, "case_count": 30, "report_only": false }, { "id": "tribal_names", "kind": "name_recall", "gate_metric": "recall", "gate_min": 0.85, "case_count": 50, "report_only": false }, { "id": "female_names", "kind": "name_recall", "gate_metric": "recall", "gate_min": 0.85, "case_count": 40, "report_only": false }, { "id": "modern_names", "kind": "name_recall", "gate_metric": "recall", "gate_min": 0.85, "case_count": 30, "report_only": false }, { "id": "honorifics", "kind": "name_recall", "gate_metric": "recall", "gate_min": 0.85, "case_count": 20, "report_only": false }, { "id": "diaspora_variants", "kind": "name_recall", "gate_metric": "recall", "gate_min": 0.85, "case_count": 20, "report_only": false }, { "id": "boundary_stress_v1", "kind": "boundary_recall", "gate_metric": "exact_recall", "gate_min": null, "case_count": 10, "report_only": true, "partial_credit": true, "license": { "name": "DataSitr synthetic evaluation fixture", "redistribution": "committed in repository" } }, { "id": "clean_open_data_fp_v1", "kind": "clean_fp_density", "gate_metric": "fp_per_10k_tokens", "case_count": 8, "gate_max": null, "report_only": true, "license": { "commercial_reuse": true, "name": "Saudi Open Data License v2.0", "produced_work_notice": "The slice text is DataSitr-authored Arabic paraphrase/summary for internal evaluation. It records source provenance but does not redistribute source documents." } }, { "id": "code_switched_hard_v1", "kind": "entity_recall", "gate_metric": "exact_recall", "gate_min": null, "case_count": 8, "report_only": true, "partial_credit": true, "license": { "name": "DataSitr synthetic evaluation fixture", "redistribution": "committed in repository" } }, { "id": "wojood_news_register_v1", "kind": "news_register", "gate_metric": "exact_recall", "gate_min": null, "case_count": 0, "report_only": true, "partial_credit": true, "optional": true, "skipped": true, "skip_reason": "optional corpus file not present" } ], "gates": { "false_positive_case_precision_min": 0.95, "name_recall_min": 0.85, "hardening_slices": "report_only" } }, "overall_pass": true, "summary": { "total_cases": 376, "total_expected_entities": 223, "total_false_positives": 20, "total_false_negatives": 27, "failed_slices": [] }, "entity_metrics": { "EMAIL_ADDRESS": { "tp": 1, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0 }, "IBAN_CODE": { "tp": 0, "fp": 0, "fn": 1, "precision": null, "recall": 0.0 }, "LOCATION": { "tp": 20, "fp": 2, "fn": 7, "precision": 0.9090909090909091, "recall": 0.7407407407407407 }, "ORGANIZATION": { "tp": 8, "fp": 7, "fn": 10, "precision": 0.5333333333333333, "recall": 0.4444444444444444 }, "PERSON": { "tp": 165, "fp": 7, "fn": 7, "precision": 0.9593023255813954, "recall": 0.9593023255813954 }, "PROFESSION": { "tp": 0, "fp": 1, "fn": 0, "precision": 0.0, "recall": null }, "SA_IBAN": { "tp": 0, "fp": 1, "fn": 0, "precision": 0.0, "recall": null }, "SA_IQAMA": { "tp": 0, "fp": 1, "fn": 1, "precision": 0.0, "recall": 0.0 }, "SA_NATIONAL_ID": { "tp": 0, "fp": 1, "fn": 1, "precision": 0.0, "recall": 0.0 }, "SA_PHONE_NUMBER": { "tp": 2, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0 } }, "slice_metrics": { "bilingual_boilerplate": { "kind": "false_positive", "category": "bilingual_boilerplate", "gate_metric": "precision", "gate_min": 0.95, "gate_max": null, "report_only": false, "partial_credit": false, "case_count": 30, "token_total": 406, "expected_total": 0, "tp_total": 0, "fp_total": 0, "fn_total": 0, "partial_tp_total": 0, "partial_fp_total": 0, "partial_fn_total": 0, "false_positive_case_count": 0, "false_negative_case_count": 0, "precision": 1.0, "precision_definition": "case_level_zero_false_positive_rate", "fp_per_10k_tokens": 0.0, "fp_density_definition": "entity_false_positives_per_10000_clean_tokens", "gate_observed_passed": true, "passed": true }, "boundary_stress_v1": { "kind": "boundary_recall", "category": "boundary_stress", "gate_metric": "exact_recall", "gate_min": null, "gate_max": null, "report_only": true, "partial_credit": true, "case_count": 10, "token_total": 143, "expected_total": 30, "tp_total": 11, "fp_total": 13, "fn_total": 19, "partial_tp_total": 21, "partial_fp_total": 3, "partial_fn_total": 9, "false_positive_case_count": 9, "false_negative_case_count": 9, "recall": 0.36666666666666664, "gate_observed_passed": null, "passed": true, "partial_recall": 0.7, "partial_precision": 0.875, "partial_credit_definition": "same-entity span-overlap credit after exact matches" }, "clean_open_data_fp_v1": { "kind": "clean_fp_density", "category": "clean_open_data_fp", "gate_metric": "fp_per_10k_tokens", "gate_min": null, "gate_max": null, "report_only": true, "partial_credit": false, "case_count": 8, "token_total": 120, "expected_total": 0, "tp_total": 0, "fp_total": 1, "fn_total": 0, "partial_tp_total": 0, "partial_fp_total": 0, "partial_fn_total": 0, "false_positive_case_count": 1, "false_negative_case_count": 0, "precision": 0.875, "precision_definition": "case_level_zero_false_positive_rate", "fp_per_10k_tokens": 83.33333333333333, "fp_density_definition": "entity_false_positives_per_10000_clean_tokens", "gate_observed_passed": null, "passed": true }, "code_switched_hard_v1": { "kind": "entity_recall", "category": "code_switched_hard", "gate_metric": "exact_recall", "gate_min": null, "gate_max": null, "report_only": true, "partial_credit": true, "case_count": 8, "token_total": 106, "expected_total": 23, "tp_total": 15, "fp_total": 4, "fn_total": 8, "partial_tp_total": 18, "partial_fp_total": 1, "partial_fn_total": 5, "false_positive_case_count": 3, "false_negative_case_count": 6, "recall": 0.6521739130434783, "gate_observed_passed": null, "passed": true, "partial_recall": 0.782608695652174, "partial_precision": 0.9473684210526315, "partial_credit_definition": "same-entity span-overlap credit after exact matches" }, "diaspora_variants": { "kind": "name_recall", "category": "diaspora_variants", "gate_metric": "recall", "gate_min": 0.85, "gate_max": null, "report_only": false, "partial_credit": false, "case_count": 20, "token_total": 177, "expected_total": 20, "tp_total": 20, "fp_total": 0, "fn_total": 0, "partial_tp_total": 0, "partial_fp_total": 0, "partial_fn_total": 0, "false_positive_case_count": 0, "false_negative_case_count": 0, "recall": 1.0, "gate_observed_passed": true, "passed": true }, "document_bodies": { "kind": "false_positive", "category": "document_bodies", "gate_metric": "precision", "gate_min": 0.95, "gate_max": null, "report_only": false, "partial_credit": false, "case_count": 40, "token_total": 543, "expected_total": 0, "tp_total": 0, "fp_total": 0, "fn_total": 0, "partial_tp_total": 0, "partial_fp_total": 0, "partial_fn_total": 0, "false_positive_case_count": 0, "false_negative_case_count": 0, "precision": 1.0, "precision_definition": "case_level_zero_false_positive_rate", "fp_per_10k_tokens": 0.0, "fp_density_definition": "entity_false_positives_per_10000_clean_tokens", "gate_observed_passed": true, "passed": true }, "female_names": { "kind": "name_recall", "category": "female_names", "gate_metric": "recall", "gate_min": 0.85, "gate_max": null, "report_only": false, "partial_credit": false, "case_count": 40, "token_total": 316, "expected_total": 42, "tp_total": 42, "fp_total": 0, "fn_total": 0, "partial_tp_total": 0, "partial_fp_total": 0, "partial_fn_total": 0, "false_positive_case_count": 0, "false_negative_case_count": 0, "recall": 1.0, "gate_observed_passed": true, "passed": true }, "honorifics": { "kind": "name_recall", "category": "honorifics", "gate_metric": "recall", "gate_min": 0.85, "gate_max": null, "report_only": false, "partial_credit": false, "case_count": 20, "token_total": 165, "expected_total": 20, "tp_total": 20, "fp_total": 1, "fn_total": 0, "partial_tp_total": 0, "partial_fp_total": 0, "partial_fn_total": 0, "false_positive_case_count": 1, "false_negative_case_count": 0, "recall": 1.0, "gate_observed_passed": true, "passed": true }, "modern_names": { "kind": "name_recall", "category": "modern_names", "gate_metric": "recall", "gate_min": 0.85, "gate_max": null, "report_only": false, "partial_credit": false, "case_count": 30, "token_total": 240, "expected_total": 32, "tp_total": 32, "fp_total": 0, "fn_total": 0, "partial_tp_total": 0, "partial_fp_total": 0, "partial_fn_total": 0, "false_positive_case_count": 0, "false_negative_case_count": 0, "recall": 1.0, "gate_observed_passed": true, "passed": true }, "numeric_ambiguity": { "kind": "false_positive", "category": "numeric_ambiguity", "gate_metric": "precision", "gate_min": 0.95, "gate_max": null, "report_only": false, "partial_credit": false, "case_count": 30, "token_total": 304, "expected_total": 0, "tp_total": 0, "fp_total": 0, "fn_total": 0, "partial_tp_total": 0, "partial_fp_total": 0, "partial_fn_total": 0, "false_positive_case_count": 0, "false_negative_case_count": 0, "precision": 1.0, "precision_definition": "case_level_zero_false_positive_rate", "fp_per_10k_tokens": 0.0, "fp_density_definition": "entity_false_positives_per_10000_clean_tokens", "gate_observed_passed": true, "passed": true }, "operational_text": { "kind": "false_positive", "category": "operational_text", "gate_metric": "precision", "gate_min": 0.95, "gate_max": null, "report_only": false, "partial_credit": false, "case_count": 30, "token_total": 406, "expected_total": 0, "tp_total": 0, "fp_total": 0, "fn_total": 0, "partial_tp_total": 0, "partial_fp_total": 0, "partial_fn_total": 0, "false_positive_case_count": 0, "false_negative_case_count": 0, "precision": 1.0, "precision_definition": "case_level_zero_false_positive_rate", "fp_per_10k_tokens": 0.0, "fp_density_definition": "entity_false_positives_per_10000_clean_tokens", "gate_observed_passed": true, "passed": true }, "policy_code": { "kind": "false_positive", "category": "policy_code", "gate_metric": "precision", "gate_min": 0.95, "gate_max": null, "report_only": false, "partial_credit": false, "case_count": 30, "token_total": 329, "expected_total": 0, "tp_total": 0, "fp_total": 0, "fn_total": 0, "partial_tp_total": 0, "partial_fp_total": 0, "partial_fn_total": 0, "false_positive_case_count": 0, "false_negative_case_count": 0, "precision": 1.0, "precision_definition": "case_level_zero_false_positive_rate", "fp_per_10k_tokens": 0.0, "fp_density_definition": "entity_false_positives_per_10000_clean_tokens", "gate_observed_passed": true, "passed": true }, "structured_noise": { "kind": "false_positive", "category": "structured_noise", "gate_metric": "precision", "gate_min": 0.95, "gate_max": null, "report_only": false, "partial_credit": false, "case_count": 30, "token_total": 328, "expected_total": 0, "tp_total": 0, "fp_total": 0, "fn_total": 0, "partial_tp_total": 0, "partial_fp_total": 0, "partial_fn_total": 0, "false_positive_case_count": 0, "false_negative_case_count": 0, "precision": 1.0, "precision_definition": "case_level_zero_false_positive_rate", "fp_per_10k_tokens": 0.0, "fp_density_definition": "entity_false_positives_per_10000_clean_tokens", "gate_observed_passed": true, "passed": true }, "tribal_names": { "kind": "name_recall", "category": "tribal_names", "gate_metric": "recall", "gate_min": 0.85, "gate_max": null, "report_only": false, "partial_credit": false, "case_count": 50, "token_total": 397, "expected_total": 56, "tp_total": 56, "fp_total": 1, "fn_total": 0, "partial_tp_total": 0, "partial_fp_total": 0, "partial_fn_total": 0, "false_positive_case_count": 1, "false_negative_case_count": 0, "recall": 1.0, "gate_observed_passed": true, "passed": true } }, "diagnostics": { "false_positives": [ { "case_id": "tribal-ar-011", "slice_id": "tribal_names", "entity_type": "PROFESSION", "start": 12, "end": 16, "text": "جهّز" }, { "case_id": "honorific-019", "slice_id": "honorifics", "entity_type": "PERSON", "start": 40, "end": 46, "text": "الليلي" }, { "case_id": "boundary-001", "slice_id": "boundary_stress_v1", "entity_type": "ORGANIZATION", "start": 23, "end": 66, "text": "شركة أبناء محمد العبدالله القابضة للاستثمار" }, { "case_id": "boundary-002", "slice_id": "boundary_stress_v1", "entity_type": "ORGANIZATION", "start": 47, "end": 72, "text": "مستشفى الملك فيصل التخصصي" }, { "case_id": "boundary-002", "slice_id": "boundary_stress_v1", "entity_type": "PERSON", "start": 21, "end": 25, "text": "نورة" }, { "case_id": "boundary-003", "slice_id": "boundary_stress_v1", "entity_type": "LOCATION", "start": 41, "end": 44, "text": "مكة" }, { "case_id": "boundary-005", "slice_id": "boundary_stress_v1", "entity_type": "ORGANIZATION", "start": 8, "end": 47, "text": "جامعة الملك عبدالله للعلوم والتقنية وفد" }, { "case_id": "boundary-006", "slice_id": "boundary_stress_v1", "entity_type": "PERSON", "start": 15, "end": 22, "text": "الجوهرة" }, { "case_id": "boundary-006", "slice_id": "boundary_stress_v1", "entity_type": "PERSON", "start": 27, "end": 38, "text": "فهد السديري" }, { "case_id": "boundary-007", "slice_id": "boundary_stress_v1", "entity_type": "LOCATION", "start": 62, "end": 80, "text": "منطقة عسير في أبها" }, { "case_id": "boundary-007", "slice_id": "boundary_stress_v1", "entity_type": "ORGANIZATION", "start": 6, "end": 38, "text": "شركة الاتصالات السعودية اس تي سي" }, { "case_id": "boundary-008", "slice_id": "boundary_stress_v1", "entity_type": "PERSON", "start": 12, "end": 16, "text": "فيصل" }, { "case_id": "boundary-008", "slice_id": "boundary_stress_v1", "entity_type": "PERSON", "start": 20, "end": 32, "text": "عمر البابطين" }, { "case_id": "boundary-009", "slice_id": "boundary_stress_v1", "entity_type": "ORGANIZATION", "start": 5, "end": 44, "text": "مؤسسة مسك الخيرية ومركز الملك عبدالعزيز" }, { "case_id": "boundary-010", "slice_id": "boundary_stress_v1", "entity_type": "ORGANIZATION", "start": 30, "end": 59, "text": "شركة القصيم للخدمات اللوجستية" }, { "case_id": "clean-003", "slice_id": "clean_open_data_fp_v1", "entity_type": "ORGANIZATION", "start": 77, "end": 95, "text": "نشاط مخالف للأنظمة" }, { "case_id": "codeswitch-003", "slice_id": "code_switched_hard_v1", "entity_type": "PERSON", "start": 11, "end": 23, "text": "أحمد‌بن‌سعيد" }, { "case_id": "codeswitch-003", "slice_id": "code_switched_hard_v1", "entity_type": "SA_IQAMA", "start": 56, "end": 72, "text": "إقامة ٢١٢٣٤٥٦٧٨٩" }, { "case_id": "codeswitch-005", "slice_id": "code_switched_hard_v1", "entity_type": "SA_IBAN", "start": 28, "end": 57, "text": "IBAN SA0380000000608010167519" }, { "case_id": "codeswitch-006", "slice_id": "code_switched_hard_v1", "entity_type": "SA_NATIONAL_ID", "start": 14, "end": 35, "text": "رقم الهوية ١٠٨٧٦٥٤٣٢١" } ], "false_negatives": [ { "case_id": "boundary-001", "slice_id": "boundary_stress_v1", "entity_type": "ORGANIZATION", "start": 23, "end": 74, "text": "شركة أبناء محمد العبدالله القابضة للاستثمار الصناعي" }, { "case_id": "boundary-002", "slice_id": "boundary_stress_v1", "entity_type": "LOCATION", "start": 88, "end": 91, "text": "جدة" }, { "case_id": "boundary-002", "slice_id": "boundary_stress_v1", "entity_type": "ORGANIZATION", "start": 47, "end": 86, "text": "مستشفى الملك فيصل التخصصي ومركز الأبحاث" }, { "case_id": "boundary-002", "slice_id": "boundary_stress_v1", "entity_type": "PERSON", "start": 21, "end": 42, "text": "نورة بنت خالد آل سعود" }, { "case_id": "boundary-003", "slice_id": "boundary_stress_v1", "entity_type": "ORGANIZATION", "start": 31, "end": 52, "text": "شركة وادي مكة للتقنية" }, { "case_id": "boundary-005", "slice_id": "boundary_stress_v1", "entity_type": "LOCATION", "start": 77, "end": 80, "text": "ثول" }, { "case_id": "boundary-005", "slice_id": "boundary_stress_v1", "entity_type": "ORGANIZATION", "start": 8, "end": 43, "text": "جامعة الملك عبدالله للعلوم والتقنية" }, { "case_id": "boundary-006", "slice_id": "boundary_stress_v1", "entity_type": "LOCATION", "start": 89, "end": 94, "text": "الخرج" }, { "case_id": "boundary-006", "slice_id": "boundary_stress_v1", "entity_type": "PERSON", "start": 15, "end": 38, "text": "الجوهرة بنت فهد السديري" }, { "case_id": "boundary-007", "slice_id": "boundary_stress_v1", "entity_type": "LOCATION", "start": 76, "end": 80, "text": "أبها" }, { "case_id": "boundary-007", "slice_id": "boundary_stress_v1", "entity_type": "ORGANIZATION", "start": 6, "end": 43, "text": "شركة الاتصالات السعودية اس تي سي حلول" }, { "case_id": "boundary-007", "slice_id": "boundary_stress_v1", "entity_type": "ORGANIZATION", "start": 56, "end": 72, "text": "أمانة منطقة عسير" }, { "case_id": "boundary-008", "slice_id": "boundary_stress_v1", "entity_type": "LOCATION", "start": 83, "end": 92, "text": "حي النرجس" }, { "case_id": "boundary-008", "slice_id": "boundary_stress_v1", "entity_type": "LOCATION", "start": 94, "end": 100, "text": "الرياض" }, { "case_id": "boundary-008", "slice_id": "boundary_stress_v1", "entity_type": "PERSON", "start": 12, "end": 32, "text": "فيصل بن عمر البابطين" }, { "case_id": "boundary-009", "slice_id": "boundary_stress_v1", "entity_type": "ORGANIZATION", "start": 5, "end": 22, "text": "مؤسسة مسك الخيرية" }, { "case_id": "boundary-009", "slice_id": "boundary_stress_v1", "entity_type": "ORGANIZATION", "start": 24, "end": 60, "text": "مركز الملك عبدالعزيز الثقافي العالمي" }, { "case_id": "boundary-010", "slice_id": "boundary_stress_v1", "entity_type": "ORGANIZATION", "start": 7, "end": 25, "text": "بلدية محافظة عنيزة" }, { "case_id": "boundary-010", "slice_id": "boundary_stress_v1", "entity_type": "ORGANIZATION", "start": 30, "end": 68, "text": "شركة القصيم للخدمات اللوجستية المحدودة" }, { "case_id": "codeswitch-002", "slice_id": "code_switched_hard_v1", "entity_type": "PERSON", "start": 20, "end": 34, "text": "Noura‍Al‍Harbi" }, { "case_id": "codeswitch-003", "slice_id": "code_switched_hard_v1", "entity_type": "PERSON", "start": 11, "end": 33, "text": "أحمد‌بن‌سعيد Al-Otaibi" }, { "case_id": "codeswitch-003", "slice_id": "code_switched_hard_v1", "entity_type": "SA_IQAMA", "start": 62, "end": 72, "text": "٢١٢٣٤٥٦٧٨٩" }, { "case_id": "codeswitch-005", "slice_id": "code_switched_hard_v1", "entity_type": "IBAN_CODE", "start": 33, "end": 57, "text": "SA0380000000608010167519" }, { "case_id": "codeswitch-006", "slice_id": "code_switched_hard_v1", "entity_type": "PERSON", "start": 47, "end": 62, "text": "Maha​Al​Mutairi" }, { "case_id": "codeswitch-006", "slice_id": "code_switched_hard_v1", "entity_type": "SA_NATIONAL_ID", "start": 25, "end": 35, "text": "١٠٨٧٦٥٤٣٢١" }, { "case_id": "codeswitch-007", "slice_id": "code_switched_hard_v1", "entity_type": "LOCATION", "start": 60, "end": 66, "text": "الرياض" }, { "case_id": "codeswitch-008", "slice_id": "code_switched_hard_v1", "entity_type": "PERSON", "start": 23, "end": 38, "text": "Layla‌Al‌Shehri" } ] } }