From f0d55dce72de01c7973d976bc4002055db6d9010 Mon Sep 17 00:00:00 2001 From: hywznn Date: Mon, 27 Jul 2026 14:36:19 +0900 Subject: [PATCH] =?UTF-8?q?data(intent):=20=EB=AA=A8=EB=8D=B8=EB=A7=81=20?= =?UTF-8?q?=EB=B6=84=ED=95=A0=EA=B3=BC=20=ED=8F=89=EA=B0=80=20=EA=B3=84?= =?UTF-8?q?=EC=95=BD=20=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- fowoco-knowledge/CHANGELOG.md | 6 + fowoco-knowledge/README.md | 11 +- fowoco-knowledge/data/intent/manifest.yaml | 5 + .../splits/provisional-v1/manifest.yaml | 94 ++ .../splits/provisional-v1/train_ids.txt | 1072 +++++++++++++++++ .../splits/provisional-v1/validation_ids.txt | 268 +++++ .../docs/INTENT_GOLD_TEST_PLAN.md | 58 + .../docs/INTENT_MODELING_HANDOFF.md | 105 ++ fowoco-knowledge/docs/MODEL_PLAN.md | 122 +- .../knowledge/intent_evaluation_policy.yaml | 55 + .../knowledge/intent_model_contract.yaml | 47 + fowoco-knowledge/knowledge/manifest.yaml | 8 + .../schemas/intent-model-output.schema.json | 82 ++ .../schemas/intent-split-manifest.schema.json | 302 +++++ fowoco-knowledge/src/fowoco_knowledge/cli.py | 31 + .../src/fowoco_knowledge/intent_model.py | 84 ++ .../src/fowoco_knowledge/intent_split.py | 334 +++++ .../src/fowoco_knowledge/validation.py | 150 +++ .../tests/test_intent_modeling.py | 114 ++ 19 files changed, 2894 insertions(+), 54 deletions(-) create mode 100644 fowoco-knowledge/data/intent/splits/provisional-v1/manifest.yaml create mode 100644 fowoco-knowledge/data/intent/splits/provisional-v1/train_ids.txt create mode 100644 fowoco-knowledge/data/intent/splits/provisional-v1/validation_ids.txt create mode 100644 fowoco-knowledge/docs/INTENT_GOLD_TEST_PLAN.md create mode 100644 fowoco-knowledge/docs/INTENT_MODELING_HANDOFF.md create mode 100644 fowoco-knowledge/knowledge/intent_evaluation_policy.yaml create mode 100644 fowoco-knowledge/knowledge/intent_model_contract.yaml create mode 100644 fowoco-knowledge/schemas/intent-model-output.schema.json create mode 100644 fowoco-knowledge/schemas/intent-split-manifest.schema.json create mode 100644 fowoco-knowledge/src/fowoco_knowledge/intent_model.py create mode 100644 fowoco-knowledge/src/fowoco_knowledge/intent_split.py create mode 100644 fowoco-knowledge/tests/test_intent_modeling.py diff --git a/fowoco-knowledge/CHANGELOG.md b/fowoco-knowledge/CHANGELOG.md index 833005c..f22ba8d 100644 --- a/fowoco-knowledge/CHANGELOG.md +++ b/fowoco-knowledge/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## Unreleased + +- MVP Intent 모델 책임을 `Intent + evidence`로 제한한 Knowledge→AI 계약 추가 +- A/B consensus 변경에 재생성 가능한 provisional Train/Validation ID 분할 추가 +- 모델 출력 Schema, 구조적 Gate, 오류 유형과 Gold Test 240건 구성 계획 추가 + ## 0.2.0 - 2026-07-16 - 필요서류 187건과 EPS 세부업종 847건의 원본 해시·버전을 고정 diff --git a/fowoco-knowledge/README.md b/fowoco-knowledge/README.md index a41426c..581305a 100644 --- a/fowoco-knowledge/README.md +++ b/fowoco-knowledge/README.md @@ -59,6 +59,9 @@ python3.11 -m venv .venv # 공식 원본 검증 후 제조업 Knowledge 스냅샷 재생성 .venv/bin/python -m fowoco_knowledge sync-official-data +# 현재 Intent 원본에서 provisional Train/Validation ID 재생성 +.venv/bin/python -m fowoco_knowledge build-intent-splits + # 신청서별 필요서류와 제조업 세부업종 조회 .venv/bin/python -m fowoco_knowledge \ list-required-documents "외국인 고용변동 등 신고" @@ -83,14 +86,20 @@ python3.11 -m venv .venv ## 데이터 상태 - `gold_seed.csv`: 프롬프트·분기 개발용 초기 Seed이며 모델 학습 완료 데이터가 아님 -- `golden_cases.jsonl`: 코드와 모델 평가에만 사용하는 독립 사례 +- `golden_cases.jsonl`: 구조·분기 확인용 Smoke Evaluation이며 최종 Gold Test가 아님 - `hr_intent_dataset.jsonl`: Intent Train/Validation 후보 1,340건이며 재검수 전 Gold Test가 아님 +- `data/intent/splits/provisional-v1/`: A/B 합의 전 재현성 검증용 80:20 ID 분할 - 공개데이터: 절차·용어·분포 보조자료이며 FOWOCO Intent의 정답 라벨로 간주하지 않음 - 실제 운영 로그: 개인정보를 제거하고 별도 승인된 경우에만 Active Learning 후보로 사용 일반 데이터 기준은 [`docs/DATA_GUIDE.md`](docs/DATA_GUIDE.md), Intent 라벨과 evidence 기준은 [`docs/INTENT_DATA.md`](docs/INTENT_DATA.md)를 확인합니다. +Intent 모델 출력·분할·평가 인계 기준은 +[`docs/INTENT_MODELING_HANDOFF.md`](docs/INTENT_MODELING_HANDOFF.md), 독립 Gold Test +구성 기준은 [`docs/INTENT_GOLD_TEST_PLAN.md`](docs/INTENT_GOLD_TEST_PLAN.md)를 +확인합니다. + 공식 데이터 변환은 [`docs/OFFICIAL_DATA_PIPELINE.md`](docs/OFFICIAL_DATA_PIPELINE.md), 신고·연장 기능의 범위는 [`docs/E9_REPORTING_WORKFLOWS.md`](docs/E9_REPORTING_WORKFLOWS.md)를 확인합니다. diff --git a/fowoco-knowledge/data/intent/manifest.yaml b/fowoco-knowledge/data/intent/manifest.yaml index d257ee5..1fa0ace 100644 --- a/fowoco-knowledge/data/intent/manifest.yaml +++ b/fowoco-knowledge/data/intent/manifest.yaml @@ -4,6 +4,10 @@ status: recheck_required path: data/intent/hr_intent_dataset.jsonl schema: schemas/intent-training-case.schema.json label_guide: docs/INTENT_DATA.md +model_contract: knowledge/intent_model_contract.yaml +model_output_schema: schemas/intent-model-output.schema.json +evaluation_policy: knowledge/intent_evaluation_policy.yaml +split_manifest: data/intent/splits/provisional-v1/manifest.yaml record_count: 1340 sha256: 4f4ebfdd4170a78def33e31edbed8315921c0b67934f5ff8612595dcd479bed2 contains_real_personal_data: false @@ -14,3 +18,4 @@ limitations: - 독립 Gold Test가 아님 - 최종 모델 성능 주장에 사용할 수 없음 - 의미 라벨은 규칙 v1.1에 따른 A/B 합의 검수가 필요함 + - provisional split은 source 또는 consensus 변경 시 재생성해야 함 diff --git a/fowoco-knowledge/data/intent/splits/provisional-v1/manifest.yaml b/fowoco-knowledge/data/intent/splits/provisional-v1/manifest.yaml new file mode 100644 index 0000000..463a3f2 --- /dev/null +++ b/fowoco-knowledge/data/intent/splits/provisional-v1/manifest.yaml @@ -0,0 +1,94 @@ +split_id: FOWOCO-HR-INTENT-PROVISIONAL-SPLIT +version: 0.1.0 +status: provisional_pending_consensus +schema: schemas/intent-split-manifest.schema.json +source: + dataset_id: FOWOCO-HR-INTENT-CANDIDATES + version: 1.1.0 + status: recheck_required + path: data/intent/hr_intent_dataset.jsonl + record_count: 1340 + sha256: 4f4ebfdd4170a78def33e31edbed8315921c0b67934f5ff8612595dcd479bed2 +consensus: + status: pending + assumed_for_provisional_split: true + dependency_issue: 30 + dependency_pr: 31 + regenerate_when_source_changes: true +policy: + method: grouped_multilabel_greedy + seed: 20260727 + train_ratio: 0.8 + validation_ratio: 0.2 + target_train_count: 1072 + target_validation_count: 268 + template_normalization: + worker_id_pattern: (?i)\bwrk[-_ ]?\d+\b + worker_id_replacement: + collapse_whitespace: true + casefold: true + preserve_dates_amounts_and_task_terms: true + stratification_features: + - intent_code + - intent_cardinality + - ordered_intent_combination +outputs: + train: + path: data/intent/splits/provisional-v1/train_ids.txt + record_count: 1072 + sha256: 264011422e233e28ab7ab4b93579f634a6e53c00730d254b7ca2749d71575cb6 + validation: + path: data/intent/splits/provisional-v1/validation_ids.txt + record_count: 268 + sha256: 96da74e15519f9610ef316bc3e7499d06832c36564180607a3565b851bc8ba85 +statistics: + template_group_count: 1335 + duplicate_template_group_count: 5 + max_template_group_size: 2 + label_counts: + source: + DOCUMENT_REQUEST: 345 + EMPLOYMENT_CHANGE: 273 + EXPIRY_RENEWAL: 234 + OUT_OF_SCOPE: 128 + PAYROLL_EXPLANATION: 275 + WORKER_ONBOARDING: 193 + WORK_INSTRUCTION: 339 + train: + DOCUMENT_REQUEST: 275 + EMPLOYMENT_CHANGE: 217 + EXPIRY_RENEWAL: 186 + OUT_OF_SCOPE: 102 + PAYROLL_EXPLANATION: 219 + WORKER_ONBOARDING: 154 + WORK_INSTRUCTION: 269 + validation: + DOCUMENT_REQUEST: 70 + EMPLOYMENT_CHANGE: 56 + EXPIRY_RENEWAL: 48 + OUT_OF_SCOPE: 26 + PAYROLL_EXPLANATION: 56 + WORKER_ONBOARDING: 39 + WORK_INSTRUCTION: 70 + intent_cardinality_counts: + source: + '1': 970 + '2': 297 + '3': 69 + '4': 4 + train: + '1': 778 + '2': 240 + '3': 52 + '4': 2 + validation: + '1': 192 + '2': 57 + '3': 17 + '4': 2 +limitations: +- Reviewer B와 A/B consensus 완료 전에는 최종 학습 분할이 아님 +- Reviewer A의 proposed_intents_json은 현재 원본에 적용하지 않은 상태임 +- 독립 Gold Test를 포함하지 않음 +- source SHA-256이 바뀌면 생성기를 다시 실행해야 함 +- 분할 비율과 내부 목표치는 baseline 비교용이며 성능 보장을 뜻하지 않음 diff --git a/fowoco-knowledge/data/intent/splits/provisional-v1/train_ids.txt b/fowoco-knowledge/data/intent/splits/provisional-v1/train_ids.txt new file mode 100644 index 0000000..c3897e5 --- /dev/null +++ b/fowoco-knowledge/data/intent/splits/provisional-v1/train_ids.txt @@ -0,0 +1,1072 @@ +1 +2 +3 +4 +8 +10 +11 +12 +14 +15 +16 +17 +18 +19 +21 +22 +23 +24 +26 +27 +28 +29 +31 +32 +34 +35 +38 +39 +40 +41 +42 +43 +44 +45 +46 +47 +48 +49 +50 +51 +52 +53 +54 +55 +56 +58 +60 +61 +62 +63 +64 +65 +66 +67 +70 +72 +73 +74 +75 +76 +78 +79 +80 +82 +83 +85 +87 +88 +89 +90 +91 +93 +94 +95 +97 +98 +99 +100 +102 +103 +104 +105 +106 +107 +108 +109 +110 +112 +113 +114 +117 +118 +120 +121 +122 +125 +126 +127 +128 +129 +130 +131 +132 +133 +134 +135 +136 +137 +138 +139 +142 +143 +144 +145 +146 +147 +148 +149 +150 +151 +152 +153 +154 +155 +156 +157 +158 +159 +161 +162 +163 +164 +165 +166 +168 +170 +171 +172 +173 +174 +175 +176 +177 +178 +179 +180 +181 +182 +183 +184 +185 +186 +187 +188 +189 +190 +191 +192 +193 +194 +195 +197 +198 +200 +201 +202 +203 +204 +205 +206 +207 +208 +209 +210 +211 +212 +213 +214 +215 +216 +218 +219 +220 +221 +222 +223 +224 +225 +226 +227 +228 +229 +230 +231 +232 +233 +235 +237 +238 +240 +241 +243 +244 +247 +248 +249 +250 +251 +252 +253 +254 +255 +256 +257 +258 +259 +260 +261 +263 +264 +266 +268 +269 +270 +271 +273 +274 +276 +277 +278 +279 +280 +282 +283 +285 +287 +289 +290 +291 +292 +293 +294 +295 +296 +297 +298 +299 +300 +301 +302 +305 +306 +308 +309 +310 +311 +312 +314 +315 +316 +317 +320 +321 +324 +325 +326 +327 +328 +329 +331 +332 +333 +334 +335 +336 +337 +338 +339 +340 +341 +342 +343 +344 +345 +346 +347 +348 +349 +351 +352 +353 +354 +356 +357 +358 +359 +360 +361 +363 +364 +366 +367 +368 +370 +371 +372 +373 +374 +375 +376 +377 +378 +379 +381 +383 +384 +386 +387 +388 +389 +390 +391 +392 +393 +395 +396 +397 +398 +399 +401 +405 +406 +407 +408 +409 +411 +412 +413 +414 +415 +416 +417 +418 +419 +420 +421 +422 +423 +424 +425 +427 +429 +430 +431 +432 +433 +435 +436 +438 +439 +442 +443 +444 +445 +446 +447 +448 +450 +451 +452 +453 +456 +457 +458 +460 +461 +462 +463 +464 +465 +466 +467 +468 +469 +471 +472 +473 +474 +475 +480 +481 +483 +484 +485 +486 +487 +488 +489 +490 +491 +493 +494 +495 +496 +497 +498 +499 +500 +501 +502 +503 +504 +505 +506 +507 +508 +509 +510 +511 +512 +513 +514 +515 +516 +518 +519 +520 +524 +525 +526 +527 +528 +530 +531 +532 +533 +535 +536 +537 +538 +540 +541 +542 +543 +544 +545 +546 +547 +548 +549 +551 +552 +553 +555 +556 +557 +558 +559 +560 +561 +562 +564 +565 +566 +567 +570 +571 +572 +573 +574 +576 +577 +578 +579 +580 +581 +582 +583 +584 +585 +586 +587 +588 +589 +590 +591 +593 +594 +595 +596 +597 +598 +599 +600 +601 +602 +603 +607 +608 +609 +610 +611 +612 +613 +614 +615 +616 +617 +618 +619 +621 +622 +624 +625 +626 +627 +628 +629 +630 +631 +633 +634 +635 +637 +640 +643 +644 +645 +646 +649 +651 +652 +654 +655 +656 +657 +658 +659 +660 +661 +662 +663 +665 +666 +667 +668 +669 +671 +672 +673 +674 +677 +678 +679 +680 +681 +683 +684 +685 +686 +688 +689 +690 +691 +693 +694 +695 +696 +697 +698 +699 +701 +703 +704 +705 +706 +707 +708 +709 +710 +711 +712 +713 +715 +716 +717 +718 +719 +722 +723 +725 +729 +730 +731 +732 +733 +734 +735 +736 +737 +738 +740 +742 +743 +744 +745 +746 +747 +748 +749 +751 +752 +753 +755 +756 +758 +759 +760 +761 +762 +763 +764 +765 +766 +767 +768 +770 +771 +772 +774 +775 +776 +778 +780 +781 +783 +784 +785 +786 +788 +789 +790 +791 +794 +796 +797 +798 +800 +802 +803 +805 +806 +807 +808 +809 +811 +813 +815 +816 +819 +820 +821 +822 +823 +824 +825 +828 +829 +830 +831 +832 +833 +834 +835 +836 +837 +838 +839 +840 +842 +843 +844 +846 +848 +849 +850 +851 +853 +854 +855 +856 +857 +858 +859 +860 +861 +862 +863 +865 +866 +867 +868 +869 +870 +871 +872 +873 +874 +875 +876 +877 +878 +881 +882 +883 +884 +885 +887 +888 +890 +891 +895 +896 +897 +898 +900 +901 +903 +904 +905 +906 +907 +909 +910 +911 +912 +913 +914 +916 +917 +918 +919 +921 +922 +923 +924 +925 +926 +927 +928 +929 +930 +931 +933 +935 +936 +937 +938 +939 +940 +941 +942 +943 +944 +945 +946 +947 +949 +950 +951 +952 +953 +954 +955 +956 +957 +958 +959 +960 +961 +962 +963 +964 +965 +966 +968 +969 +971 +972 +973 +975 +976 +977 +978 +979 +981 +982 +983 +984 +985 +986 +987 +989 +990 +991 +992 +993 +995 +996 +997 +998 +999 +1000 +1001 +1003 +1004 +1006 +1007 +1008 +1009 +1012 +1014 +1015 +1016 +1017 +1018 +1020 +1021 +1022 +1025 +1026 +1027 +1029 +1030 +1031 +1032 +1033 +1034 +1035 +1036 +1038 +1039 +1040 +1041 +1043 +1044 +1045 +1047 +1048 +1050 +1052 +1053 +1054 +1055 +1056 +1057 +1058 +1061 +1063 +1064 +1066 +1067 +1068 +1069 +1070 +1071 +1072 +1073 +1074 +1076 +1077 +1078 +1081 +1082 +1083 +1084 +1085 +1086 +1087 +1088 +1089 +1090 +1092 +1093 +1095 +1096 +1097 +1098 +1100 +1101 +1103 +1105 +1106 +1107 +1108 +1109 +1110 +1111 +1112 +1113 +1114 +1115 +1117 +1119 +1120 +1121 +1126 +1127 +1128 +1129 +1131 +1132 +1133 +1134 +1136 +1137 +1138 +1141 +1143 +1144 +1145 +1146 +1147 +1148 +1149 +1150 +1151 +1152 +1153 +1156 +1158 +1159 +1161 +1164 +1165 +1166 +1167 +1168 +1171 +1172 +1173 +1174 +1176 +1177 +1178 +1179 +1180 +1181 +1182 +1183 +1184 +1185 +1186 +1188 +1189 +1190 +1191 +1192 +1194 +1195 +1196 +1198 +1200 +1201 +1203 +1204 +1205 +1206 +1209 +1210 +1212 +1213 +1214 +1215 +1216 +1217 +1218 +1219 +1220 +1222 +1223 +1224 +1225 +1226 +1227 +1228 +1230 +1231 +1232 +1233 +1234 +1235 +1236 +1237 +1238 +1239 +1242 +1243 +1246 +1247 +1248 +1249 +1250 +1251 +1252 +1253 +1255 +1256 +1258 +1260 +1261 +1263 +1267 +1268 +1269 +1270 +1271 +1272 +1273 +1274 +1276 +1278 +1279 +1280 +1281 +1283 +1284 +1285 +1286 +1287 +1288 +1290 +1291 +1292 +1293 +1294 +1295 +1296 +1297 +1298 +1300 +1301 +1302 +1303 +1306 +1307 +1308 +1309 +1310 +1312 +1314 +1316 +1317 +1319 +1321 +1322 +1323 +1324 +1325 +1326 +1328 +1329 +1330 +1331 +1333 +1334 +1335 +1336 +1337 +1338 +1339 +1340 diff --git a/fowoco-knowledge/data/intent/splits/provisional-v1/validation_ids.txt b/fowoco-knowledge/data/intent/splits/provisional-v1/validation_ids.txt new file mode 100644 index 0000000..06fcea2 --- /dev/null +++ b/fowoco-knowledge/data/intent/splits/provisional-v1/validation_ids.txt @@ -0,0 +1,268 @@ +5 +6 +7 +9 +13 +20 +25 +30 +33 +36 +37 +57 +59 +68 +69 +71 +77 +81 +84 +86 +92 +96 +101 +111 +115 +116 +119 +123 +124 +140 +141 +160 +167 +169 +196 +199 +217 +234 +236 +239 +242 +245 +246 +262 +265 +267 +272 +275 +281 +284 +286 +288 +303 +304 +307 +313 +318 +319 +322 +323 +330 +350 +355 +362 +365 +369 +380 +382 +385 +394 +400 +402 +403 +404 +410 +426 +428 +434 +437 +440 +441 +449 +454 +455 +459 +470 +476 +477 +478 +479 +482 +492 +517 +521 +522 +523 +529 +534 +539 +550 +554 +563 +568 +569 +575 +592 +604 +605 +606 +620 +623 +632 +636 +638 +639 +641 +642 +647 +648 +650 +653 +664 +670 +675 +676 +682 +687 +692 +700 +702 +714 +720 +721 +724 +726 +727 +728 +739 +741 +750 +754 +757 +769 +773 +777 +779 +782 +787 +792 +793 +795 +799 +801 +804 +810 +812 +814 +817 +818 +826 +827 +841 +845 +847 +852 +864 +879 +880 +886 +889 +892 +893 +894 +899 +902 +908 +915 +920 +932 +934 +948 +967 +970 +974 +980 +988 +994 +1002 +1005 +1010 +1011 +1013 +1019 +1023 +1024 +1028 +1037 +1042 +1046 +1049 +1051 +1059 +1060 +1062 +1065 +1075 +1079 +1080 +1091 +1094 +1099 +1102 +1104 +1116 +1118 +1122 +1123 +1124 +1125 +1130 +1135 +1139 +1140 +1142 +1154 +1155 +1157 +1160 +1162 +1163 +1169 +1170 +1175 +1187 +1193 +1197 +1199 +1202 +1207 +1208 +1211 +1221 +1229 +1240 +1241 +1244 +1245 +1254 +1257 +1259 +1262 +1264 +1265 +1266 +1275 +1277 +1282 +1289 +1299 +1304 +1305 +1311 +1313 +1315 +1318 +1320 +1327 +1332 diff --git a/fowoco-knowledge/docs/INTENT_GOLD_TEST_PLAN.md b/fowoco-knowledge/docs/INTENT_GOLD_TEST_PLAN.md new file mode 100644 index 0000000..9fa6388 --- /dev/null +++ b/fowoco-knowledge/docs/INTENT_GOLD_TEST_PLAN.md @@ -0,0 +1,58 @@ +# Intent Gold Test 240건 구성 계획 + +## 1. 목적 + +Gold Test는 모델·프롬프트 선택 결과를 마지막에 비교하는 잠긴 독립 평가셋이다. +현재 1,340건 Train/Validation 후보와 기존 Smoke Evaluation을 Gold 성능 주장에 +사용하지 않는다. + +## 2. 목표 구성 + +총 240건을 다음 배타적 작성 버킷으로 구성한다. + +| 버킷 | 건수 | 목적 | +| --- | ---: | --- | +| 단일 Intent 표준 사례 | 140 | 7개 Intent별 20건 | +| Multi-Intent 사례 | 60 | 경계쌍, 3개 Intent, 원문 순서 | +| 경계·교란 사례 | 40 | 완료 상태, 일반 휴가, 급여계좌, 외부 실행, 긴 evidence | + +`OUT_OF_SCOPE`는 단일 Intent 표준 사례에 포함하되, HR과 유사한 표현의 범위 밖 +요청을 충분히 포함한다. + +## 3. 작성 규칙 + +- 기존 1,340건을 단순 복사하거나 WRK 번호만 바꾸지 않는다. +- 실제 개인정보·기업정보를 사용하지 않는다. +- 날짜, 금액, 서류명은 모두 더미 값으로 작성한다. +- evidence는 원문의 가장 짧고 완결된 exact substring이다. +- 정답은 Reviewer A/B 독립 검수 후 불일치만 합의한다. +- 작성자와 최종 검수자는 같은 사람이 단독으로 담당하지 않는다. + +## 4. 누수 방지 + +다음 중 하나라도 충족하면 Train/Validation과 중복 후보로 검토한다. + +- WRK 식별자를 제거한 정규화 문장이 동일 +- 핵심 명사·동사와 Intent 조합이 동일한 템플릿 +- 기존 문장의 조사·어미·숫자만 바꾼 변형 + +중복 후보는 자동 제외하지 않고 사람이 의미 중복 여부를 확인한다. + +## 5. 잠금과 버전 + +Gold Test manifest에는 다음을 기록한다. + +- version과 status +- 레코드 수와 SHA-256 +- Intent·Multi-Intent·경계 유형 분포 +- 작성자와 A/B 검수 상태 +- 잠금일 +- 허용된 평가 목적 + +잠금 이후에는 문항을 수정하지 않는다. 오류가 확인되면 기존 version을 보존하고 새 +version을 만든다. + +## 6. 현재 단계의 제한 + +이 문서는 구성 계획이며 Gold 240건을 확보했다는 의미가 아니다. 실제 문항이 작성되고 +A/B 합의와 checksum 잠금이 끝날 때까지 모델의 최종 성능을 주장하지 않는다. diff --git a/fowoco-knowledge/docs/INTENT_MODELING_HANDOFF.md b/fowoco-knowledge/docs/INTENT_MODELING_HANDOFF.md new file mode 100644 index 0000000..51c9afe --- /dev/null +++ b/fowoco-knowledge/docs/INTENT_MODELING_HANDOFF.md @@ -0,0 +1,105 @@ +# Intent 모델링 Knowledge → AI 인계 계약 + +## 1. 인계 범위 + +knowledge 저장소는 다음 자료를 제공한다. + +| 항목 | 경로 | 상태 | +| --- | --- | --- | +| 라벨 규칙 | `docs/INTENT_DATA.md` | v1.1 | +| 모델 계약 | `knowledge/intent_model_contract.yaml` | provisional | +| 모델 출력 Schema | `schemas/intent-model-output.schema.json` | provisional | +| 원본 후보 | `data/intent/hr_intent_dataset.jsonl` | A/B 재검수 필요 | +| 분할 manifest | `data/intent/splits/provisional-v1/manifest.yaml` | consensus 대기 | +| 평가 정책 | `knowledge/intent_evaluation_policy.yaml` | baseline 전 임시 목표 | +| Gold 계획 | `docs/INTENT_GOLD_TEST_PLAN.md` | 작성 전 | + +AI 저장소는 프롬프트, 모델 어댑터, 추론·평가 코드와 실험 결과를 관리한다. + +## 2. 입력과 출력 + +모델 입력: + +```json +{ + "hr_input": "재계약 준비하면서 서명본도 받아서 첨부해줘" +} +``` + +모델 출력: + +```json +{ + "intents": [ + { + "intent": "EXPIRY_RENEWAL", + "evidence": "재계약 준비하면서" + }, + { + "intent": "DOCUMENT_REQUEST", + "evidence": "서명본도 받아서" + } + ] +} +``` + +`request_id`, 입력 원문, 검증 상태, 모델명과 시각은 서버 envelope에서 추가한다. +학습 정답이나 모델 출력에 넣지 않는다. + +## 3. split 소비 방식 + +Train/Validation 파일은 원본 JSONL을 복제하지 않고 ID만 제공한다. AI 저장소는 +manifest의 다음 절차로 데이터를 읽는다. + +1. 원본 JSONL의 SHA-256을 split manifest의 `source.sha256`과 비교한다. +2. `train_ids.txt`와 `validation_ids.txt`를 읽는다. +3. 원본 레코드를 `id`로 선택한다. +4. ID 누락·중복·교집합이 있으면 실험을 중단한다. +5. manifest가 `locked`가 아니면 결과를 provisional로 표시한다. + +현재 분할: + +- Train 1,072건 +- Validation 268건 +- Gold Test 미포함 +- Reviewer A 변경 제안은 원본 JSONL에 아직 적용하지 않음 + +분할 재생성: + +```bash +python -m fowoco_knowledge build-intent-splits +``` + +## 4. baseline 실험 순서 + +1. A.X zero-shot Structured Output +2. 동일 Validation에서 prompt v1 비교 +3. Train에서만 few-shot 예시 선택 +4. 오류 유형별 분석 +5. prompt와 Knowledge 규칙·데이터 변경을 분리해 기록 + +Validation을 보고 prompt를 수정할 수 있지만 Gold Test는 최종 후보 비교 전까지 +열람하지 않는다. + +## 5. 필수 검증 + +모델 출력마다 다음 순서로 검증한다. + +1. JSON 파싱 +2. `intent-model-output.schema.json` +3. 중복 Intent 금지 +4. evidence exact substring +5. evidence 원문 순서 +6. `OUT_OF_SCOPE` 단독성 + +구조 검증 실패는 낮은 confidence로 대체하지 않고 실패 유형으로 기록한다. + +## 6. 변경 대응 + +A/B consensus 결과가 현재 원본과 다르면 다음을 수행한다. + +1. 원본 JSONL과 `data/intent/manifest.yaml` version·SHA-256 갱신 +2. split 생성기 재실행 +3. Train/Validation ID와 분포 diff 검토 +4. AI 실험에서 새 데이터 version으로 다시 실행 +5. 이전 version 결과와 직접 혼합하지 않음 diff --git a/fowoco-knowledge/docs/MODEL_PLAN.md b/fowoco-knowledge/docs/MODEL_PLAN.md index 388455c..8313114 100644 --- a/fowoco-knowledge/docs/MODEL_PLAN.md +++ b/fowoco-knowledge/docs/MODEL_PLAN.md @@ -1,74 +1,90 @@ -# 현실적인 모델링 계획 +# Intent 모델링 계획 -## 1. Intent·Domain 다중분류 +- 현재 단계: A.X 기반 MVP baseline 준비 +- 데이터 규칙: Intent 라벨 규칙 v1.1 +- 데이터 상태: A/B consensus 대기 중인 provisional Train/Validation 후보 +- 모델 코드 위치: `fowoco/ai` -### 입력과 출력 +이 문서는 knowledge 저장소가 제공할 데이터·출력 계약·평가 기준을 정의한다. +모델 호출, 프롬프트, 학습, 실험 추적 코드는 AI 저장소에서 관리한다. -- 입력: HR 자연어 + 입력 모드 + 선택된 화면 컨텍스트 -- 출력: 하나 이상의 Intent, Domain, 후보 Workflow, confidence -- 복합 요청: Multi-label로 평가하고 후보 업무카드로 분리 +## 1. MVP 모델 책임 -### 비교 순서 +입력은 HR 담당자의 원문 `hr_input` 하나이며 출력은 `Intent + evidence`로 제한한다. -1. 규칙/키워드 baseline -2. TF-IDF + Linear SVM 또는 Logistic Regression -3. 한국어 Encoder 모델(KLUE-RoBERTa·KoELECTRA 계열) 미세조정 -4. 로컬 LLM Structured Output zero/few-shot +```json +{ + "intents": [ + { + "intent": "EXPIRY_RENEWAL", + "evidence": "재계약 준비하면서" + }, + { + "intent": "DOCUMENT_REQUEST", + "evidence": "서명본도 받아서" + } + ] +} +``` -### 지표 +정식 출력 계약은 +[`schemas/intent-model-output.schema.json`](../schemas/intent-model-output.schema.json)이다. -- 단일 요청: Macro-F1, 클래스별 Recall -- 복합 요청: Micro-F1, Exact Match Ratio -- 운영 안전성: `OUT_OF_SCOPE` Recall, 낮은 확신도 검토 전환율 -- 모델 선택: 정확도뿐 아니라 지연시간·메모리·호출비용 함께 비교 +모델이 담당하지 않는 항목: -## 2. Slot Filling +- Domain, Workflow, Slot 결정 +- `request_id`, `status`, 모델명, 생성 시각 envelope +- confidence를 이용한 최종 승인 또는 자동 실행 +- 외부기관 신고·접수·제출 +- 법률·체류·급여의 최종 판단 -초기에는 LLM의 JSON Structured Output과 필수 Slot 규칙을 사용합니다. -날짜·서류명·장소·금액 라벨이 충분히 쌓인 뒤 NER/Token Classification과 비교합니다. +## 2. 데이터 단계 -| 단계 | 데이터 | 평가 | -| --- | --- | --- | -| MVP | 팀 Gold 문장 + Workflow Slot 정의 | Field-level Precision·Recall·F1 | -| 보조학습 | AIHub 시간표현, KLUE NER | 날짜·일반 개체 인식 초기화 | -| 도메인 고도화 | HR 수정 로그의 서류명·급여항목 Span | Slot별 F1, 날짜 정규화 정확도 | +1. Intent 규칙 v1.1 고정 +2. Reviewer A/B 독립 검수와 consensus 생성 +3. consensus를 반영한 원본 1,340건의 version·SHA-256 고정 +4. 그룹 기반 Train 80% / Validation 20% 분할 +5. 독립 Gold Test 240건 수제 작성·이중 검수·잠금 +6. Smoke Test와 Gold Test를 분리해 운영 -공개 NER의 사람·장소 라벨만으로 여권 사본·급여항목 같은 FOWOCO Slot을 해결할 수 없습니다. +현재 committed split은 4단계를 미리 검증하기 위한 provisional 산출물이다. source +SHA-256이나 consensus가 바뀌면 생성기를 다시 실행하고 최종 잠금 전 성능 주장에 +사용하지 않는다. -## 3. 근로자 응답 분류 +## 3. 모델 비교 순서 -버튼 응답은 모델이 아니라 코드로 처리합니다. -자유질문이 들어온 경우에만 다음 유형을 분류합니다. +1. A.X Structured Output zero-shot +2. A.X prompt·few-shot baseline +3. 오류 유형별 prompt 및 데이터 개선 +4. 비용·지연시간·배포 제약이 확인된 경우에만 작은 Encoder 모델 비교 -- `QUESTION` -- `NOT_UNDERSTOOD` -- `IN_PROGRESS` -- `SUBMITTED` -- `CANNOT_COMPLETE` -- `SENSITIVE_ISSUE` +BERT 계열 다중분류 모델은 Intent를 예측할 수 있지만 exact evidence를 자동으로 +생성하지 않는다. 동일한 출력 계약을 사용하려면 별도의 span/token classification +구조 또는 evidence 추출 단계가 필요하다. 따라서 BERT 증류는 MVP baseline 이후 +별도 설계한다. -초기에는 LLM 분류를 사용하고, 익명·검수 로그가 약 200~400건 쌓이면 작은 분류모델과 비교합니다. -핵심 지표는 전체 Accuracy보다 `CANNOT_COMPLETE`와 `SENSITIVE_ISSUE` Recall입니다. +## 4. 평가 -## 4. 별도 모델을 만들지 않는 영역 +구조적 Gate는 100%를 요구한다. -| 영역 | 구현 | -| --- | --- | -| D-day와 내부 업무일 | 날짜 규칙 + 공휴일 API | -| 필수서류 | Workflow Catalog + 공식자료 검수 | -| 급여 차이 계산 | Python/SQL 계산식 | -| OCR | 상용 또는 검증된 OCR API 사용 | -| 번역 | 상용 API 또는 다국어 LLM + 핵심값 검증 | -| 법령 근거 | 버전형 검색·출처 표시, 전문가 확인 | +- JSON Schema 유효성 +- evidence exact substring +- Multi-Intent 원문 순서 +- `OUT_OF_SCOPE` 단독성 -모델을 만드는 것보다 정답이 정해진 영역을 정확한 코드로 처리하는 편이 서비스 신뢰도에 유리합니다. +모델 후보 비교에는 Intent Exact Match, Macro/Micro F1, Intent별 Recall과 오류 +유형을 함께 사용한다. 초기 내부 목표는 baseline 비교를 위한 provisional 값이며 +서비스 성능 보장을 뜻하지 않는다. -## 5. Active Learning +정식 기준은 +[`knowledge/intent_evaluation_policy.yaml`](../knowledge/intent_evaluation_policy.yaml)을 +따른다. -1. 모델 confidence가 낮거나 HR이 수정한 사례를 후보 큐에 저장 -2. 개인정보 제거 -3. 두 명이 Intent·Slot을 교차 검수 -4. 합의 사례만 다음 학습 버전에 포함 -5. 이전 독립 평가셋으로 회귀 테스트 +## 5. 운영 경계 -이는 운영 피드백으로 데이터를 효율적으로 고르는 방식이며, 강화학습과 동일하지 않습니다. +confidence는 MVP 모델 출력 계약에 넣지 않는다. 형식 오류, evidence 오류, 알 수 없는 +Intent, `OUT_OF_SCOPE` 혼합은 규칙 검증으로 차단한다. 체류·고용변동과 외부 실행은 +confidence와 무관하게 HR 승인과 Guardrail 검사를 적용한다. + +운영 로그를 학습에 사용할 때는 개인정보를 제거하고 두 명의 검수자가 합의한 사례만 +새 버전에 포함한다. 이전 locked Gold Test로 회귀를 확인한다. diff --git a/fowoco-knowledge/knowledge/intent_evaluation_policy.yaml b/fowoco-knowledge/knowledge/intent_evaluation_policy.yaml new file mode 100644 index 0000000..dc75ab4 --- /dev/null +++ b/fowoco-knowledge/knowledge/intent_evaluation_policy.yaml @@ -0,0 +1,55 @@ +policy_id: FOWOCO-INTENT-EVALUATION +version: 0.1.0 +status: provisional_until_baseline +applies_to: FOWOCO-INTENT-MODEL-CONTRACT +structural_gates: + - id: JSON_SCHEMA_VALID + description: 출력이 intent-model-output JSON Schema를 만족함 + required_rate: 1.0 + - id: EVIDENCE_EXACT_SUBSTRING + description: OUT_OF_SCOPE 외 모든 evidence가 hr_input의 연속 부분 문자열임 + required_rate: 1.0 + - id: INTENT_ORDER + description: Multi-Intent 배열이 evidence의 원문 등장 순서와 같음 + required_rate: 1.0 + - id: OUT_OF_SCOPE_EXCLUSIVE + description: OUT_OF_SCOPE는 단독 Intent이며 evidence는 null임 + required_rate: 1.0 +selection_metrics: + - id: INTENT_EXACT_MATCH + description: 예측 Intent 배열과 정답 Intent 배열이 순서까지 동일한 비율 + provisional_target: 0.8 + - id: MACRO_F1 + description: 7개 Intent별 F1의 산술평균 + provisional_target: 0.85 + - id: MICRO_F1 + description: 전체 Intent label decision 기준 Micro F1 + provisional_target: null +sensitive_recall_targets: + EXPIRY_RENEWAL: 0.9 + EMPLOYMENT_CHANGE: 0.9 +error_taxonomy: + - id: MISSING_INTENT + description: 정답 Intent를 하나 이상 누락 + - id: EXTRA_INTENT + description: 근거 없이 Intent를 추가 + - id: WRONG_BOUNDARY + description: 휴가·급여계좌·완료 상태 등 정의된 경계를 잘못 적용 + - id: EVIDENCE_NOT_SUBSTRING + description: evidence가 원문과 정확히 일치하지 않음 + - id: EVIDENCE_TOO_BROAD + description: 다른 Intent 근거나 외부 실행 문구까지 evidence에 포함 + - id: INTENT_ORDER_ERROR + description: Multi-Intent가 evidence 원문 순서와 다름 + - id: OUT_OF_SCOPE_MIXED + description: OUT_OF_SCOPE를 다른 Intent와 함께 출력 + - id: DUPLICATE_INTENT + description: 동일 Intent를 evidence만 바꿔 중복 출력 + - id: INVALID_JSON + description: JSON 파싱 또는 Schema 검증 실패 +reporting_rules: + - 구조적 Gate 실패율을 Intent 성능 지표와 분리해 보고 + - 전체 Accuracy 하나로 모델을 선택하지 않음 + - Intent별 Recall과 오류 유형별 건수를 함께 보고 + - Smoke Evaluation 결과를 최종 성능 주장에 사용하지 않음 + - locked Gold Test 결과에 사용한 모델·프롬프트·데이터 버전을 함께 기록 diff --git a/fowoco-knowledge/knowledge/intent_model_contract.yaml b/fowoco-knowledge/knowledge/intent_model_contract.yaml new file mode 100644 index 0000000..d089662 --- /dev/null +++ b/fowoco-knowledge/knowledge/intent_model_contract.yaml @@ -0,0 +1,47 @@ +contract_id: FOWOCO-INTENT-MODEL-CONTRACT +version: 0.1.0 +status: provisional_pending_consensus +task: + name: HR 발화 Intent와 evidence 추출 + input_fields: + - hr_input + output_schema: schemas/intent-model-output.schema.json + responsibilities: + - 하나 이상의 지원 Intent 분류 + - Intent별 evidence exact substring 추출 + - evidence 원문 등장 순서대로 Intent 배열 정렬 + excluded_responsibilities: + - Domain 또는 Workflow 선택 + - Slot Filling + - request_id, status, meta envelope 생성 + - confidence를 이용한 최종 승인 판단 + - 외부기관 신고·접수·제출 실행 + - 법률·체류·급여의 최종 판단 +allowed_intents: + - WORK_INSTRUCTION + - DOCUMENT_REQUEST + - PAYROLL_EXPLANATION + - WORKER_ONBOARDING + - EMPLOYMENT_CHANGE + - EXPIRY_RENEWAL + - OUT_OF_SCOPE +data: + source_manifest: data/intent/manifest.yaml + split_manifest: data/intent/splits/provisional-v1/manifest.yaml + evaluation_policy: knowledge/intent_evaluation_policy.yaml + label_guide: docs/INTENT_DATA.md + handoff_guide: docs/INTENT_MODELING_HANDOFF.md + gold_test_plan: docs/INTENT_GOLD_TEST_PLAN.md +consumer_rules: + - split manifest의 source SHA-256이 현재 원본과 같을 때만 사용 + - provisional split으로 최종 성능을 주장하지 않음 + - Gold Test는 모델·프롬프트 선택 과정에서 열람하거나 학습에 사용하지 않음 + - 모델 출력은 JSON Schema와 exact substring 규칙 검증 후에만 후속 단계로 전달 + - 민감 Intent와 모든 외부 실행은 HR 승인·Guardrail 검사를 별도로 적용 +regeneration: + required_when: + - A/B consensus가 현재 라벨을 변경 + - source record count 또는 SHA-256이 변경 + - Intent 규칙 버전 또는 split 정책이 변경 + command: >- + python -m fowoco_knowledge build-intent-splits diff --git a/fowoco-knowledge/knowledge/manifest.yaml b/fowoco-knowledge/knowledge/manifest.yaml index 9bed371..614a8de 100644 --- a/fowoco-knowledge/knowledge/manifest.yaml +++ b/fowoco-knowledge/knowledge/manifest.yaml @@ -26,6 +26,11 @@ files: easy_korean: knowledge/easy_korean_rules.yaml multilingual_templates: knowledge/multilingual_templates.yaml guardrails: knowledge/guardrail_rules.yaml + intent_model_contract: knowledge/intent_model_contract.yaml + intent_evaluation_policy: knowledge/intent_evaluation_policy.yaml +schemas: + intent_model_output: schemas/intent-model-output.schema.json + intent_split_manifest: schemas/intent-split-manifest.schema.json datasets: source_manifest: data/external/source_manifest.yaml processed_manifest: data/processed/manifest.yaml @@ -33,6 +38,9 @@ datasets: manufacturing_industries: data/processed/manufacturing_industries.csv intent_manifest: data/intent/manifest.yaml intent_training_candidates: data/intent/hr_intent_dataset.jsonl + intent_split_manifest: data/intent/splits/provisional-v1/manifest.yaml + intent_train_ids: data/intent/splits/provisional-v1/train_ids.txt + intent_validation_ids: data/intent/splits/provisional-v1/validation_ids.txt review_policy: minimum_reviewers_for_official_knowledge: 2 all_outbound_messages_require_hr_approval: true diff --git a/fowoco-knowledge/schemas/intent-model-output.schema.json b/fowoco-knowledge/schemas/intent-model-output.schema.json new file mode 100644 index 0000000..4d68790 --- /dev/null +++ b/fowoco-knowledge/schemas/intent-model-output.schema.json @@ -0,0 +1,82 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://fowoco.dev/schemas/intent-model-output.schema.json", + "title": "FOWOCO MVP Intent model output", + "type": "object", + "additionalProperties": false, + "required": [ + "intents" + ], + "properties": { + "intents": { + "type": "array", + "minItems": 1, + "maxItems": 6, + "items": { + "$ref": "#/$defs/intentEvidence" + } + } + }, + "$defs": { + "intentCode": { + "type": "string", + "enum": [ + "WORK_INSTRUCTION", + "DOCUMENT_REQUEST", + "PAYROLL_EXPLANATION", + "WORKER_ONBOARDING", + "EMPLOYMENT_CHANGE", + "EXPIRY_RENEWAL", + "OUT_OF_SCOPE" + ] + }, + "intentEvidence": { + "type": "object", + "additionalProperties": false, + "required": [ + "intent", + "evidence" + ], + "properties": { + "intent": { + "$ref": "#/$defs/intentCode" + }, + "evidence": { + "type": [ + "string", + "null" + ] + } + }, + "allOf": [ + { + "if": { + "properties": { + "intent": { + "const": "OUT_OF_SCOPE" + } + }, + "required": [ + "intent" + ] + }, + "then": { + "properties": { + "evidence": { + "type": "null" + } + } + }, + "else": { + "properties": { + "evidence": { + "type": "string", + "minLength": 1 + } + } + } + } + ] + } + } +} diff --git a/fowoco-knowledge/schemas/intent-split-manifest.schema.json b/fowoco-knowledge/schemas/intent-split-manifest.schema.json new file mode 100644 index 0000000..e0262dc --- /dev/null +++ b/fowoco-knowledge/schemas/intent-split-manifest.schema.json @@ -0,0 +1,302 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://fowoco.dev/schemas/intent-split-manifest.schema.json", + "title": "FOWOCO Intent Train and Validation split manifest", + "type": "object", + "additionalProperties": false, + "required": [ + "split_id", + "version", + "status", + "schema", + "source", + "consensus", + "policy", + "outputs", + "statistics", + "limitations" + ], + "properties": { + "split_id": { + "type": "string", + "const": "FOWOCO-HR-INTENT-PROVISIONAL-SPLIT" + }, + "version": { + "type": "string", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "status": { + "type": "string", + "enum": [ + "provisional_pending_consensus", + "locked" + ] + }, + "schema": { + "type": "string" + }, + "source": { + "$ref": "#/$defs/source" + }, + "consensus": { + "$ref": "#/$defs/consensus" + }, + "policy": { + "$ref": "#/$defs/policy" + }, + "outputs": { + "type": "object", + "additionalProperties": false, + "required": [ + "train", + "validation" + ], + "properties": { + "train": { + "$ref": "#/$defs/output" + }, + "validation": { + "$ref": "#/$defs/output" + } + } + }, + "statistics": { + "$ref": "#/$defs/statistics" + }, + "limitations": { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "minLength": 1 + } + } + }, + "$defs": { + "source": { + "type": "object", + "additionalProperties": false, + "required": [ + "dataset_id", + "version", + "status", + "path", + "record_count", + "sha256" + ], + "properties": { + "dataset_id": { + "type": "string" + }, + "version": { + "type": "string" + }, + "status": { + "type": "string" + }, + "path": { + "type": "string" + }, + "record_count": { + "type": "integer", + "minimum": 1 + }, + "sha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + } + } + }, + "consensus": { + "type": "object", + "additionalProperties": false, + "required": [ + "status", + "assumed_for_provisional_split", + "dependency_issue", + "dependency_pr", + "regenerate_when_source_changes" + ], + "properties": { + "status": { + "type": "string", + "enum": [ + "pending", + "completed" + ] + }, + "assumed_for_provisional_split": { + "type": "boolean" + }, + "dependency_issue": { + "type": "integer", + "minimum": 1 + }, + "dependency_pr": { + "type": "integer", + "minimum": 1 + }, + "regenerate_when_source_changes": { + "type": "boolean" + } + } + }, + "policy": { + "type": "object", + "additionalProperties": false, + "required": [ + "method", + "seed", + "train_ratio", + "validation_ratio", + "target_train_count", + "target_validation_count", + "template_normalization", + "stratification_features" + ], + "properties": { + "method": { + "type": "string", + "const": "grouped_multilabel_greedy" + }, + "seed": { + "type": "integer" + }, + "train_ratio": { + "type": "number", + "exclusiveMinimum": 0, + "exclusiveMaximum": 1 + }, + "validation_ratio": { + "type": "number", + "exclusiveMinimum": 0, + "exclusiveMaximum": 1 + }, + "target_train_count": { + "type": "integer", + "minimum": 1 + }, + "target_validation_count": { + "type": "integer", + "minimum": 1 + }, + "template_normalization": { + "type": "object", + "additionalProperties": false, + "required": [ + "worker_id_pattern", + "worker_id_replacement", + "collapse_whitespace", + "casefold", + "preserve_dates_amounts_and_task_terms" + ], + "properties": { + "worker_id_pattern": { + "type": "string" + }, + "worker_id_replacement": { + "type": "string" + }, + "collapse_whitespace": { + "type": "boolean" + }, + "casefold": { + "type": "boolean" + }, + "preserve_dates_amounts_and_task_terms": { + "type": "boolean" + } + } + }, + "stratification_features": { + "type": "array", + "minItems": 1, + "items": { + "type": "string" + } + } + } + }, + "output": { + "type": "object", + "additionalProperties": false, + "required": [ + "path", + "record_count", + "sha256" + ], + "properties": { + "path": { + "type": "string" + }, + "record_count": { + "type": "integer", + "minimum": 1 + }, + "sha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + } + } + }, + "statistics": { + "type": "object", + "additionalProperties": false, + "required": [ + "template_group_count", + "duplicate_template_group_count", + "max_template_group_size", + "label_counts", + "intent_cardinality_counts" + ], + "properties": { + "template_group_count": { + "type": "integer", + "minimum": 1 + }, + "duplicate_template_group_count": { + "type": "integer", + "minimum": 0 + }, + "max_template_group_size": { + "type": "integer", + "minimum": 1 + }, + "label_counts": { + "$ref": "#/$defs/partitionedCounts" + }, + "intent_cardinality_counts": { + "$ref": "#/$defs/partitionedCounts" + } + } + }, + "partitionedCounts": { + "type": "object", + "additionalProperties": false, + "required": [ + "source", + "train", + "validation" + ], + "properties": { + "source": { + "$ref": "#/$defs/counts" + }, + "train": { + "$ref": "#/$defs/counts" + }, + "validation": { + "$ref": "#/$defs/counts" + } + } + }, + "counts": { + "type": "object", + "minProperties": 1, + "additionalProperties": { + "type": "integer", + "minimum": 0 + } + } + } +} diff --git a/fowoco-knowledge/src/fowoco_knowledge/cli.py b/fowoco-knowledge/src/fowoco_knowledge/cli.py index 07caa07..db3a84a 100644 --- a/fowoco-knowledge/src/fowoco_knowledge/cli.py +++ b/fowoco-knowledge/src/fowoco_knowledge/cli.py @@ -7,6 +7,11 @@ from .engine import RequestEvaluator from .ingestion import OfficialDataPipeline +from .intent_split import ( + DEFAULT_SEED, + DEFAULT_VALIDATION_RATIO, + build_and_write_intent_split, +) from .repository import KnowledgeRepository from .validation import KnowledgeValidator @@ -62,6 +67,22 @@ def build_parser() -> argparse.ArgumentParser: ) industry_parser.add_argument("query") industry_parser.add_argument("--limit", type=int, default=20) + + split_parser = subparsers.add_parser( + "build-intent-splits", + help="Build provisional grouped Train/Validation ID manifests", + ) + split_parser.add_argument( + "--output-dir", + type=Path, + help="Output directory inside the knowledge project", + ) + split_parser.add_argument("--seed", type=int, default=DEFAULT_SEED) + split_parser.add_argument( + "--validation-ratio", + type=float, + default=DEFAULT_VALIDATION_RATIO, + ) return parser @@ -130,4 +151,14 @@ def main(argv: Sequence[str] | None = None) -> int: print(f"{row['industry_id']}\t{row['middle_category']}\t{row['business_content_ko']}") return 0 + if args.command == "build-intent-splits": + manifest = build_and_write_intent_split( + repository.root, + output_dir=args.output_dir, + seed=args.seed, + validation_ratio=args.validation_ratio, + ) + print(json.dumps(manifest, ensure_ascii=False, indent=2)) + return 0 + return 2 diff --git a/fowoco-knowledge/src/fowoco_knowledge/intent_model.py b/fowoco-knowledge/src/fowoco_knowledge/intent_model.py new file mode 100644 index 0000000..7d53658 --- /dev/null +++ b/fowoco-knowledge/src/fowoco_knowledge/intent_model.py @@ -0,0 +1,84 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +from jsonschema import Draft202012Validator + + +@dataclass(frozen=True) +class IntentOutputIssue: + code: str + message: str + + +def validate_intent_model_output( + hr_input: str, + output: Any, + schema: dict[str, Any], +) -> tuple[IntentOutputIssue, ...]: + """Validate model output constraints that JSON Schema alone cannot express.""" + issues: list[IntentOutputIssue] = [] + validator = Draft202012Validator(schema) + for error in sorted( + validator.iter_errors(output), + key=lambda item: "/".join(str(part) for part in item.path), + ): + path = ".".join(str(item) for item in error.path) + location = f" at {path}" if path else "" + issues.append( + IntentOutputIssue( + code="INVALID_JSON", + message=f"output schema violation{location}: {error.message}", + ) + ) + + if not isinstance(output, dict) or not isinstance(output.get("intents"), list): + return tuple(issues) + + intent_codes: list[str] = [] + evidence_positions: list[int] = [] + for item in output["intents"]: + if not isinstance(item, dict): + continue + intent = item.get("intent") + evidence = item.get("evidence") + if isinstance(intent, str): + if intent in intent_codes: + issues.append( + IntentOutputIssue( + code="DUPLICATE_INTENT", + message=f"duplicate intent: {intent}", + ) + ) + intent_codes.append(intent) + + if intent == "OUT_OF_SCOPE": + continue + if isinstance(evidence, str) and evidence: + position = hr_input.find(evidence) + if position < 0: + issues.append( + IntentOutputIssue( + code="EVIDENCE_NOT_SUBSTRING", + message=f"evidence is not an exact substring: {evidence}", + ) + ) + else: + evidence_positions.append(position) + + if "OUT_OF_SCOPE" in intent_codes and len(intent_codes) != 1: + issues.append( + IntentOutputIssue( + code="OUT_OF_SCOPE_MIXED", + message="OUT_OF_SCOPE must be the only intent", + ) + ) + if evidence_positions != sorted(evidence_positions): + issues.append( + IntentOutputIssue( + code="INTENT_ORDER_ERROR", + message="intents are not in evidence order", + ) + ) + return tuple(issues) diff --git a/fowoco-knowledge/src/fowoco_knowledge/intent_split.py b/fowoco-knowledge/src/fowoco_knowledge/intent_split.py new file mode 100644 index 0000000..a240bfb --- /dev/null +++ b/fowoco-knowledge/src/fowoco_knowledge/intent_split.py @@ -0,0 +1,334 @@ +from __future__ import annotations + +import hashlib +import json +import re +from collections import Counter, defaultdict +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import yaml + +from .ingestion import file_sha256 + +DEFAULT_SEED = 20260727 +DEFAULT_VALIDATION_RATIO = 0.2 +DEFAULT_OUTPUT_DIR = Path("data/intent/splits/provisional-v1") +WORKER_ID_PATTERN = re.compile(r"(?i)\bwrk[-_ ]?\d+\b") + + +class IntentSplitError(ValueError): + """Raised when an Intent split cannot be generated safely.""" + + +@dataclass(frozen=True) +class TemplateGroup: + key: str + cases: tuple[dict[str, Any], ...] + features: Counter[str] + rank: str + + @property + def size(self) -> int: + return len(self.cases) + + +@dataclass(frozen=True) +class IntentSplitResult: + train_ids: tuple[int, ...] + validation_ids: tuple[int, ...] + template_group_count: int + duplicate_template_group_count: int + max_template_group_size: int + + +def normalize_intent_template(hr_input: str) -> str: + """Normalize identifiers without erasing task-defining dates, amounts, or nouns.""" + without_worker_id = WORKER_ID_PATTERN.sub("", hr_input) + return " ".join(without_worker_id.split()).casefold() + + +def load_intent_cases(path: Path) -> list[dict[str, Any]]: + cases: list[dict[str, Any]] = [] + for line_number, raw_line in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1): + if not raw_line.strip(): + continue + try: + case = json.loads(raw_line) + except json.JSONDecodeError as exc: + raise IntentSplitError(f"intent source line {line_number}: invalid JSON") from exc + if not isinstance(case, dict): + raise IntentSplitError(f"intent source line {line_number}: record must be an object") + cases.append(case) + return cases + + +def read_split_ids(path: Path) -> tuple[int, ...]: + ids: list[int] = [] + for line_number, raw_line in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1): + value = raw_line.strip() + if not value: + continue + try: + ids.append(int(value)) + except ValueError as exc: + raise IntentSplitError(f"{path} line {line_number}: invalid ID") from exc + return tuple(ids) + + +def _case_features(cases: list[dict[str, Any]] | tuple[dict[str, Any], ...]) -> Counter[str]: + features: Counter[str] = Counter() + for case in cases: + intent_codes = tuple(item["intent"] for item in case["intents"]) + for intent_code in intent_codes: + features[f"intent:{intent_code}"] += 1 + features[f"cardinality:{len(intent_codes)}"] += 1 + features[f"combination:{'|'.join(intent_codes)}"] += 1 + return features + + +def _build_template_groups(cases: list[dict[str, Any]], seed: int) -> list[TemplateGroup]: + grouped: defaultdict[str, list[dict[str, Any]]] = defaultdict(list) + for case in cases: + grouped[normalize_intent_template(case["hr_input"])].append(case) + + groups: list[TemplateGroup] = [] + for key, grouped_cases in grouped.items(): + ordered_cases = tuple(sorted(grouped_cases, key=lambda item: item["id"])) + rank = hashlib.sha256(f"{seed}\0{key}".encode()).hexdigest() + groups.append( + TemplateGroup( + key=key, + cases=ordered_cases, + features=_case_features(ordered_cases), + rank=rank, + ) + ) + return groups + + +def _distribution_cost( + counts: Counter[str], + record_count: int, + target_counts: dict[str, float], + target_record_count: int, +) -> float: + cost = 4 * ((record_count - target_record_count) ** 2 / max(target_record_count, 1)) + for feature, target in target_counts.items(): + if feature.startswith("intent:"): + weight = 1.0 + elif feature.startswith("cardinality:"): + weight = 0.6 + else: + weight = 0.35 + cost += weight * ((counts[feature] - target) ** 2 / max(target, 1)) + return cost + + +def build_intent_split( + cases: list[dict[str, Any]], + *, + seed: int = DEFAULT_SEED, + validation_ratio: float = DEFAULT_VALIDATION_RATIO, +) -> IntentSplitResult: + if not cases: + raise IntentSplitError("intent source is empty") + if not 0 < validation_ratio < 1: + raise IntentSplitError("validation_ratio must be between 0 and 1") + + source_ids = [case["id"] for case in cases] + if any(not isinstance(case_id, int) for case_id in source_ids): + raise IntentSplitError("every intent case must have an integer id") + if len(source_ids) != len(set(source_ids)): + raise IntentSplitError("intent source contains duplicate ids") + + groups = _build_template_groups(cases, seed) + total_features = _case_features(cases) + target_record_count = round(len(cases) * validation_ratio) + target_counts = {feature: count * validation_ratio for feature, count in total_features.items()} + + remaining = groups.copy() + selected: list[TemplateGroup] = [] + validation_features: Counter[str] = Counter() + validation_count = 0 + + while validation_count < target_record_count: + remaining_capacity = target_record_count - validation_count + candidates = [group for group in remaining if group.size <= remaining_capacity] + if not candidates: + raise IntentSplitError("template grouping cannot satisfy the requested split size") + + selected_group = min( + candidates, + key=lambda group: ( + _distribution_cost( + validation_features + group.features, + validation_count + group.size, + target_counts, + target_record_count, + ), + group.rank, + ), + ) + selected.append(selected_group) + validation_features.update(selected_group.features) + validation_count += selected_group.size + remaining.remove(selected_group) + + validation_ids = tuple(sorted(case["id"] for group in selected for case in group.cases)) + train_ids = tuple(sorted(set(source_ids) - set(validation_ids))) + group_sizes = [group.size for group in groups] + return IntentSplitResult( + train_ids=train_ids, + validation_ids=validation_ids, + template_group_count=len(groups), + duplicate_template_group_count=sum(size > 1 for size in group_sizes), + max_template_group_size=max(group_sizes), + ) + + +def _label_counts(cases: list[dict[str, Any]]) -> dict[str, int]: + counts = Counter(item["intent"] for case in cases for item in case["intents"]) + return dict(sorted(counts.items())) + + +def _cardinality_counts(cases: list[dict[str, Any]]) -> dict[str, int]: + counts = Counter(str(len(case["intents"])) for case in cases) + return dict(sorted(counts.items(), key=lambda item: int(item[0]))) + + +def _relative_path(path: Path, root: Path) -> str: + try: + return path.resolve().relative_to(root.resolve()).as_posix() + except ValueError as exc: + raise IntentSplitError("split outputs must stay inside the knowledge project") from exc + + +def _write_ids(path: Path, ids: tuple[int, ...]) -> None: + path.write_text("".join(f"{case_id}\n" for case_id in ids), encoding="utf-8") + + +def build_and_write_intent_split( + root: Path, + *, + output_dir: Path | None = None, + seed: int = DEFAULT_SEED, + validation_ratio: float = DEFAULT_VALIDATION_RATIO, +) -> dict[str, Any]: + root = root.resolve() + intent_manifest_path = root / "data/intent/manifest.yaml" + intent_manifest = yaml.safe_load(intent_manifest_path.read_text(encoding="utf-8")) + source_path = root / intent_manifest["path"] + actual_source_sha256 = file_sha256(source_path) + if actual_source_sha256 != intent_manifest["sha256"]: + raise IntentSplitError("intent source checksum differs from data/intent/manifest.yaml") + + cases = load_intent_cases(source_path) + if len(cases) != intent_manifest["record_count"]: + raise IntentSplitError("intent source record count differs from its manifest") + + result = build_intent_split( + cases, + seed=seed, + validation_ratio=validation_ratio, + ) + if output_dir is None: + resolved_output_dir = (root / DEFAULT_OUTPUT_DIR).resolve() + elif output_dir.is_absolute(): + resolved_output_dir = output_dir.resolve() + else: + resolved_output_dir = (root / output_dir).resolve() + _relative_path(resolved_output_dir, root) + resolved_output_dir.mkdir(parents=True, exist_ok=True) + + train_path = resolved_output_dir / "train_ids.txt" + validation_path = resolved_output_dir / "validation_ids.txt" + manifest_path = resolved_output_dir / "manifest.yaml" + _write_ids(train_path, result.train_ids) + _write_ids(validation_path, result.validation_ids) + + cases_by_id = {case["id"]: case for case in cases} + train_cases = [cases_by_id[case_id] for case_id in result.train_ids] + validation_cases = [cases_by_id[case_id] for case_id in result.validation_ids] + train_ratio = 1 - validation_ratio + split_manifest: dict[str, Any] = { + "split_id": "FOWOCO-HR-INTENT-PROVISIONAL-SPLIT", + "version": "0.1.0", + "status": "provisional_pending_consensus", + "schema": "schemas/intent-split-manifest.schema.json", + "source": { + "dataset_id": intent_manifest["dataset_id"], + "version": intent_manifest["version"], + "status": intent_manifest["status"], + "path": intent_manifest["path"], + "record_count": intent_manifest["record_count"], + "sha256": actual_source_sha256, + }, + "consensus": { + "status": "pending", + "assumed_for_provisional_split": True, + "dependency_issue": 30, + "dependency_pr": 31, + "regenerate_when_source_changes": True, + }, + "policy": { + "method": "grouped_multilabel_greedy", + "seed": seed, + "train_ratio": train_ratio, + "validation_ratio": validation_ratio, + "target_train_count": len(result.train_ids), + "target_validation_count": len(result.validation_ids), + "template_normalization": { + "worker_id_pattern": WORKER_ID_PATTERN.pattern, + "worker_id_replacement": "", + "collapse_whitespace": True, + "casefold": True, + "preserve_dates_amounts_and_task_terms": True, + }, + "stratification_features": [ + "intent_code", + "intent_cardinality", + "ordered_intent_combination", + ], + }, + "outputs": { + "train": { + "path": _relative_path(train_path, root), + "record_count": len(result.train_ids), + "sha256": file_sha256(train_path), + }, + "validation": { + "path": _relative_path(validation_path, root), + "record_count": len(result.validation_ids), + "sha256": file_sha256(validation_path), + }, + }, + "statistics": { + "template_group_count": result.template_group_count, + "duplicate_template_group_count": result.duplicate_template_group_count, + "max_template_group_size": result.max_template_group_size, + "label_counts": { + "source": _label_counts(cases), + "train": _label_counts(train_cases), + "validation": _label_counts(validation_cases), + }, + "intent_cardinality_counts": { + "source": _cardinality_counts(cases), + "train": _cardinality_counts(train_cases), + "validation": _cardinality_counts(validation_cases), + }, + }, + "limitations": [ + "Reviewer B와 A/B consensus 완료 전에는 최종 학습 분할이 아님", + "Reviewer A의 proposed_intents_json은 현재 원본에 적용하지 않은 상태임", + "독립 Gold Test를 포함하지 않음", + "source SHA-256이 바뀌면 생성기를 다시 실행해야 함", + "분할 비율과 내부 목표치는 baseline 비교용이며 성능 보장을 뜻하지 않음", + ], + } + manifest_path.write_text( + yaml.safe_dump(split_manifest, allow_unicode=True, sort_keys=False), + encoding="utf-8", + ) + return split_manifest diff --git a/fowoco-knowledge/src/fowoco_knowledge/validation.py b/fowoco-knowledge/src/fowoco_knowledge/validation.py index 471dac5..b463275 100644 --- a/fowoco-knowledge/src/fowoco_knowledge/validation.py +++ b/fowoco-knowledge/src/fowoco_knowledge/validation.py @@ -3,11 +3,20 @@ import csv import json import re +from collections import Counter from typing import Any from jsonschema import Draft202012Validator +from jsonschema.exceptions import SchemaError from .ingestion import file_sha256 +from .intent_split import ( + IntentSplitError, + build_intent_split, + load_intent_cases, + normalize_intent_template, + read_split_ids, +) from .repository import KnowledgeRepository SEED_COLUMNS = { @@ -85,6 +94,8 @@ def validate_all(self) -> list[str]: self._validate_seed_data() self._validate_evaluation_data() self._validate_intent_data() + self._validate_intent_model_contract() + self._validate_intent_split_data() return self.errors def _validate_manifest_files(self) -> None: @@ -95,6 +106,15 @@ def _validate_manifest_files(self) -> None: for key, relative_path in manifest.get("datasets", {}).items(): if not (self.repository.root / relative_path).is_file(): self.errors.append(f"manifest dataset missing: {key} -> {relative_path}") + for key, relative_path in manifest.get("schemas", {}).items(): + path = self.repository.root / relative_path + if not path.is_file(): + self.errors.append(f"manifest schema missing: {key} -> {relative_path}") + continue + try: + Draft202012Validator.check_schema(self.repository.load_json(relative_path)) + except SchemaError as exc: + self.errors.append(f"manifest schema invalid: {key} ({exc.message})") def _validate_processed_datasets(self) -> None: processed_manifest_path = self.repository.root / "data/processed/manifest.yaml" @@ -417,6 +437,136 @@ def _validate_intent_data(self) -> None: if file_sha256(path) != intent_manifest["sha256"]: self.errors.append("intent data: checksum mismatch") + def _validate_intent_model_contract(self) -> None: + manifest = self.repository.manifest + contract = self.repository.load_yaml(manifest["files"]["intent_model_contract"]) + evaluation_policy = self.repository.load_yaml(manifest["files"]["intent_evaluation_policy"]) + context = self.repository.load_context_files() + known_intents = {item["id"] for item in context["intents"]["intents"]} | { + context["intents"]["out_of_scope_label"] + } + + if set(contract["allowed_intents"]) != known_intents: + self.errors.append("intent model contract: allowed intents differ from knowledge") + if contract["task"]["output_schema"] != manifest["schemas"]["intent_model_output"]: + self.errors.append("intent model contract: output schema path mismatch") + if evaluation_policy["applies_to"] != contract["contract_id"]: + self.errors.append("intent evaluation policy: contract reference mismatch") + required_gate_ids = { + "JSON_SCHEMA_VALID", + "EVIDENCE_EXACT_SUBSTRING", + "INTENT_ORDER", + "OUT_OF_SCOPE_EXCLUSIVE", + } + actual_gate_ids = {item["id"] for item in evaluation_policy["structural_gates"]} + if actual_gate_ids != required_gate_ids: + self.errors.append("intent evaluation policy: structural gate set mismatch") + if any(item["required_rate"] != 1.0 for item in evaluation_policy["structural_gates"]): + self.errors.append("intent evaluation policy: structural gates must require 100%") + + def _validate_intent_split_data(self) -> None: + intent_manifest = self.repository.load_yaml("data/intent/manifest.yaml") + split_manifest_path = intent_manifest["split_manifest"] + split_manifest_file = self.repository.root / split_manifest_path + if not split_manifest_file.is_file(): + self.errors.append(f"intent split manifest missing: {split_manifest_path}") + return + + split_manifest = self.repository.load_yaml(split_manifest_path) + split_schema = self.repository.load_json(split_manifest["schema"]) + for error in Draft202012Validator(split_schema).iter_errors(split_manifest): + error_path = ".".join(str(item) for item in error.path) + location = f" [{error_path}]" if error_path else "" + self.errors.append(f"intent split manifest{location}: {error.message}") + + source = split_manifest["source"] + source_path = self.repository.root / source["path"] + if source["path"] != intent_manifest["path"]: + self.errors.append("intent split: source path differs from intent manifest") + if source["version"] != intent_manifest["version"]: + self.errors.append("intent split: source version differs from intent manifest") + if source["record_count"] != intent_manifest["record_count"]: + self.errors.append("intent split: source count differs from intent manifest") + if source["sha256"] != intent_manifest["sha256"]: + self.errors.append("intent split: source checksum differs from intent manifest") + if source_path.is_file() and file_sha256(source_path) != source["sha256"]: + self.errors.append("intent split: source file checksum mismatch") + + split_ids: dict[str, tuple[int, ...]] = {} + for split_name in ("train", "validation"): + output = split_manifest["outputs"][split_name] + output_path = self.repository.root / output["path"] + if not output_path.is_file(): + self.errors.append(f"intent split: missing {split_name} ID file") + continue + if file_sha256(output_path) != output["sha256"]: + self.errors.append(f"intent split: {split_name} checksum mismatch") + try: + ids = read_split_ids(output_path) + except IntentSplitError as exc: + self.errors.append(f"intent split: {exc}") + continue + split_ids[split_name] = ids + if len(ids) != output["record_count"]: + self.errors.append(f"intent split: {split_name} count mismatch") + if len(ids) != len(set(ids)): + self.errors.append(f"intent split: duplicate ID in {split_name}") + if tuple(sorted(ids)) != ids: + self.errors.append(f"intent split: {split_name} IDs are not sorted") + + if set(split_ids) != {"train", "validation"} or not source_path.is_file(): + return + + cases = load_intent_cases(source_path) + source_ids = {case["id"] for case in cases} + train_ids = set(split_ids["train"]) + validation_ids = set(split_ids["validation"]) + if train_ids & validation_ids: + self.errors.append("intent split: Train and Validation overlap") + if train_ids | validation_ids != source_ids: + self.errors.append("intent split: IDs do not cover the full source") + + split_by_id = { + **{case_id: "train" for case_id in train_ids}, + **{case_id: "validation" for case_id in validation_ids}, + } + template_splits: dict[str, set[str]] = {} + for case in cases: + template = normalize_intent_template(case["hr_input"]) + template_splits.setdefault(template, set()).add(split_by_id.get(case["id"], "missing")) + if any(len(splits) != 1 for splits in template_splits.values()): + self.errors.append("intent split: normalized template leaks across splits") + + policy = split_manifest["policy"] + regenerated = build_intent_split( + cases, + seed=policy["seed"], + validation_ratio=policy["validation_ratio"], + ) + if regenerated.train_ids != split_ids["train"]: + self.errors.append("intent split: Train IDs are not reproducible") + if regenerated.validation_ids != split_ids["validation"]: + self.errors.append("intent split: Validation IDs are not reproducible") + + cases_by_id = {case["id"]: case for case in cases} + partition_cases = { + "source": cases, + "train": [cases_by_id[case_id] for case_id in split_ids["train"]], + "validation": [cases_by_id[case_id] for case_id in split_ids["validation"]], + } + expected_label_counts = { + split_name: dict( + sorted( + Counter( + item["intent"] for case in split_cases for item in case["intents"] + ).items() + ) + ) + for split_name, split_cases in partition_cases.items() + } + if expected_label_counts != split_manifest["statistics"]["label_counts"]: + self.errors.append("intent split: label count statistics mismatch") + def _index_unique(self, items: list[dict[str, Any]], kind: str) -> dict[str, dict[str, Any]]: indexed: dict[str, dict[str, Any]] = {} for item in items: diff --git a/fowoco-knowledge/tests/test_intent_modeling.py b/fowoco-knowledge/tests/test_intent_modeling.py new file mode 100644 index 0000000..085f4c9 --- /dev/null +++ b/fowoco-knowledge/tests/test_intent_modeling.py @@ -0,0 +1,114 @@ +from __future__ import annotations + +import json +from collections import Counter +from pathlib import Path + +from jsonschema import Draft202012Validator + +from fowoco_knowledge.ingestion import file_sha256 +from fowoco_knowledge.intent_model import validate_intent_model_output +from fowoco_knowledge.intent_split import ( + build_intent_split, + load_intent_cases, + normalize_intent_template, + read_split_ids, +) +from fowoco_knowledge.repository import KnowledgeRepository + +ROOT = Path(__file__).resolve().parents[1] + + +def test_intent_model_output_contract_and_runtime_constraints() -> None: + repository = KnowledgeRepository(ROOT) + schema = repository.load_json("schemas/intent-model-output.schema.json") + valid_output = { + "intents": [ + {"intent": "EXPIRY_RENEWAL", "evidence": "재계약 준비하면서"}, + {"intent": "DOCUMENT_REQUEST", "evidence": "서명본도 받아서"}, + ] + } + hr_input = "재계약 준비하면서 서명본도 받아서 첨부해줘" + + assert list(Draft202012Validator(schema).iter_errors(valid_output)) == [] + assert validate_intent_model_output(hr_input, valid_output, schema) == () + + invalid_output = { + "request_id": "req_should_be_server_owned", + "intents": [ + {"intent": "DOCUMENT_REQUEST", "evidence": "원문에 없는 증거"}, + {"intent": "OUT_OF_SCOPE", "evidence": None}, + ], + } + issue_codes = { + issue.code for issue in validate_intent_model_output(hr_input, invalid_output, schema) + } + assert issue_codes == { + "INVALID_JSON", + "EVIDENCE_NOT_SUBSTRING", + "OUT_OF_SCOPE_MIXED", + } + + +def test_provisional_intent_split_is_reproducible_and_grouped() -> None: + repository = KnowledgeRepository(ROOT) + manifest = repository.load_yaml("data/intent/splits/provisional-v1/manifest.yaml") + cases = load_intent_cases(ROOT / manifest["source"]["path"]) + train_ids = read_split_ids(ROOT / manifest["outputs"]["train"]["path"]) + validation_ids = read_split_ids(ROOT / manifest["outputs"]["validation"]["path"]) + + assert len(train_ids) == 1072 + assert len(validation_ids) == 268 + assert not set(train_ids) & set(validation_ids) + assert set(train_ids) | set(validation_ids) == {case["id"] for case in cases} + + regenerated = build_intent_split( + cases, + seed=manifest["policy"]["seed"], + validation_ratio=manifest["policy"]["validation_ratio"], + ) + assert regenerated.train_ids == train_ids + assert regenerated.validation_ids == validation_ids + + split_by_id = { + **{case_id: "train" for case_id in train_ids}, + **{case_id: "validation" for case_id in validation_ids}, + } + template_splits: dict[str, set[str]] = {} + for case in cases: + key = normalize_intent_template(case["hr_input"]) + template_splits.setdefault(key, set()).add(split_by_id[case["id"]]) + assert all(len(splits) == 1 for splits in template_splits.values()) + + source_counts = Counter(item["intent"] for case in cases for item in case["intents"]) + validation_id_set = set(validation_ids) + validation_cases = [case for case in cases if case["id"] in validation_id_set] + validation_counts = Counter( + item["intent"] for case in validation_cases for item in case["intents"] + ) + for intent, source_count in source_counts.items(): + observed_ratio = validation_counts[intent] / source_count + assert abs(observed_ratio - manifest["policy"]["validation_ratio"]) <= 0.01 + + +def test_provisional_split_output_checksums_match_manifest() -> None: + repository = KnowledgeRepository(ROOT) + manifest = repository.load_yaml("data/intent/splits/provisional-v1/manifest.yaml") + for output in manifest["outputs"].values(): + assert file_sha256(ROOT / output["path"]) == output["sha256"] + + +def test_training_source_has_no_direct_gold_test_overlap_contract() -> None: + intent_cases = load_intent_cases(ROOT / "data/intent/hr_intent_dataset.jsonl") + smoke_cases = [ + json.loads(line) + for line in (ROOT / "data/evaluation/golden_cases.jsonl") + .read_text(encoding="utf-8") + .splitlines() + if line.strip() + ] + normalized_training_inputs = { + normalize_intent_template(case["hr_input"]) for case in intent_cases + } + normalized_smoke_inputs = {normalize_intent_template(case["utterance"]) for case in smoke_cases} + assert not normalized_training_inputs & normalized_smoke_inputs