hanarang/reloop-v2
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(parse): extract reusable parser + fix segmentation — 7F.4

Browse Source
This commit is contained in:
reloop
2026-04-12 06:15:07 +09:00
parent 350320fa8d
commit c9935cf303
9 changed files with 769 additions and 413 deletions
Download Patch File Download Diff File Expand all files Collapse all files

580
backend/scripts/kice-import.ts
View File

@@ -1,29 +1,12 @@
/**
* KICE (한국교육과정평가원) 수능 기출 PDF import 스크립트.
*
* 사용법:
* pnpm cli:kice-import --dry-run # 파일 목록만 출력
* pnpm cli:kice-import --year=2026 # 특정 연도만
* pnpm cli:kice-import --subject=국어 # 특정 과목만
* pnpm cli:kice-import --year=2026 --subject=국어 --only=answer
* pnpm cli:kice-import --year=2026 --subject=국어 --only=problems --sample=1
* pnpm cli:kice-import # 전체 (2025/2026 × 4과목)
*
* 전제:
* - poppler-utils (pdftotext) 가 PATH 에 있어야 함
* - data/kice/ 하위에 PDF 파일이 배치돼있어야 함
* - backend/.env 에 DATABASE_URL 설정
*
* MVP 범위: 국어 / 영어 / 한국사 / 생활과 윤리 (사탐 1과목). 2025/2026 학년도.
* 수학/과탐은 수식이 벡터 그래픽이라 텍스트 파싱 부적합 — Phase 8 별도 트랙.
*/
import * as fs from 'fs';
import * as path from 'path';
import { execSync } from 'child_process';
import { PrismaClient, Prisma } from '@prisma/client';
import { parseExamPaper } from '../src/problem-sets/parsing';
// ─── CLI 옵션 ────────────────────────────────────────────────────────
interface CliOptions {
dryRun: boolean;
year?: number;
@@ -34,23 +17,31 @@ interface CliOptions {
function parseArgs(): CliOptions {
const args = process.argv.slice(2);
const opts: CliOptions = { dryRun: false, only: 'all' };
for (const a of args) {
if (a === '--dry-run') opts.dryRun = true;
else if (a.startsWith('--year=')) opts.year = Number(a.split('=')[1]);
else if (a.startsWith('--subject=')) opts.subject = a.split('=')[1];
else if (a.startsWith('--only=')) opts.only = a.split('=')[1] as CliOptions['only'];
else if (a.startsWith('--sample=')) opts.sample = Number(a.split('=')[1]);
const options: CliOptions = { dryRun: false, only: 'all' };
for (const arg of args) {
if (arg === '--dry-run') options.dryRun = true;
else if (arg.startsWith('--year=')) options.year = Number(arg.split('=')[1]);
else if (arg.startsWith('--subject=')) options.subject = arg.split('=')[1];
else if (arg.startsWith('--only=')) options.only = arg.split('=')[1] as CliOptions['only'];
else if (arg.startsWith('--sample=')) options.sample = Number(arg.split('=')[1]);
}
return opts;
return options;
}
// ─── 경로 상수 ────────────────────────────────────────────────────────
const DATA_DIR = path.resolve(__dirname, '../../data/kice');
const TARGET_YEARS = [2025, 2026];
const TARGET_SUBJECTS = ['국어', '영어', '한국사', '생활과 윤리'] as const;
type TargetSubject = (typeof TARGET_SUBJECTS)[number];
const EXPECTED_PROBLEM_COUNT: Record<TargetSubject, number> = {
국어: 45,
영어: 45,
한국사: 20,
'생활과 윤리': 20,
};
interface PdfFile {
year: number;
subject: TargetSubject;
@@ -61,10 +52,9 @@ interface PdfFile {
interface ScanResult {
files: PdfFile[];
audioMp3s: Map<string, string[]>; // key: `${year}/영어`, value: mp3 상대경로 배열
audioMp3s: Map<string, string[]>;
}
// ─── 1. 스캐너 ────────────────────────────────────────────────────────
function scanKiceData(filter: { year?: number; subject?: string }): ScanResult {
const files: PdfFile[] = [];
const audioMp3s = new Map<string, string[]>();
@@ -76,41 +66,20 @@ function scanKiceData(filter: { year?: number; subject?: string }): ScanResult {
if (filter.subject && filter.subject !== subject) continue;
if (subject === '생활과 윤리') {
// 사탐 1번 과목만 선별
const problemPath = path.join(
DATA_DIR,
String(year),
'사회탐구/사회탐구영역_문제지',
);
const answerPath = path.join(
DATA_DIR,
String(year),
'사회탐구/사회탐구영역_정답표',
);
const problemPath = path.join(DATA_DIR, String(year), '사회탐구/사회탐구영역_문제지');
const answerPath = path.join(DATA_DIR, String(year), '사회탐구/사회탐구영역_정답표');
const problemFile = findFile(problemPath, /01 생활과 윤리_(문제|문제지)\.pdf$/);
const answerFile = findFile(answerPath, /01 생활과 윤리_(정답|정답표)\.pdf$/);
if (problemFile) {
files.push({
year,
subject,
role: 'problems',
filepath: problemFile,
sizeKB: Math.round(fs.statSync(problemFile).size / 1024),
});
files.push(buildPdfFile(year, subject, 'problems', problemFile));
}
if (answerFile) {
files.push({
year,
subject,
role: 'answer',
filepath: answerFile,
sizeKB: Math.round(fs.statSync(answerFile).size / 1024),
});
files.push(buildPdfFile(year, subject, 'answer', answerFile));
}
continue;
}
// 국어 / 영어 / 한국사
const subjectDir = path.join(DATA_DIR, String(year), subject);
if (!fs.existsSync(subjectDir)) continue;
@@ -122,34 +91,15 @@ function scanKiceData(filter: { year?: number; subject?: string }): ScanResult {
const scriptFile = findFile(subjectDir, /_듣기평가대본\.pdf$/);
if (problemFile) {
files.push({
year,
subject,
role: 'problems',
filepath: problemFile,
sizeKB: Math.round(fs.statSync(problemFile).size / 1024),
});
files.push(buildPdfFile(year, subject, 'problems', problemFile));
}
if (answerFile) {
files.push({
year,
subject,
role: 'answer',
filepath: answerFile,
sizeKB: Math.round(fs.statSync(answerFile).size / 1024),
});
files.push(buildPdfFile(year, subject, 'answer', answerFile));
}
if (scriptFile) {
files.push({
year,
subject,
role: 'audio-script',
filepath: scriptFile,
sizeKB: Math.round(fs.statSync(scriptFile).size / 1024),
});
files.push(buildPdfFile(year, subject, 'audio-script', scriptFile));
}
// 영어 듣기 mp3 경로 수집 (상대경로)
if (subject === '영어') {
const audioDir1 = path.join(subjectDir, '영어영역_듣기평가음원');
const audioDir2 = path.join(subjectDir, '영어영역듣기평가음원');
@@ -158,12 +108,13 @@ function scanKiceData(filter: { year?: number; subject?: string }): ScanResult {
: fs.existsSync(audioDir2)
? audioDir2
: null;
if (audioDir) {
const mp3s = fs
.readdirSync(audioDir)
.filter((f) => f.endsWith('.mp3'))
.filter((entry) => entry.endsWith('.mp3'))
.sort()
.map((f) => path.relative(DATA_DIR, path.join(audioDir, f)));
.map((entry) => path.relative(DATA_DIR, path.join(audioDir, entry)));
if (mp3s.length > 0) {
audioMp3s.set(`${year}/영어`, mp3s);
}
@@ -175,279 +126,64 @@ function scanKiceData(filter: { year?: number; subject?: string }): ScanResult {
return { files, audioMp3s };
}
function buildPdfFile(
year: number,
subject: TargetSubject,
role: PdfFile['role'],
filepath: string,
): PdfFile {
return {
year,
subject,
role,
filepath,
sizeKB: Math.round(fs.statSync(filepath).size / 1024),
};
}
function findFile(dir: string, pattern: RegExp): string | null {
if (!fs.existsSync(dir)) return null;
const entries = fs.readdirSync(dir);
const match = entries.find((e) => pattern.test(e));
const match = fs.readdirSync(dir).find((entry) => pattern.test(entry));
return match ? path.join(dir, match) : null;
}
// ─── 2. pdftotext 래퍼 ──────────────────────────────────────────────
/**
* pdftotext 호출.
*
* mode:
* 'layout' = 시각적 레이아웃 보존 — 정답표 같은 표 형식에 적합
* 'raw' = physical reading order — 2단 컬럼 문제지에 적합 (좌→우 아님)
*/
function extractText(pdfPath: string, mode: 'layout' | 'raw' = 'layout'): string {
const flag = mode === 'raw' ? '-raw' : '-layout';
const cmd = `pdftotext ${flag} -enc UTF-8 "${pdfPath}" -`;
return execSync(cmd, { encoding: 'utf-8', maxBuffer: 20 * 1024 * 1024 });
}
// ─── 3. 정답표 파서 ──────────────────────────────────────────────────
const CIRCLED: Record<string, number> = {
'①': 1,
'②': 2,
'③': 3,
'④': 4,
'⑤': 5,
};
interface AnswerEntry {
number: number;
answerNumber: number;
}
function parseAnswerTable(text: string): AnswerEntry[] {
// 홀수형 섹션만 사용 (짝수형은 무시 — 문항 번호는 같고 선택지 순서만 다름)
const oddOnly = splitBeforeEvenForm(text);
const seen = new Map<number, number>();
// 정규식: 숫자 (번) + (공백) + 원문자 ①②③④⑤
// "1 ③ 2" 또는 "1 ③ 2" 또는 "1③218" 같은 패턴 모두.
const re = /(\d{1,2})\s*[번]?\s*([①②③④⑤])/g;
let m: RegExpExecArray | null;
while ((m = re.exec(oddOnly)) !== null) {
const number = Number(m[1]);
const answer = CIRCLED[m[2]];
if (number >= 1 && number <= 45 && !seen.has(number)) {
seen.set(number, answer);
}
}
const entries: AnswerEntry[] = [];
for (const [number, answerNumber] of Array.from(seen.entries()).sort((a, b) => a[0] - b[0])) {
entries.push({ number, answerNumber });
}
return entries;
}
/** 정답표 텍스트에서 짝수형 섹션 제외하고 홀수형만 반환 */
function splitBeforeEvenForm(text: string): string {
// 짝수형 header 패턴: "( 짝수 ) 형" 또는 "( 짝 수 )" 또는 "짝수형"
const evenIdx = text.search(/\(\s*짝수\s*\)\s*형|짝수형/);
return evenIdx === -1 ? text : text.slice(0, evenIdx);
}
// ─── 4. 문제지 파서 ──────────────────────────────────────────────────
interface ParsedPassage {
startNumber: number;
endNumber: number;
bodyText: string;
}
interface ParsedProblem {
number: number;
bodyText: string;
choices: Record<string, string>;
passageStart?: number; // 연결된 passage 의 startNumber
needsReview: boolean;
reviewReason?: string;
}
interface ParseProblemResult {
passages: ParsedPassage[];
problems: ParsedProblem[];
}
/**
* 문제지 PDF 텍스트에서 문항과 공통 지문을 분리.
*
* 평가원 포맷 기본 가정:
* - "1." 로 시작하는 라인이 문항 시작
* - 선택지는 ①②③④⑤ 5개
* - "[1~3]" 같은 표기가 공통 지문 헤더
* - 지문 본문은 [1~3] 바로 앞이 아니라 바로 **뒤** 몇 줄 ("다음 글을 읽고 물음에 답하시오")
*/
function parseProblemPaper(text: string): ParseProblemResult {
// 홀수형만 사용 — 짝수형은 같은 문제의 선택지 순서 변형
text = splitBeforeEvenForm(text);
// 공통 지문 범위 탐지: [1~3], [ 1 ~ 3 ] 등
const passageRanges: Array<{ start: number; end: number; matchIdx: number }> = [];
const passageRe = /\[\s*(\d+)\s*[~～－-]\s*(\d+)\s*\]/g;
let pm: RegExpExecArray | null;
while ((pm = passageRe.exec(text)) !== null) {
passageRanges.push({
start: Number(pm[1]),
end: Number(pm[2]),
matchIdx: pm.index,
});
}
// 문항 분할
// 문항 시작 정규식: 줄 시작 또는 공백 뒤 "숫자." + 공백
// 평가원 PDF 에선 번호가 라인 앞 padding 이 있음
const problemRe = /(?:^|\n)[ \t]*(\d{1,2})\.\s+/g;
const problemStarts: Array<{ number: number; idx: number }> = [];
let prm: RegExpExecArray | null;
while ((prm = problemRe.exec(text)) !== null) {
const num = Number(prm[1]);
if (num >= 1 && num <= 45) {
problemStarts.push({ number: num, idx: prm.index });
}
}
// 중복 제거 + 번호 단조 증가 유지 (같은 번호 여러 매치 중 첫 번째만)
const uniqueStarts: Array<{ number: number; idx: number }> = [];
const seenNums = new Set<number>();
for (const s of problemStarts) {
if (seenNums.has(s.number)) continue;
if (uniqueStarts.length > 0 && s.number <= uniqueStarts[uniqueStarts.length - 1].number) {
continue; // 번호가 뒤로 가면 무시
}
uniqueStarts.push(s);
seenNums.add(s.number);
}
// passage bodyText 추출: 범위 start 직전에서 해당 번호의 problem start 까지 사이
// 대부분 "[1~3] 다음 글을 읽고 물음에 답하시오.\n\n<본문>\n\n1. 뒤의 본문은 질문."
// 구현: [N~M] 매칭 이후부터 첫 problem N 까지가 passage body
const passages: ParsedPassage[] = [];
for (const range of passageRanges) {
const firstProblem = uniqueStarts.find((p) => p.number === range.start);
if (!firstProblem) continue;
const passageBody = text.slice(range.matchIdx + 10, firstProblem.idx).trim();
// 너무 짧으면 passage 아님
if (passageBody.length < 50) continue;
passages.push({
startNumber: range.start,
endNumber: range.end,
bodyText: normalizeWhitespace(passageBody),
});
}
// 각 문항의 bodyText 와 choices 추출
const problems: ParsedProblem[] = [];
for (let i = 0; i < uniqueStarts.length; i++) {
const current = uniqueStarts[i];
const next = uniqueStarts[i + 1];
const spanStart = current.idx;
const spanEnd = next ? next.idx : text.length;
let span = text.slice(spanStart, spanEnd);
// 문항 번호 prefix 제거 ("1. ")
span = span.replace(/^\s*\d{1,2}\.\s+/, '');
// 선택지 ①②③④⑤ 위치 찾기
const choiceMatches: Array<{ key: string; idx: number }> = [];
for (const [c, n] of Object.entries(CIRCLED)) {
let idx = span.indexOf(c);
while (idx !== -1) {
choiceMatches.push({ key: String(n), idx });
idx = span.indexOf(c, idx + 1);
}
}
// 각 원문자는 여러 번 나올 수 있지만 선택지로 첫 번째 등장만 채택하고 key 1..5 순서 보장
const firstByKey = new Map<string, number>();
for (const m of choiceMatches) {
if (!firstByKey.has(m.key)) firstByKey.set(m.key, m.idx);
}
const orderedKeys = ['1', '2', '3', '4', '5'].filter((k) => firstByKey.has(k));
let bodyText = '';
const choices: Record<string, string> = {};
if (orderedKeys.length === 5) {
// bodyText = 첫 ① 이전
const firstIdx = firstByKey.get('1')!;
bodyText = span.slice(0, firstIdx);
// 각 선택지는 해당 원문자 직후부터 다음 원문자 직전까지
for (let k = 0; k < orderedKeys.length; k++) {
const key = orderedKeys[k];
const start = firstByKey.get(key)! + 1;
const nextKey = orderedKeys[k + 1];
const end = nextKey ? firstByKey.get(nextKey)! : span.length;
choices[key] = normalizeWhitespace(span.slice(start, end));
}
} else {
// 선택지 5개 찾지 못함 — bodyText 에 전체 span 저장, needsReview
bodyText = span;
}
const reasons: string[] = [];
if (bodyText.trim().length < 10) reasons.push('bodyText 10자 미만');
if (Object.keys(choices).length !== 5) reasons.push(`choices ${Object.keys(choices).length}/5`);
for (const [k, v] of Object.entries(choices)) {
if (!v || v.trim().length === 0) reasons.push(`choice ${k} 빈값`);
}
// 연결된 passage 의 start 번호
const linkedPassage = passages.find(
(p) => current.number >= p.startNumber && current.number <= p.endNumber,
);
problems.push({
number: current.number,
bodyText: normalizeWhitespace(bodyText),
choices,
passageStart: linkedPassage?.startNumber,
needsReview: reasons.length > 0,
reviewReason: reasons.join(', ') || undefined,
});
}
return { passages, problems };
}
function normalizeWhitespace(s: string): string {
return s
.replace(/[ \t]+/g, ' ')
.replace(/\n{3,}/g, '\n\n')
.replace(/^\s+|\s+$/g, '');
}
// ─── 5. 제목 생성 ───────────────────────────────────────────────────
function buildProblemSetTitle(year: number, subject: TargetSubject): string {
return `${year}학년도 대학수학능력시험 ${subject}`;
}
// ─── 6. DB import ────────────────────────────────────────────────────
async function importKiceSet(
prisma: PrismaClient,
year: number,
subject: TargetSubject,
files: PdfFile[],
audioMp3s: Map<string, string[]>,
): Promise<{ problems: number; passages: number; needsReview: number }> {
const answerFile = files.find((f) => f.role === 'answer');
const problemFile = files.find((f) => f.role === 'problems');
): Promise<{ problems: number; passages: number; needsReview: number; warnings: string[] }> {
const answerFile = files.find((file) => file.role === 'answer');
const problemFile = files.find((file) => file.role === 'problems');
if (!answerFile || !problemFile) {
console.warn(`⚠ [${year} ${subject}] PDF 누락 (answer=${!!answerFile}, problems=${!!problemFile})`);
return { problems: 0, passages: 0, needsReview: 0 };
if (!problemFile) {
console.warn(`⚠ [${year} ${subject}] 문제지 PDF 누락`);
return { problems: 0, passages: 0, needsReview: 0, warnings: ['missing problem PDF'] };
}
// 파싱
const answerText = extractText(answerFile.filepath, 'layout');
const answers = parseAnswerTable(answerText);
const answerMap = new Map(answers.map((a) => [a.number, a.answerNumber]));
if (answers.length === 0) {
console.warn(
`⚠ [${year} ${subject}] 정답표 추출 실패 (이미지 기반 PDF?) — answerNumber 전부 null`,
);
const parsed = await parseExamPaper({
paperPdfPath: problemFile.filepath,
answerPdfPath: answerFile?.filepath,
format: 'kice',
expectedProblemCount: EXPECTED_PROBLEM_COUNT[subject],
});
const answerMap = new Map(parsed.answers.map((answer) => [answer.number, answer.answerNumber]));
if (answerFile && parsed.answers.length === 0) {
console.warn(`⚠ [${year} ${subject}] 정답표 추출 실패 (이미지 기반 PDF?) — answerNumber 전부 null`);
}
const problemText = extractText(problemFile.filepath, 'raw');
const parsed = parseProblemPaper(problemText);
for (const warning of parsed.warnings) {
console.warn(` ⚠ [${year} ${subject}] ${warning}`);
}
// 영어 과목이면 mp3 경로
const audioUrls = subject === '영어' ? audioMp3s.get(`${year}/영어`) || null : null;
// upsert ProblemSet
const ps = await prisma.problemSet.upsert({
const problemSet = await prisma.problemSet.upsert({
where: {
year_examType_subjectName: {
year,
@@ -470,61 +206,59 @@ async function importKiceSet(
},
});
// Passage upsert — 기존 것 모두 삭제 후 재삽입 (멱등성 + passageStart 매핑 단순화)
await prisma.passage.deleteMany({ where: { problemSetId: ps.id } });
await prisma.passage.deleteMany({ where: { problemSetId: problemSet.id } });
const createdPassages = await Promise.all(
parsed.passages.map((p) =>
parsed.passages.map((passage) =>
prisma.passage.create({
data: {
problemSetId: ps.id,
startNumber: p.startNumber,
endNumber: p.endNumber,
bodyText: p.bodyText,
problemSetId: problemSet.id,
startNumber: passage.startNumber,
endNumber: passage.endNumber,
bodyText: passage.bodyText,
},
}),
),
);
const passageIdByStart = new Map(
createdPassages.map((p) => [p.startNumber, p.id]),
);
const passageIdByStart = new Map(createdPassages.map((passage) => [passage.startNumber, passage.id]));
// Problem upsert
let needsReviewCount = 0;
for (const p of parsed.problems) {
const passageId = p.passageStart
? passageIdByStart.get(p.passageStart) || null
for (const problem of parsed.problems) {
const passageId = problem.passageStart
? passageIdByStart.get(problem.passageStart) || null
: null;
const answerNumber = answerMap.get(p.number) || null;
const nr = p.needsReview || !answerNumber;
if (nr) needsReviewCount++;
const answerNumber = answerMap.get(problem.number) || null;
const needsReview = problem.needsReview || !answerNumber;
if (needsReview) {
needsReviewCount++;
}
await prisma.problem.upsert({
where: {
problemSetId_number: { problemSetId: ps.id, number: p.number },
problemSetId_number: { problemSetId: problemSet.id, number: problem.number },
},
update: {
bodyText: p.bodyText,
choices: p.choices as Prisma.InputJsonValue,
bodyText: problem.bodyText,
choices: problem.choices as Prisma.InputJsonValue,
answerNumber,
passageId,
needsReview: nr,
needsReview,
},
create: {
problemSetId: ps.id,
number: p.number,
title: `${buildProblemSetTitle(year, subject)} ${p.number}번`,
difficulty: 0.5, // 기본값 — Phase 8 에서 실제 정답률로 보정
bodyText: p.bodyText,
choices: p.choices as Prisma.InputJsonValue,
problemSetId: problemSet.id,
number: problem.number,
title: `${buildProblemSetTitle(year, subject)} ${problem.number}번`,
difficulty: 0.5,
bodyText: problem.bodyText,
choices: problem.choices as Prisma.InputJsonValue,
answerNumber,
passageId,
needsReview: nr,
needsReview,
},
});
if (p.needsReview) {
if (problem.needsReview) {
console.warn(
` ⚠ [${year} ${subject}] Problem ${p.number} needsReview: ${p.reviewReason}`,
` ⚠ [${year} ${subject}] Problem ${problem.number} needsReview: ${problem.needsReviewReasons.join(', ')}`,
);
}
}
@@ -533,21 +267,68 @@ async function importKiceSet(
problems: parsed.problems.length,
passages: parsed.passages.length,
needsReview: needsReviewCount,
warnings: parsed.warnings,
};
}
// ─── 7. main ─────────────────────────────────────────────────────────
async function main() {
const opts = parseArgs();
console.log('🏫 KICE import 시작', opts);
async function inspectParseResult(
year: number,
subject: TargetSubject,
files: PdfFile[],
mode: 'answer' | 'problems',
sample?: number,
) {
const answerFile = files.find((file) => file.role === 'answer');
const problemFile = files.find((file) => file.role === 'problems');
const scan = scanKiceData({ year: opts.year, subject: opts.subject });
if (!problemFile) {
console.warn(`⚠ [${year} ${subject}] 문제지 PDF 누락`);
return;
}
const parsed = await parseExamPaper({
paperPdfPath: problemFile.filepath,
answerPdfPath: answerFile?.filepath,
format: 'kice',
expectedProblemCount: EXPECTED_PROBLEM_COUNT[subject],
});
if (mode === 'answer') {
console.log(` ${year} / ${subject}: ${parsed.answers.length} 문항`);
for (const answer of parsed.answers.slice(0, sample ? 5 : 0)) {
console.log(` ${answer.number}번 → ${answer.answerNumber}`);
}
return;
}
console.log(` ${year} / ${subject}: ${parsed.problems.length} problems, ${parsed.passages.length} passages`);
const badProblems = parsed.problems.filter((problem) => problem.needsReview);
if (badProblems.length > 0) {
console.log(` ⚠ needsReview: ${badProblems.length}`);
for (const problem of badProblems.slice(0, 5)) {
console.log(` #${problem.number}: ${problem.needsReviewReasons.join(', ')}`);
}
}
if (sample) {
const first = parsed.problems[0];
if (first) {
console.log(` sample #${first.number}: "${first.bodyText.slice(0, 80)}..."`);
console.log(` choices: ${Object.keys(first.choices).length} keys`);
}
}
}
async function main() {
const options = parseArgs();
console.log('🏫 KICE import 시작', options);
const scan = scanKiceData({ year: options.year, subject: options.subject });
console.log(`스캔: ${scan.files.length} files, ${scan.audioMp3s.size} audio sets`);
if (opts.dryRun) {
if (options.dryRun) {
console.log('\n=== 파일 목록 (dry-run) ===');
for (const f of scan.files) {
console.log(` ${f.year} / ${f.subject} / ${f.role.padEnd(14)} — ${f.sizeKB}KB — ${path.basename(f.filepath)}`);
for (const file of scan.files) {
console.log(` ${file.year} / ${file.subject} / ${file.role.padEnd(14)} — ${file.sizeKB}KB — ${path.basename(file.filepath)}`);
}
for (const [key, mp3s] of scan.audioMp3s) {
console.log(` audio ${key}: ${mp3s.length} mp3 files`);
@@ -555,59 +336,33 @@ async function main() {
return;
}
if (opts.only === 'answer') {
console.log('\n=== 정답표 파서 테스트 ===');
for (const f of scan.files.filter((f) => f.role === 'answer')) {
const text = extractText(f.filepath);
const answers = parseAnswerTable(text);
console.log(` ${f.year} / ${f.subject}: ${answers.length} 문항`);
if (opts.sample) {
for (const a of answers.slice(0, 5)) {
console.log(` ${a.number}번 → ${a.answerNumber}`);
}
if (options.only === 'answer' || options.only === 'problems') {
console.log(options.only === 'answer' ? '\n=== 정답표 파서 테스트 ===' : '\n=== 문제지 파서 테스트 ===');
for (const year of TARGET_YEARS) {
if (options.year && options.year !== year) continue;
for (const subject of TARGET_SUBJECTS) {
if (options.subject && options.subject !== subject) continue;
const files = scan.files.filter((file) => file.year === year && file.subject === subject);
if (files.length === 0) continue;
await inspectParseResult(year, subject, files, options.only, options.sample);
}
}
return;
}
if (opts.only === 'problems') {
console.log('\n=== 문제지 파서 테스트 ===');
for (const f of scan.files.filter((f) => f.role === 'problems')) {
const text = extractText(f.filepath, 'raw');
const parsed = parseProblemPaper(text);
console.log(
` ${f.year} / ${f.subject}: ${parsed.problems.length} problems, ${parsed.passages.length} passages`,
);
const badProblems = parsed.problems.filter((p) => p.needsReview);
if (badProblems.length > 0) {
console.log(` ⚠ needsReview: ${badProblems.length}`);
for (const p of badProblems.slice(0, 5)) {
console.log(` #${p.number}: ${p.reviewReason}`);
}
}
if (opts.sample) {
const sample = parsed.problems[0];
if (sample) {
console.log(` sample #${sample.number}: "${sample.bodyText.slice(0, 80)}..."`);
console.log(` choices: ${Object.keys(sample.choices).length} keys`);
}
}
}
return;
}
// 실제 DB import
const prisma = new PrismaClient();
try {
let totalProblems = 0;
let totalPassages = 0;
let totalNeedsReview = 0;
for (const year of TARGET_YEARS) {
if (opts.year && opts.year !== year) continue;
if (options.year && options.year !== year) continue;
for (const subject of TARGET_SUBJECTS) {
if (opts.subject && opts.subject !== subject) continue;
const files = scan.files.filter((f) => f.year === year && f.subject === subject);
if (options.subject && options.subject !== subject) continue;
const files = scan.files.filter((file) => file.year === year && file.subject === subject);
if (files.length === 0) continue;
const result = await importKiceSet(prisma, year, subject, files, scan.audioMp3s);
console.log(
`✓ ${year} ${subject}: ${result.problems} problems, ${result.passages} passages, ${result.needsReview} needsReview`,
@@ -617,15 +372,14 @@ async function main() {
totalNeedsReview += result.needsReview;
}
}
console.log(
`\n🎯 합계: ${totalProblems} problems, ${totalPassages} passages, ${totalNeedsReview} needsReview`,
);
console.log(`\n🎯 합계: ${totalProblems} problems, ${totalPassages} passages, ${totalNeedsReview} needsReview`);
} finally {
await prisma.$disconnect();
}
}
main().catch((e) => {
console.error(e);
main().catch((error) => {
console.error(error);
process.exit(1);
});

21
backend/src/problem-sets/parsing/README.md Normal file
View File

@@ -0,0 +1,21 @@
# Problem Set Parsing
`parseExamPaper()` is the Prisma-free entry point for converting an exam paper PDF and an optional answer-table PDF into structured `problems`, `passages`, `answers`, and parser `warnings`.
## Files
- `index.ts`: public API and strategy resolution
- `types.ts`: parser contracts
- `extract-text.ts`: `pdftotext` wrapper
- `strip-page-chrome.ts`: full-line header/footer stripping
- `parse-answer-table.ts`: answer key extraction
- `parse-problem-paper.ts`: passage and problem segmentation
- `strategies/kice.ts`: KICE-specific page chrome and answer parsing rules
## Adding a New Format
1. Add a new strategy file under `strategies/`.
2. Define the full-line page chrome patterns that should be removed.
3. Set `maxProblemNumber` and any format-specific `evenFormSplitPattern` or answer regex.
4. Register the strategy in `resolveParseStrategy()` in `index.ts`.
5. Reuse `parseExamPaper()` unless the new format needs a different passage or choice grammar.

9
backend/src/problem-sets/parsing/extract-text.ts Normal file
View File

@@ -0,0 +1,9 @@
import { execFileSync } from 'child_process';
export function extractText(pdfPath: string, mode: 'layout' | 'raw' = 'layout'): string {
const flag = mode === 'raw' ? '-raw' : '-layout';
return execFileSync('pdftotext', [flag, '-enc', 'UTF-8', pdfPath, '-'], {
encoding: 'utf-8',
maxBuffer: 20 * 1024 * 1024,
});
}

71
backend/src/problem-sets/parsing/index.ts Normal file
View File

@@ -0,0 +1,71 @@
import { extractText } from './extract-text';
import { parseAnswerTable } from './parse-answer-table';
import { parseProblemPaper } from './parse-problem-paper';
import { kiceStrategy } from './strategies/kice';
import { ParseOptions, ParseResult, PageStripStrategy } from './types';
const genericStrategy: PageStripStrategy = {
name: 'generic',
format: 'generic',
maxProblemNumber: 200,
chromeLinePatterns: [/^\s*\d+\s*$/],
leakedChromePatterns: [],
};
const ebsStrategy: PageStripStrategy = {
...genericStrategy,
name: 'ebs',
format: 'ebs',
};
export async function parseExamPaper(options: ParseOptions): Promise<ParseResult> {
const strategy = resolveParseStrategy(options.format);
const warnings: string[] = [];
const paperText = extractText(options.paperPdfPath, 'raw');
const problemResult = parseProblemPaper(paperText, strategy);
warnings.push(...problemResult.warnings);
let answers: ParseResult['answers'] = [];
if (options.answerPdfPath) {
const answerText = extractText(options.answerPdfPath, 'layout');
const answerResult = parseAnswerTable(answerText, strategy);
answers = answerResult.answers;
warnings.push(...answerResult.warnings);
}
if (
typeof options.expectedProblemCount === 'number' &&
problemResult.problems.length !== options.expectedProblemCount
) {
warnings.push(
`expected ${options.expectedProblemCount} problems, parsed ${problemResult.problems.length}`,
);
}
return {
problems: problemResult.problems,
passages: problemResult.passages,
answers,
warnings: Array.from(new Set(warnings)),
};
}
export function resolveParseStrategy(
format: ParseOptions['format'] = 'generic',
): PageStripStrategy {
switch (format) {
case 'kice':
return kiceStrategy;
case 'ebs':
return ebsStrategy;
case 'generic':
default:
return genericStrategy;
}
}
export * from './types';
export { extractText } from './extract-text';
export { parseAnswerTable } from './parse-answer-table';
export { parseProblemPaper } from './parse-problem-paper';
export { stripPageChrome } from './strip-page-chrome';

53
backend/src/problem-sets/parsing/parse-answer-table.ts Normal file
View File

@@ -0,0 +1,53 @@
import { ParseAnswerTableResult, PageStripStrategy } from './types';
const CIRCLED_TO_NUMBER: Record<string, 1 | 2 | 3 | 4 | 5> = {
'①': 1,
'②': 2,
'③': 3,
'④': 4,
'⑤': 5,
};
export function parseAnswerTable(
text: string,
strategy: PageStripStrategy,
): ParseAnswerTableResult {
const warnings: string[] = [];
const scoped = splitBeforeEvenForm(text, strategy);
const pattern = strategy.answerNumberPattern ?? /(\d{1,2})\s*[번]?\s*([①②③④⑤])/g;
const seen = new Map<number, 1 | 2 | 3 | 4 | 5>();
pattern.lastIndex = 0;
let match: RegExpExecArray | null;
while ((match = pattern.exec(scoped)) !== null) {
const number = Number(match[1]);
const answerNumber = CIRCLED_TO_NUMBER[match[2]];
if (
number >= 1 &&
number <= strategy.maxProblemNumber &&
answerNumber &&
!seen.has(number)
) {
seen.set(number, answerNumber);
}
}
const answers = Array.from(seen.entries())
.sort((left, right) => left[0] - right[0])
.map(([number, answerNumber]) => ({ number, answerNumber }));
if (answers.length === 0) {
warnings.push('answer table extraction returned 0 answers');
}
return { answers, warnings };
}
function splitBeforeEvenForm(text: string, strategy: PageStripStrategy): string {
if (!strategy.evenFormSplitPattern) {
return text;
}
const matchIndex = text.search(strategy.evenFormSplitPattern);
return matchIndex === -1 ? text : text.slice(0, matchIndex);
}

307
backend/src/problem-sets/parsing/parse-problem-paper.ts Normal file
View File

@@ -0,0 +1,307 @@
import {
ParsedPassage,
ParsedProblem,
ParseProblemPaperResult,
PageStripStrategy,
} from './types';
import { stripPageChrome } from './strip-page-chrome';
const PROBLEM_START_RE = /^(\d{1,2})\.\s+/gm;
const PASSAGE_HEADER_RE = /\[\s*(\d+)\s*[~∼～-]\s*(\d+)\s*\][^\n]*/gm;
const CHOICE_MARKERS = ['①', '②', '③', '④', '⑤'] as const;
type ProblemChoiceKey = '1' | '2' | '3' | '4' | '5';
interface CandidateProblemStart {
number: number;
idx: number;
}
interface PassageMarker {
startNumber: number;
endNumber: number;
idx: number;
}
export function parseProblemPaper(
rawText: string,
strategy: PageStripStrategy,
): ParseProblemPaperResult {
const warnings: string[] = [];
const scopedText = splitBeforeEvenForm(rawText, strategy);
const strippedText = stripPageChrome(scopedText, strategy);
const passageMarkers = collectPassageMarkers(strippedText);
const problemStarts = collectProblemStarts(strippedText, strategy, warnings);
const passages = buildPassages(strippedText, passageMarkers, problemStarts);
const problems = buildProblems(strippedText, passageMarkers, problemStarts, passages, strategy);
if (problemStarts.length === 0) {
warnings.push('problem parser found 0 problem starts');
}
return { problems, passages, warnings };
}
function splitBeforeEvenForm(text: string, strategy: PageStripStrategy): string {
if (!strategy.evenFormSplitPattern) {
return text;
}
const matchIndex = text.search(strategy.evenFormSplitPattern);
return matchIndex === -1 ? text : text.slice(0, matchIndex);
}
function collectPassageMarkers(text: string): PassageMarker[] {
const markers: PassageMarker[] = [];
PASSAGE_HEADER_RE.lastIndex = 0;
let match: RegExpExecArray | null;
while ((match = PASSAGE_HEADER_RE.exec(text)) !== null) {
markers.push({
startNumber: Number(match[1]),
endNumber: Number(match[2]),
idx: match.index,
});
}
return markers;
}
function collectProblemStarts(
text: string,
strategy: PageStripStrategy,
warnings: string[],
): CandidateProblemStart[] {
const rawStarts: CandidateProblemStart[] = [];
PROBLEM_START_RE.lastIndex = 0;
let match: RegExpExecArray | null;
while ((match = PROBLEM_START_RE.exec(text)) !== null) {
const number = Number(match[1]);
if (number >= 1 && number <= strategy.maxProblemNumber) {
rawStarts.push({ number, idx: match.index });
}
}
const accepted: CandidateProblemStart[] = [];
let lastAcceptedNumber = 0;
for (let index = 0; index < rawStarts.length; index++) {
const current = rawStarts[index];
const nextRaw = rawStarts[index + 1];
if (current.number < lastAcceptedNumber) {
continue;
}
if (current.number === lastAcceptedNumber) {
warnings.push(`duplicate problem start ignored: ${current.number}`);
continue;
}
const probeEnd = Math.min(nextRaw?.idx ?? text.length, current.idx + 1500);
const probeText = text.slice(current.idx, probeEnd);
if (/[①②③④⑤]/.test(probeText)) {
accepted.push(current);
lastAcceptedNumber = current.number;
continue;
}
if (looksLikeChoiceLessProblem(text, current, nextRaw)) {
warnings.push(`problem ${current.number} accepted without inline choices`);
accepted.push(current);
lastAcceptedNumber = current.number;
continue;
}
}
return accepted;
}
function looksLikeChoiceLessProblem(
text: string,
current: CandidateProblemStart,
nextRaw: CandidateProblemStart | undefined,
): boolean {
if (!nextRaw || nextRaw.number !== current.number + 1) {
return false;
}
const stemBlock = text.slice(current.idx, nextRaw.idx).trim();
if (stemBlock.length > 300) {
return false;
}
return /(고르시오|답하시오)\.?\s*$/.test(stemBlock) || /것은\?\s*$/.test(stemBlock);
}
function buildPassages(
text: string,
passageMarkers: PassageMarker[],
problemStarts: CandidateProblemStart[],
): ParsedPassage[] {
const passages: ParsedPassage[] = [];
for (const marker of passageMarkers) {
const nextProblem = problemStarts.find((problem) => problem.idx > marker.idx);
if (!nextProblem) {
continue;
}
const bodyText = normalizeBodyWhitespace(text.slice(marker.idx, nextProblem.idx));
if (!bodyText) {
continue;
}
passages.push({
startNumber: marker.startNumber,
endNumber: marker.endNumber,
bodyText,
});
}
return passages;
}
function buildProblems(
text: string,
passageMarkers: PassageMarker[],
problemStarts: CandidateProblemStart[],
passages: ParsedPassage[],
strategy: PageStripStrategy,
): ParsedProblem[] {
const problems: ParsedProblem[] = [];
for (let index = 0; index < problemStarts.length; index++) {
const current = problemStarts[index];
const nextProblem = problemStarts[index + 1];
const nextPassage = passageMarkers.find((marker) => marker.idx > current.idx);
const spanEnd = Math.min(nextProblem?.idx ?? text.length, nextPassage?.idx ?? text.length);
const span = text
.slice(current.idx, spanEnd)
.replace(/^\s*\d{1,2}\.\s+/, '')
.trim();
const split = splitProblemChoices(span);
const linkedPassage = passages.find(
(passage) =>
current.number >= passage.startNumber && current.number <= passage.endNumber,
);
const needsReviewReasons = buildNeedsReviewReasons(
split.bodyText,
split.choices,
split.foundMarkers,
current.number,
problemStarts[index - 1]?.number,
strategy,
);
problems.push({
number: current.number,
bodyText: split.bodyText,
choices: split.choices,
passageStart: linkedPassage?.startNumber,
passageEnd: linkedPassage?.endNumber,
needsReview: needsReviewReasons.length > 0,
needsReviewReasons,
});
}
return problems;
}
function splitProblemChoices(span: string): {
bodyText: string;
choices: Record<ProblemChoiceKey, string>;
foundMarkers: number;
} {
const choices: Record<ProblemChoiceKey, string> = {
'1': '',
'2': '',
'3': '',
'4': '',
'5': '',
};
const markerPositions: Array<{ marker: typeof CHOICE_MARKERS[number]; idx: number }> = [];
let cursor = 0;
for (const marker of CHOICE_MARKERS) {
const idx = span.indexOf(marker, cursor);
if (idx === -1) {
break;
}
markerPositions.push({ marker, idx });
cursor = idx + marker.length;
}
const bodyBoundary = markerPositions[0]?.idx ?? span.length;
const bodyText = normalizeBodyWhitespace(span.slice(0, bodyBoundary));
for (let index = 0; index < markerPositions.length; index++) {
const current = markerPositions[index];
const next = markerPositions[index + 1];
const choiceKey = String(index + 1) as ProblemChoiceKey;
const choiceStart = current.idx + current.marker.length;
const choiceEnd = next?.idx ?? span.length;
choices[choiceKey] = normalizeChoiceWhitespace(span.slice(choiceStart, choiceEnd));
}
return {
bodyText,
choices,
foundMarkers: markerPositions.length,
};
}
function buildNeedsReviewReasons(
bodyText: string,
choices: Record<ProblemChoiceKey, string>,
foundMarkers: number,
currentNumber: number,
previousNumber: number | undefined,
strategy: PageStripStrategy,
): string[] {
const reasons: string[] = [];
if (bodyText.length < 10) {
reasons.push('body length < 10');
}
if (foundMarkers !== 5) {
reasons.push(`choice markers found: ${foundMarkers}`);
}
for (const choiceKey of ['1', '2', '3', '4', '5'] as const) {
const value = choices[choiceKey];
if (!value) {
reasons.push(`choice ${choiceKey} empty`);
continue;
}
if (value.length < 3) {
reasons.push(`choice ${choiceKey} too short`);
}
}
if (previousNumber !== undefined && currentNumber !== previousNumber + 1) {
reasons.push(`problem number gap after ${previousNumber}`);
}
const leakedText = [bodyText, ...Object.values(choices)].join('\n');
if (strategy.leakedChromePatterns.some((pattern) => pattern.test(leakedText))) {
reasons.push('page chrome leak detected');
}
return Array.from(new Set(reasons));
}
function normalizeBodyWhitespace(value: string): string {
return value
.replace(/[ \t]+/g, ' ')
.replace(/\n[ \t]+/g, '\n')
.replace(/[ \t]+\n/g, '\n')
.replace(/\n{3,}/g, '\n\n')
.trim();
}
function normalizeChoiceWhitespace(value: string): string {
return value.replace(/\s+/g, ' ').trim();
}

51
backend/src/problem-sets/parsing/strategies/kice.ts Normal file
View File

@@ -0,0 +1,51 @@
import { PageStripStrategy } from '../types';
export const kiceStrategy: PageStripStrategy = {
name: 'kice',
format: 'kice',
maxProblemNumber: 45,
chromeLinePatterns: [
/^\s*\d+\s*$/,
/^\s*\d+\s+홀수형\s*$/,
/^\s*\d+\s+짝수형\s*$/,
/^\s*홀수형\s+\d+\s*$/,
/^\s*짝수형\s+\d+\s*$/,
/^\s*홀수형\s*$/,
/^\s*짝수형\s*$/,
/^\s*제\s*\d+\s*교시\s*.*$/,
/^\s*\d{4}학년도.*대학수학능력시험.*문제지.*$/,
/^\s*이 문제지에 관한 저작권은.*$/,
/^\s*(?:국어|영어|수학|한국사)\s+영역\s*$/,
/^\s*(?:국어|영어|수학|한국사)\s+영역\s+\d+\s*$/,
/^\s*(?:국어|영어|수학|한국사)\s+영역\s+홀수형\s*$/,
/^\s*(?:국어|영어|수학|한국사)\s+영역\s+짝수형\s*$/,
/^\s*홀수형\s+(?:국어|영어|수학|한국사)\s+영역\s+\d+\s*$/,
/^\s*짝수형\s+(?:국어|영어|수학|한국사)\s+영역\s+\d+\s*$/,
/^\s*사회탐구\s+영역\s*$/,
/^\s*사회탐구\s+영역\s+\d+\s*$/,
/^\s*사회탐구\s+영역\s*\(.+\)\s*$/,
/^\s*\(.+\)\s+\d+\s*$/,
/^\s*\(.+\)\s*$/,
/^\s*\(.+\)\s+홀수형\s*$/,
/^\s*\(.+\)\s+짝수형\s*$/,
/^\s*\d+\s+\(.+\)\s+홀수형\s*$/,
/^\s*\d+\s+\(.+\)\s+짝수형\s*$/,
/^\s*홀수형\s+\(.+\)\s+\d+\s*$/,
/^\s*짝수형\s+\(.+\)\s+\d+\s*$/,
/^\s*생활과\s+윤리\s*$/,
/^\s*성명\s+수험\s+번호.*$/,
/^\s*\*\s*확인\s*사항\s*$/,
/^\s*◦\s*답안지의 해당란에 필요한 내용을 정확히 기입\(표기\)했는지 확인\s*$/,
/^\s*◦\s*이어서,\s*｢선택과목\(.+\)｣ 문제가 제시되오니,\s*자신이\s*$/,
/^\s*선택한 과목인지 확인하시오\.\s*$/,
],
leakedChromePatterns: [
/대학수학능력시험\s+문제지/,
/이 문제지에 관한 저작권은/,
/(?:^|\s)홀수형(?:\s|$)/,
/(?:^|\s)짝수형(?:\s|$)/,
/제\s*\d+\s*교시/,
],
evenFormSplitPattern: /\(\s*짝수\s*\)\s*형|짝수형/,
answerNumberPattern: /(\d{1,2})\s*[번]?\s*([①②③④⑤])/g,
};

36
backend/src/problem-sets/parsing/strip-page-chrome.ts Normal file
View File

@@ -0,0 +1,36 @@
import { PageStripStrategy } from './types';
export function stripPageChrome(rawText: string, strategy: PageStripStrategy): string {
const normalized = rawText.replace(/\r\n/g, '\n').replace(/\f/g, '\n');
const lines = normalized.split('\n');
const keptLines: string[] = [];
let stripWrappedContinuation = false;
for (const line of lines) {
const shouldStrip = strategy.chromeLinePatterns.some((pattern) => pattern.test(line));
const continuationLine =
stripWrappedContinuation &&
/^\s*(?:하시오\.|선택한 과목인지 확인하시오\.)\s*$/.test(line);
if (shouldStrip) {
stripWrappedContinuation = true;
continue;
}
if (continuationLine) {
continue;
}
stripWrappedContinuation = false;
keptLines.push(line);
}
return collapseBlankLines(keptLines.join('\n')).trim();
}
function collapseBlankLines(text: string): string {
return text
.replace(/[ \t]+\n/g, '\n')
.replace(/\n[ \t]+/g, '\n')
.replace(/\n{3,}/g, '\n\n');
}

54
backend/src/problem-sets/parsing/types.ts Normal file
View File

@@ -0,0 +1,54 @@
export interface ParseOptions {
paperPdfPath: string;
answerPdfPath?: string;
format?: 'kice' | 'ebs' | 'generic';
expectedProblemCount?: number;
}
export interface ParsedProblem {
number: number;
bodyText: string;
choices: Record<'1' | '2' | '3' | '4' | '5', string>;
passageStart?: number;
passageEnd?: number;
needsReview: boolean;
needsReviewReasons: string[];
}
export interface ParsedPassage {
startNumber: number;
endNumber: number;
bodyText: string;
}
export interface ParseResult {
problems: ParsedProblem[];
passages: ParsedPassage[];
answers: Array<{ number: number; answerNumber: 1 | 2 | 3 | 4 | 5 }>;
warnings: string[];
}
export interface ParseWarningContext {
warnings: string[];
}
export interface PageStripStrategy {
name: string;
format: ParseOptions['format'];
maxProblemNumber: number;
chromeLinePatterns: RegExp[];
leakedChromePatterns: RegExp[];
evenFormSplitPattern?: RegExp;
answerNumberPattern?: RegExp;
}
export interface ParseAnswerTableResult {
answers: Array<{ number: number; answerNumber: 1 | 2 | 3 | 4 | 5 }>;
warnings: string[];
}
export interface ParseProblemPaperResult {
problems: ParsedProblem[];
passages: ParsedPassage[];
warnings: string[];
}