@@ -1,20 +1,82 @@
import * as fs from 'fs' ;
import * as path from 'path' ;
import { execFileSync } from 'child_process' ;
import { parseAnswerTable } from './parse-answer-table' ;
import { parseProblemPaper } from './parse-problem-paper' ;
import { extractText } from './extract-text' ;
import { ocrAnswerTable , renderPdfPageToPng } from './ocr-fallback' ;
import { ocrAnswerTable } from './ocr-fallback' ;
import { parseProblemPaper } from './parse-problem-paper' ;
import { kiceMathStrategy } from './strategies/math' ;
import { ImageBasedParseOptions , ParseResult , ParsedProblem } from './types ' ;
import { kiceStrategy } from './strategies/kice ' ;
import {
ImageBasedParseOptions ,
PageStripStrategy ,
ParseResult ,
ParsedProblem ,
} from './types' ;
const PAGE_PROBLEM_RE = /^(\d{1,2})\.\s+/gm ;
const PROBLEM_WORD_RE = /^\d{1,2}\.$/ ;
const PASSAGE_WORD_RE = /^\[\s*(\d+)\s*[~∼ ~ -]\s*(\d+)\s*\]$/ ;
const DEFAULT_DPI = 300 ;
const TOP_PADDING_PT = 15 ;
const NEXT_PROBLEM_PADDING_PT = 5 ;
const MIN_CROP_HEIGHT_PX = 200 ;
const COLUMN_OVERLAP_PT = 18 ;
export interface ImageCropOptions {
paperPdfPath : string ;
outputDir : string ;
year : number ;
subjectName : string ;
dpi? : number ;
expectedProblemCount? : number ;
}
export interface ImageCropResult {
problems : Array < {
number : number ;
imageUrl : string ;
pageNumber : number ;
cropBox : { x : number ; y : number ; width : number ; height : number } ;
} > ;
warnings : string [ ] ;
}
interface PdfPageInfo {
pageNumber : number ;
widthPt : number ;
heightPt : number ;
}
interface BboxWord {
text : string ;
xMin : number ;
yMin : number ;
xMax : number ;
yMax : number ;
}
interface ProblemAnchor {
number : number ;
pageNumber : number ;
xMin : number ;
yMin : number ;
}
interface PassageAnchor {
startNumber : number ;
endNumber : number ;
pageNumber : number ;
yMin : number ;
}
type ProblemColumn = 'left' | 'right' ;
export async function parseImageBasedExam (
options : ImageBasedParseOptions ,
) : Promise < ParseResult > {
const strategy = kiceMathStrategy ;
const strategy = resolveImageStrategy ( options . format ) ;
const warnings : string [ ] = [ ] ;
const pageCount = getPdfPageCount ( options . paperPdfPath ) ;
const textResult = parseProblemPaper ( extractText ( options . paperPdfPath , 'raw' ) , strategy ) ;
const textProblemByNumber = new Map (
textResult . problems . map ( ( problem ) = > [ problem . number , problem ] ) ,
@@ -42,7 +104,7 @@ export async function parseImageBasedExam(
if ( merged . size > answers . length ) {
warnings . push (
` math answer OCR fallback merged ${ merged . size - answers . length } missing answers ` ,
` answer OCR fallback merged ${ merged . size - answers . length } missing answers ` ,
) ;
}
@@ -52,35 +114,32 @@ export async function parseImageBasedExam(
}
}
fs . mkdirSync ( options . renderedImageDir , { recursive : true } ) ;
const pageAssignments = buildProblemPageAssignments (
options . paperPdfPath ,
pageCount ,
options . expectedProblemCount ? ? strategy . maxProblemNumber ,
) ;
const cropResult = await cropProblemsFromPdf ( {
paperPdfPath : options.paperPdfPath ,
outputDir : options.renderedImageDir ,
year : inferYearFromBaseUrl ( options . renderedImageBaseUrl ) ,
subjectName : inferSubjectFromBaseUrl ( options . renderedImageBaseUrl ) ,
dpi : DEFAULT_DPI ,
expectedProblemCount : options.expectedProblemCount ? ? strategy . maxProblemNumber ,
} ) ;
warnings . push ( . . . cropResult . warnings ) ;
const problemCount =
options . expectedProblemCount ? ? inferProblemCount ( textProblemByNumber , strategy ) ;
const cropByNumber = new Map ( cropResult . problems . map ( ( problem ) = > [ problem . number , problem ] ) ) ;
const problems : ParsedProblem [ ] = [ ] ;
const problemCount = options . expectedProblemCount ? ? inferProblemCount ( textProblemByNumber , strategy ) ;
const renderedPages = new Map < number , { imagePath : string ; imageUrl : string } > ( ) ;
for ( let problemNumber = 1 ; problemNumber <= problemCount ; problemNumber ++ ) {
const pageNumber = pageAssignments . get ( problemNumber ) ? ? 1 ;
const renderedPage =
renderedPages . get ( pageNumber ) ? ?
( await renderAndRegisterPage (
options . paperPdfPath ,
pageNumber ,
options . renderedImageDir ,
options . renderedImageBaseUrl ,
) ) ;
renderedPages . set ( pageNumber , renderedPage ) ;
const textProblem = textProblemByNumber . get ( problemNumber ) ;
const needsReviewReasons = Array . from (
new Set ( [ . . . ( textProblem ? . needsReviewReasons ? ? [ ] ) , 'image-based' ] ) ,
) ;
const crop = cropByNumber . get ( problemNumber ) ;
const needsReviewReasons = new Set < string > ( textProblem ? . needsReviewReasons ? ? [ ] ) ;
if ( crop ) {
needsReviewReasons . add ( 'image-based' ) ;
} else {
warnings . push ( ` problem ${ problemNumber } missing crop image ` ) ;
needsReviewReasons . add ( 'missing-image-crop' ) ;
}
problems . push ( {
number : problemNumber ,
@@ -88,10 +147,10 @@ export async function parseImageBasedExam(
choices : textProblem?.choices ? ? emptyChoices ( ) ,
passageStart : textProblem?.passageStart ,
passageEnd : textProblem?.passageEnd ,
imageUrl : renderedPage .imageUrl,
pageImageUrl : renderedPage .imageUrl,
needsReview : true ,
needsReviewReasons ,
imageUrl : crop? .imageUrl,
pageImageUrl : crop? .imageUrl,
needsReview : needsReviewReasons.size > 0 ,
needsReviewReasons : Array.from ( needsReviewReasons ) ,
} ) ;
}
@@ -99,9 +158,7 @@ export async function parseImageBasedExam(
typeof options . expectedProblemCount === 'number' &&
problems . length !== options . expectedProblemCount
) {
warnings . push (
` expected ${ options . expectedProblemCount } problems, parsed ${ problems . length } ` ,
) ;
warnings . push ( ` expected ${ options . expectedProblemCount } problems, parsed ${ problems . length } ` ) ;
}
return {
@@ -112,17 +169,384 @@ export async function parseImageBasedExam(
} ;
}
function getPdfPageCount ( pdfPath : string ) : number {
export async function cropProblemsFromPdf (
options : ImageCropOptions ,
) : Promise < ImageCropResult > {
const dpi = options . dpi ? ? DEFAULT_DPI ;
const warnings : string [ ] = [ ] ;
const pageInfos = getPdfPageInfos ( options . paperPdfPath ) ;
const pageCount = pageInfos . length ;
fs . mkdirSync ( options . outputDir , { recursive : true } ) ;
const problemAnchors : ProblemAnchor [ ] = [ ] ;
const passageAnchors : PassageAnchor [ ] = [ ] ;
for ( const pageInfo of pageInfos ) {
const xml = extractPageBboxXml ( options . paperPdfPath , pageInfo . pageNumber ) ;
const words = extractWordsFromBboxXml ( xml ) ;
for ( const word of words ) {
if ( PROBLEM_WORD_RE . test ( word . text ) ) {
const number = Number ( word . text . slice ( 0 , - 1 ) ) ;
problemAnchors . push ( {
number ,
pageNumber : pageInfo.pageNumber ,
xMin : word.xMin ,
yMin : word.yMin ,
} ) ;
continue ;
}
const passageMatch = word . text . match ( PASSAGE_WORD_RE ) ;
if ( passageMatch ) {
passageAnchors . push ( {
startNumber : Number ( passageMatch [ 1 ] ) ,
endNumber : Number ( passageMatch [ 2 ] ) ,
pageNumber : pageInfo.pageNumber ,
yMin : word.yMin ,
} ) ;
}
}
}
const expectedProblemCount = options . expectedProblemCount ? ? inferExpectedCount ( problemAnchors ) ;
const orderedAnchors = orderProblemAnchors ( problemAnchors , expectedProblemCount ) ;
const cropProblems : ImageCropResult [ 'problems' ] = [ ] ;
for ( let index = 0 ; index < orderedAnchors . length ; index ++ ) {
const current = orderedAnchors [ index ] ;
const pageInfo = pageInfos . find ( ( page ) = > page . pageNumber === current . pageNumber ) ;
if ( ! pageInfo ) {
warnings . push ( ` problem ${ current . number } page metadata missing ` ) ;
continue ;
}
const cropBox = buildCropBox ( current , orderedAnchors , pageInfo , passageAnchors , dpi ) ;
const outputPath = buildProblemImagePath ( options . outputDir , current . number ) ;
const outputPrefix = path . join (
options . outputDir ,
` .tmp-problem- ${ String ( current . number ) . padStart ( 3 , '0' ) } ` ,
) ;
renderCropToPng ( options . paperPdfPath , current . pageNumber , cropBox , dpi , outputPrefix , outputPath ) ;
cropProblems . push ( {
number : current . number ,
imageUrl : buildProblemImageUrl ( options . year , options . subjectName , current . number ) ,
pageNumber : current.pageNumber ,
cropBox ,
} ) ;
}
const missingNumbers = collectMissingNumbers ( expectedProblemCount , cropProblems ) ;
if ( missingNumbers . length > 0 ) {
const pageAssignments = buildProblemPageAssignments (
options . paperPdfPath ,
pageCount ,
expectedProblemCount ,
) ;
for ( const number of missingNumbers ) {
const pageNumber = pageAssignments . get ( number ) ? ? 1 ;
const pageInfo = pageInfos . find ( ( page ) = > page . pageNumber === pageNumber ) ;
if ( ! pageInfo ) {
warnings . push ( ` Problem ${ number } not detected in bbox; fallback page metadata missing ` ) ;
continue ;
}
const cropBox = {
x : 0 ,
y : 0 ,
width : pointsToPixels ( pageInfo . widthPt , dpi ) ,
height : pointsToPixels ( pageInfo . heightPt , dpi ) ,
} ;
const outputPath = buildProblemImagePath ( options . outputDir , number ) ;
const outputPrefix = path . join (
options . outputDir ,
` .tmp-fallback- ${ String ( number ) . padStart ( 3 , '0' ) } ` ,
) ;
renderCropToPng ( options . paperPdfPath , pageNumber , cropBox , dpi , outputPrefix , outputPath ) ;
cropProblems . push ( {
number ,
imageUrl : buildProblemImageUrl ( options . year , options . subjectName , number ) ,
pageNumber ,
cropBox ,
} ) ;
warnings . push ( ` Problem ${ number } not detected in bbox; using full-page fallback ` ) ;
}
}
cropProblems . sort ( ( left , right ) = > left . number - right . number ) ;
return {
problems : cropProblems ,
warnings : Array.from ( new Set ( warnings ) ) ,
} ;
}
function resolveImageStrategy ( format : ImageBasedParseOptions [ 'format' ] ) : PageStripStrategy {
return format === 'kice-math' ? kiceMathStrategy : kiceStrategy ;
}
function inferYearFromBaseUrl ( baseUrl : string ) : number {
const match = baseUrl . match ( /\/uploads\/problems\/(\d{4})\// ) ;
return match ? Number ( match [ 1 ] ) : new Date ( ) . getFullYear ( ) ;
}
function inferSubjectFromBaseUrl ( baseUrl : string ) : string {
const match = baseUrl . match ( /\/uploads\/problems\/\d{4}\/(.+)$/ ) ;
return match ? decodeURIComponent ( match [ 1 ] ) : 'unknown' ;
}
function getPdfPageInfos ( pdfPath : string ) : PdfPageInfo [ ] {
const output = execFileSync ( 'pdfinfo' , [ pdfPath ] , {
encoding : 'utf-8' ,
maxBuffer : 1024 * 1024 ,
} ) ;
const m atch = output . match ( /^Pages:\s+(\d+)/m ) ;
if ( ! m atch) {
const pageCountM atch = output . match ( /^Pages:\s+(\d+)/m ) ;
if ( ! pageCountM atch) {
throw new Error ( ` failed to read page count from pdfinfo output for ${ pdfPath } ` ) ;
}
return Number ( m atch[ 1 ] ) ;
const pageCount = Number ( pageCountM atch[ 1 ] ) ;
const pageSizeMatch = output . match ( /^Page size:\s+([\d.]+)\s+x\s+([\d.]+)\s+pts/m ) ;
if ( ! pageSizeMatch ) {
throw new Error ( ` failed to read page size from pdfinfo output for ${ pdfPath } ` ) ;
}
const widthPt = Number ( pageSizeMatch [ 1 ] ) ;
const heightPt = Number ( pageSizeMatch [ 2 ] ) ;
return Array . from ( { length : pageCount } , ( _ , index ) = > ( {
pageNumber : index + 1 ,
widthPt ,
heightPt ,
} ) ) ;
}
function extractPageBboxXml ( pdfPath : string , pageNumber : number ) : string {
return execFileSync (
'pdftotext' ,
[ '-bbox' , '-f' , String ( pageNumber ) , '-l' , String ( pageNumber ) , '-enc' , 'UTF-8' , pdfPath , '-' ] ,
{
encoding : 'utf-8' ,
maxBuffer : 20 * 1024 * 1024 ,
} ,
) ;
}
function extractWordsFromBboxXml ( xml : string ) : BboxWord [ ] {
const words : BboxWord [ ] = [ ] ;
const wordRe =
/<word\b[^>]*xMin="([^"]+)"[^>]*yMin="([^"]+)"[^>]*xMax="([^"]+)"[^>]*yMax="([^"]+)"[^>]*>([\s\S]*?)<\/word>/g ;
let match : RegExpExecArray | null ;
while ( ( match = wordRe . exec ( xml ) ) !== null ) {
const text = decodeXmlText ( match [ 5 ] ) . trim ( ) ;
if ( ! text ) {
continue ;
}
words . push ( {
text ,
xMin : Number ( match [ 1 ] ) ,
yMin : Number ( match [ 2 ] ) ,
xMax : Number ( match [ 3 ] ) ,
yMax : Number ( match [ 4 ] ) ,
} ) ;
}
return words ;
}
function decodeXmlText ( value : string ) : string {
return value
. replace ( /</g , '<' )
. replace ( />/g , '>' )
. replace ( /&/g , '&' )
. replace ( /"/g , '"' )
. replace ( /'/g , "'" ) ;
}
function orderProblemAnchors (
anchors : ProblemAnchor [ ] ,
expectedProblemCount : number ,
) : ProblemAnchor [ ] {
const byNumber = new Map < number , ProblemAnchor > ( ) ;
anchors
. slice ( )
. sort (
( left , right ) = >
left . pageNumber - right . pageNumber ||
left . yMin - right . yMin ||
left . xMin - right . xMin ,
)
. forEach ( ( anchor ) = > {
if ( anchor . number < 1 || anchor . number > expectedProblemCount || byNumber . has ( anchor . number ) ) {
return ;
}
byNumber . set ( anchor . number , anchor ) ;
} ) ;
return Array . from ( byNumber . values ( ) ) . sort ( ( left , right ) = > left . number - right . number ) ;
}
function buildCropBox (
current : ProblemAnchor ,
anchors : ProblemAnchor [ ] ,
pageInfo : PdfPageInfo ,
passageAnchors : PassageAnchor [ ] ,
dpi : number ,
) : { x : number ; y : number ; width : number ; height : number } {
const passageHeader = passageAnchors . find (
( anchor ) = >
anchor . startNumber === current . number && anchor . pageNumber === current . pageNumber ,
) ;
const column = resolveProblemColumn ( current , pageInfo ) ;
const samePageAnchors = anchors . filter ( ( anchor ) = > anchor . pageNumber === current . pageNumber ) ;
const sameColumnAnchors = samePageAnchors
. filter ( ( anchor ) = > resolveProblemColumn ( anchor , pageInfo ) === column )
. sort ( ( left , right ) = > left . yMin - right . yMin || left . number - right . number ) ;
const currentColumnIndex = sameColumnAnchors . findIndex (
( anchor ) = > anchor . number === current . number ,
) ;
const nextSameColumn = currentColumnIndex >= 0 ? sameColumnAnchors [ currentColumnIndex + 1 ] : undefined ;
const nextAfterPassage = passageHeader
? anchors . find (
( anchor ) = >
anchor . number > passageHeader . endNumber && anchor . pageNumber >= current . pageNumber ,
)
: undefined ;
const startPt = Math . max (
0 ,
Math . min ( current . yMin , passageHeader ? . yMin ? ? current . yMin ) - TOP_PADDING_PT ,
) ;
const defaultEndPt = passageHeader
? nextAfterPassage && nextAfterPassage . pageNumber === current . pageNumber
? Math . max ( nextAfterPassage . yMin - NEXT_PROBLEM_PADDING_PT , startPt )
: pageInfo.heightPt
: nextSameColumn
? Math . max ( nextSameColumn . yMin - NEXT_PROBLEM_PADDING_PT , startPt )
: pageInfo . heightPt ;
let startPx = pointsToPixels ( startPt , dpi ) ;
let endPx = pointsToPixels ( defaultEndPt , dpi ) ;
const pageHeightPx = pointsToPixels ( pageInfo . heightPt , dpi ) ;
const pageWidthPx = pointsToPixels ( pageInfo . widthPt , dpi ) ;
const overlapPx = pointsToPixels ( COLUMN_OVERLAP_PT , dpi ) ;
const halfWidthPx = Math . round ( pageWidthPx / 2 ) ;
if ( endPx - startPx < MIN_CROP_HEIGHT_PX ) {
endPx = Math . min ( pageHeightPx , startPx + MIN_CROP_HEIGHT_PX ) ;
}
if ( endPx <= startPx ) {
startPx = Math . max ( 0 , Math . min ( startPx , pageHeightPx - MIN_CROP_HEIGHT_PX ) ) ;
endPx = Math . min ( pageHeightPx , startPx + MIN_CROP_HEIGHT_PX ) ;
}
return {
x : passageHeader ? 0 : column === 'left' ? 0 : Math.max ( 0 , halfWidthPx - overlapPx ) ,
y : startPx ,
width : passageHeader
? pageWidthPx
: column === 'left'
? Math . min ( pageWidthPx , halfWidthPx + overlapPx )
: Math . min ( pageWidthPx , pageWidthPx - Math . max ( 0 , halfWidthPx - overlapPx ) ) ,
height : Math.max ( MIN_CROP_HEIGHT_PX , endPx - startPx ) ,
} ;
}
function resolveProblemColumn ( current : ProblemAnchor , pageInfo : PdfPageInfo ) : ProblemColumn {
return current . xMin < pageInfo . widthPt / 2 ? 'left' : 'right' ;
}
function renderCropToPng (
pdfPath : string ,
pageNumber : number ,
cropBox : { x : number ; y : number ; width : number ; height : number } ,
dpi : number ,
outPrefix : string ,
outputPath : string ,
) : void {
cleanupGeneratedFiles ( outPrefix ) ;
fs . rmSync ( outputPath , { force : true } ) ;
try {
execFileSync (
'pdftoppm' ,
[
'-r' ,
String ( dpi ) ,
'-png' ,
'-f' ,
String ( pageNumber ) ,
'-l' ,
String ( pageNumber ) ,
'-x' ,
String ( cropBox . x ) ,
'-y' ,
String ( cropBox . y ) ,
'-W' ,
String ( cropBox . width ) ,
'-H' ,
String ( cropBox . height ) ,
pdfPath ,
outPrefix ,
] ,
{
encoding : 'utf-8' ,
stdio : 'pipe' ,
} ,
) ;
} catch ( error ) {
cleanupGeneratedFiles ( outPrefix ) ;
throw error ;
}
const generatedPath = findGeneratedPng ( outPrefix ) ;
if ( ! generatedPath ) {
cleanupGeneratedFiles ( outPrefix ) ;
throw new Error ( ` pdftoppm did not produce expected crop for ${ outputPath } ` ) ;
}
fs . renameSync ( generatedPath , outputPath ) ;
cleanupGeneratedFiles ( outPrefix ) ;
}
function buildProblemImagePath ( outputDir : string , number : number ) : string {
return path . join ( outputDir , ` ${ String ( number ) . padStart ( 3 , '0' ) } .png ` ) ;
}
function buildProblemImageUrl ( year : number , subjectName : string , number : number ) : string {
return ` /uploads/problems/ ${ year } / ${ subjectName } / ${ String ( number ) . padStart ( 3 , '0' ) } .png ` ;
}
function inferExpectedCount ( anchors : ProblemAnchor [ ] ) : number {
const numbers = anchors . map ( ( anchor ) = > anchor . number ) ;
return numbers . length > 0 ? Math . max ( . . . numbers ) : 0 ;
}
function collectMissingNumbers (
expectedProblemCount : number ,
problems : Array < { number : number } > ,
) : number [ ] {
const seen = new Set ( problems . map ( ( problem ) = > problem . number ) ) ;
const missing : number [ ] = [ ] ;
for ( let number = 1 ; number <= expectedProblemCount ; number ++ ) {
if ( ! seen . has ( number ) ) {
missing . push ( number ) ;
}
}
return missing ;
}
function pointsToPixels ( points : number , dpi : number ) : number {
return Math . max ( 0 , Math . round ( ( points * dpi ) / 72 ) ) ;
}
function buildProblemPageAssignments (
@@ -163,7 +587,10 @@ function buildProblemPageAssignments(
for ( let index = 0 ; index < anchors . length ; index ++ ) {
const current = anchors [ index ] ;
const next = anchors [ index + 1 ] ;
const endNumber = Math . min ( next ? . number ? next . number - 1 : expectedProblemCount , expectedProblemCount ) ;
const endNumber = Math . min (
next ? . number ? next . number - 1 : expectedProblemCount ,
expectedProblemCount ,
) ;
for ( let problemNumber = current . number ; problemNumber <= endNumber ; problemNumber ++ ) {
assignments . set ( problemNumber , current . pageNumber ) ;
@@ -210,27 +637,9 @@ function collectPageProblemNumbers(pageText: string, maxProblemNumber: number):
return numbers . sort ( ( left , right ) = > left - right ) ;
}
async function renderAndRegisterPage (
pdfPath : string ,
pageNumber : number ,
renderedImageDir : string ,
renderedImageBaseUrl : string ,
) : Promise < { imagePath : string ; imageUrl : string } > {
const imagePath = await renderPdfPageToPng (
pdfPath ,
pageNumber ,
` ${ renderedImageDir } /page.png ` ,
) ;
return {
imagePath ,
imageUrl : ` ${ stripTrailingSlash ( renderedImageBaseUrl ) } /page- ${ pageNumber } .png ` ,
} ;
}
function inferProblemCount (
textProblemByNumber : Map < number , ParsedProblem > ,
strategy : typeof kiceMath Strategy,
strategy : PageStrip Strategy,
) : number {
const parsedNumbers = Array . from ( textProblemByNumber . keys ( ) ) ;
if ( parsedNumbers . length === 0 ) {
@@ -240,8 +649,36 @@ function inferProblemCount(
return Math . max ( . . . parsedNumbers ) ;
}
function stripTrailingSlash ( value : string ) : string {
return value . replace ( /\/+$/ , '' ) ;
function cleanupGeneratedFiles ( outPrefix : string ) : void {
const dir = path . dirname ( outPrefix ) ;
const base = path . basename ( outPrefix ) ;
if ( ! fs . existsSync ( dir ) ) {
return ;
}
for ( const entry of fs . readdirSync ( dir ) ) {
if ( ! entry . startsWith ( ` ${ base } - ` ) || ! entry . endsWith ( '.png' ) ) {
continue ;
}
fs . rmSync ( path . join ( dir , entry ) , { force : true } ) ;
}
}
function findGeneratedPng ( outPrefix : string ) : string | null {
const dir = path . dirname ( outPrefix ) ;
const base = path . basename ( outPrefix ) ;
if ( ! fs . existsSync ( dir ) ) {
return null ;
}
const matches = fs
. readdirSync ( dir )
. filter ( ( entry ) = > entry . startsWith ( ` ${ base } - ` ) && entry . endsWith ( '.png' ) )
. sort ( ) ;
return matches [ 0 ] ? path . join ( dir , matches [ 0 ] ) : null ;
}
function emptyChoices ( ) : Record < '1' | '2' | '3' | '4' | '5' , string > {