|
| 1 | +# Generate OpenDataLoader reference outputs for every PDF in playground. |
| 2 | +# |
| 3 | +# For each <name>.pdf, produces: |
| 4 | +# <name>/ODL/<name>.json |
| 5 | +# <name>/ODL/<name>.md |
| 6 | +# <name>/ODL/<name>_annotated.pdf |
| 7 | +# <name>/ODL/page-NNN.png |
| 8 | +# <name>/ODL/images/* - extracted images when ODL emits them |
| 9 | +# |
| 10 | +# Usage: |
| 11 | +# .\run-odl-golden.ps1 |
| 12 | +# .\run-odl-golden.ps1 -Path D:\some\folder |
| 13 | +# .\run-odl-golden.ps1 -Clean |
| 14 | + |
| 15 | +[CmdletBinding()] |
| 16 | +param( |
| 17 | + [string] $Path = (Join-Path $PSScriptRoot 'playground'), |
| 18 | + [string] $OdlCli = (Join-Path $PSScriptRoot 'out\odl-venv\Scripts\opendataloader-pdf.exe'), |
| 19 | + [string] $PdfToPng = 'pdftoppm', |
| 20 | + [switch] $Clean |
| 21 | +) |
| 22 | + |
| 23 | +if (-not (Test-Path -LiteralPath $OdlCli -PathType Leaf)) { |
| 24 | + Write-Error "ODL CLI not found: $OdlCli`nInstall it first, for example: python -m venv out\odl-venv; out\odl-venv\Scripts\python.exe -m pip install opendataloader-pdf" |
| 25 | + exit 1 |
| 26 | +} |
| 27 | + |
| 28 | +if (-not (Get-Command $PdfToPng -ErrorAction SilentlyContinue)) { |
| 29 | + Write-Error "PDF-to-PNG renderer not found: $PdfToPng" |
| 30 | + exit 1 |
| 31 | +} |
| 32 | + |
| 33 | +if (-not (Test-Path -LiteralPath $Path -PathType Container)) { |
| 34 | + Write-Error "Playground folder not found: $Path" |
| 35 | + exit 1 |
| 36 | +} |
| 37 | + |
| 38 | +$pdfs = Get-ChildItem -LiteralPath $Path -Filter '*.pdf' -File |
| 39 | +if ($pdfs.Count -eq 0) { |
| 40 | + Write-Warning "No PDFs found in $Path" |
| 41 | + exit 0 |
| 42 | +} |
| 43 | + |
| 44 | +Write-Host "Found $($pdfs.Count) PDF(s) in $Path" -ForegroundColor Cyan |
| 45 | +Write-Host "Using ODL CLI: $OdlCli" -ForegroundColor DarkGray |
| 46 | +Write-Host "Using renderer: $PdfToPng`n" -ForegroundColor DarkGray |
| 47 | + |
| 48 | +$results = @() |
| 49 | +$totalStart = Get-Date |
| 50 | + |
| 51 | +foreach ($pdf in $pdfs) { |
| 52 | + $name = [IO.Path]::GetFileNameWithoutExtension($pdf.Name) |
| 53 | + $pdfOutputDir = Join-Path $Path $name |
| 54 | + $odlOutputDir = Join-Path $pdfOutputDir 'ODL' |
| 55 | + $imageDir = Join-Path $odlOutputDir 'images' |
| 56 | + |
| 57 | + Write-Host "[$name] " -NoNewline -ForegroundColor Yellow |
| 58 | + Write-Host $pdf.Name -ForegroundColor White |
| 59 | + |
| 60 | + if ($Clean -and (Test-Path -LiteralPath $odlOutputDir)) { |
| 61 | + Remove-Item -LiteralPath $odlOutputDir -Recurse -Force |
| 62 | + } |
| 63 | + New-Item -ItemType Directory -Path $odlOutputDir -Force | Out-Null |
| 64 | + New-Item -ItemType Directory -Path $imageDir -Force | Out-Null |
| 65 | + |
| 66 | + Get-ChildItem -LiteralPath $odlOutputDir -Filter 'page-*.png' -File -ErrorAction SilentlyContinue | |
| 67 | + Remove-Item -Force |
| 68 | + |
| 69 | + $start = Get-Date |
| 70 | + $odlArgs = @( |
| 71 | + $pdf.FullName, |
| 72 | + '--output-dir', $odlOutputDir, |
| 73 | + '--format', 'json,markdown,pdf', |
| 74 | + '--image-output', 'external', |
| 75 | + '--image-format', 'png', |
| 76 | + '--image-dir', $imageDir, |
| 77 | + '--quiet' |
| 78 | + ) |
| 79 | + |
| 80 | + $odlOutput = & $OdlCli @odlArgs 2>&1 |
| 81 | + $odlExitCode = $LASTEXITCODE |
| 82 | + foreach ($line in $odlOutput) { Write-Host " $line" -ForegroundColor DarkGray } |
| 83 | + |
| 84 | + $pngExitCode = 0 |
| 85 | + $annotatedPdf = Join-Path $odlOutputDir "$($name)_annotated.pdf" |
| 86 | + if ($odlExitCode -eq 0 -and (Test-Path -LiteralPath $annotatedPdf -PathType Leaf)) { |
| 87 | + $pagePrefix = Join-Path $odlOutputDir 'page' |
| 88 | + $pngOutput = & $PdfToPng -png -r 144 $annotatedPdf $pagePrefix 2>&1 |
| 89 | + $pngExitCode = $LASTEXITCODE |
| 90 | + foreach ($line in $pngOutput) { Write-Host " $line" -ForegroundColor DarkGray } |
| 91 | + |
| 92 | + $renderedPages = Get-ChildItem -LiteralPath $odlOutputDir -Filter 'page-*.png' -File | |
| 93 | + Where-Object { $_.Name -match '^page-\d+\.png$' } | |
| 94 | + Sort-Object { |
| 95 | + if ($_.BaseName -match '^page-(\d+)$') { [int]$Matches[1] } else { 0 } |
| 96 | + } |
| 97 | + |
| 98 | + foreach ($page in $renderedPages) { |
| 99 | + if ($page.BaseName -match '^page-(\d+)$') { |
| 100 | + $target = Join-Path $odlOutputDir ('page-{0:D3}.png' -f [int]$Matches[1]) |
| 101 | + if ($page.FullName -ne $target) { |
| 102 | + if (Test-Path -LiteralPath $target) { |
| 103 | + Remove-Item -LiteralPath $target -Force |
| 104 | + } |
| 105 | + Move-Item -LiteralPath $page.FullName -Destination $target |
| 106 | + } |
| 107 | + } |
| 108 | + } |
| 109 | + } |
| 110 | + |
| 111 | + $elapsed = (Get-Date) - $start |
| 112 | + $status = if ($odlExitCode -eq 0 -and $pngExitCode -eq 0) { 'OK' } else { "FAIL (odl=$odlExitCode png=$pngExitCode)" } |
| 113 | + $results += [PSCustomObject]@{ |
| 114 | + Name = $name |
| 115 | + Status = $status |
| 116 | + Seconds = [math]::Round($elapsed.TotalSeconds, 2) |
| 117 | + Json = if (Test-Path -LiteralPath (Join-Path $odlOutputDir "$name.json")) { (Get-Item -LiteralPath (Join-Path $odlOutputDir "$name.json")).Length } else { 0 } |
| 118 | + Markdown = if (Test-Path -LiteralPath (Join-Path $odlOutputDir "$name.md")) { (Get-Item -LiteralPath (Join-Path $odlOutputDir "$name.md")).Length } else { 0 } |
| 119 | + Pages = (Get-ChildItem -LiteralPath $odlOutputDir -Filter 'page-*.png' -File -ErrorAction SilentlyContinue).Count |
| 120 | + Images = (Get-ChildItem -LiteralPath $imageDir -File -ErrorAction SilentlyContinue).Count |
| 121 | + } |
| 122 | +} |
| 123 | + |
| 124 | +$totalElapsed = (Get-Date) - $totalStart |
| 125 | + |
| 126 | +Write-Host "`nSummary:" -ForegroundColor Cyan |
| 127 | +$results | Format-Table -AutoSize |
| 128 | + |
| 129 | +$failed = @($results | Where-Object Status -ne 'OK').Count |
| 130 | +Write-Host ("Total: {0} files in {1:F1}s, {2} failed" -f $results.Count, $totalElapsed.TotalSeconds, $failed) -ForegroundColor $(if ($failed -eq 0) { 'Green' } else { 'Red' }) |
| 131 | + |
| 132 | +if ($failed -gt 0) { exit 1 } |
0 commit comments