Skip to content

Commit 918f611

Browse files
diluculoclaude
andcommitted
Add run-odl-golden.ps1 to generate ODL reference outputs
For every PDF in playground/ (overridable via -Path), invokes the OpenDataLoader-pdf CLI from out/odl-venv and writes JSON, Markdown, and an annotated PDF into <name>/ODL/, plus extracted images under <name>/ODL/images/. Then rasterizes the annotated PDF through pdftoppm into zero-padded page-NNN.png files for visual diffing against PdfStruct's run-pdfstruct.ps1 -DebugLines output. Reports a per-PDF summary table and exits non-zero if any PDF failed. This is the producer side of the ODL-as-baseline workflow already referenced from docs/odl-divergence-log.md; committing it makes the divergence-log instructions reproducible. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
1 parent c113f22 commit 918f611

1 file changed

Lines changed: 132 additions & 0 deletions

File tree

run-odl-golden.ps1

Lines changed: 132 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
# Generate OpenDataLoader reference outputs for every PDF in playground.
2+
#
3+
# For each <name>.pdf, produces:
4+
# <name>/ODL/<name>.json
5+
# <name>/ODL/<name>.md
6+
# <name>/ODL/<name>_annotated.pdf
7+
# <name>/ODL/page-NNN.png
8+
# <name>/ODL/images/* - extracted images when ODL emits them
9+
#
10+
# Usage:
11+
# .\run-odl-golden.ps1
12+
# .\run-odl-golden.ps1 -Path D:\some\folder
13+
# .\run-odl-golden.ps1 -Clean
14+
15+
[CmdletBinding()]
16+
param(
17+
[string] $Path = (Join-Path $PSScriptRoot 'playground'),
18+
[string] $OdlCli = (Join-Path $PSScriptRoot 'out\odl-venv\Scripts\opendataloader-pdf.exe'),
19+
[string] $PdfToPng = 'pdftoppm',
20+
[switch] $Clean
21+
)
22+
23+
if (-not (Test-Path -LiteralPath $OdlCli -PathType Leaf)) {
24+
Write-Error "ODL CLI not found: $OdlCli`nInstall it first, for example: python -m venv out\odl-venv; out\odl-venv\Scripts\python.exe -m pip install opendataloader-pdf"
25+
exit 1
26+
}
27+
28+
if (-not (Get-Command $PdfToPng -ErrorAction SilentlyContinue)) {
29+
Write-Error "PDF-to-PNG renderer not found: $PdfToPng"
30+
exit 1
31+
}
32+
33+
if (-not (Test-Path -LiteralPath $Path -PathType Container)) {
34+
Write-Error "Playground folder not found: $Path"
35+
exit 1
36+
}
37+
38+
$pdfs = Get-ChildItem -LiteralPath $Path -Filter '*.pdf' -File
39+
if ($pdfs.Count -eq 0) {
40+
Write-Warning "No PDFs found in $Path"
41+
exit 0
42+
}
43+
44+
Write-Host "Found $($pdfs.Count) PDF(s) in $Path" -ForegroundColor Cyan
45+
Write-Host "Using ODL CLI: $OdlCli" -ForegroundColor DarkGray
46+
Write-Host "Using renderer: $PdfToPng`n" -ForegroundColor DarkGray
47+
48+
$results = @()
49+
$totalStart = Get-Date
50+
51+
foreach ($pdf in $pdfs) {
52+
$name = [IO.Path]::GetFileNameWithoutExtension($pdf.Name)
53+
$pdfOutputDir = Join-Path $Path $name
54+
$odlOutputDir = Join-Path $pdfOutputDir 'ODL'
55+
$imageDir = Join-Path $odlOutputDir 'images'
56+
57+
Write-Host "[$name] " -NoNewline -ForegroundColor Yellow
58+
Write-Host $pdf.Name -ForegroundColor White
59+
60+
if ($Clean -and (Test-Path -LiteralPath $odlOutputDir)) {
61+
Remove-Item -LiteralPath $odlOutputDir -Recurse -Force
62+
}
63+
New-Item -ItemType Directory -Path $odlOutputDir -Force | Out-Null
64+
New-Item -ItemType Directory -Path $imageDir -Force | Out-Null
65+
66+
Get-ChildItem -LiteralPath $odlOutputDir -Filter 'page-*.png' -File -ErrorAction SilentlyContinue |
67+
Remove-Item -Force
68+
69+
$start = Get-Date
70+
$odlArgs = @(
71+
$pdf.FullName,
72+
'--output-dir', $odlOutputDir,
73+
'--format', 'json,markdown,pdf',
74+
'--image-output', 'external',
75+
'--image-format', 'png',
76+
'--image-dir', $imageDir,
77+
'--quiet'
78+
)
79+
80+
$odlOutput = & $OdlCli @odlArgs 2>&1
81+
$odlExitCode = $LASTEXITCODE
82+
foreach ($line in $odlOutput) { Write-Host " $line" -ForegroundColor DarkGray }
83+
84+
$pngExitCode = 0
85+
$annotatedPdf = Join-Path $odlOutputDir "$($name)_annotated.pdf"
86+
if ($odlExitCode -eq 0 -and (Test-Path -LiteralPath $annotatedPdf -PathType Leaf)) {
87+
$pagePrefix = Join-Path $odlOutputDir 'page'
88+
$pngOutput = & $PdfToPng -png -r 144 $annotatedPdf $pagePrefix 2>&1
89+
$pngExitCode = $LASTEXITCODE
90+
foreach ($line in $pngOutput) { Write-Host " $line" -ForegroundColor DarkGray }
91+
92+
$renderedPages = Get-ChildItem -LiteralPath $odlOutputDir -Filter 'page-*.png' -File |
93+
Where-Object { $_.Name -match '^page-\d+\.png$' } |
94+
Sort-Object {
95+
if ($_.BaseName -match '^page-(\d+)$') { [int]$Matches[1] } else { 0 }
96+
}
97+
98+
foreach ($page in $renderedPages) {
99+
if ($page.BaseName -match '^page-(\d+)$') {
100+
$target = Join-Path $odlOutputDir ('page-{0:D3}.png' -f [int]$Matches[1])
101+
if ($page.FullName -ne $target) {
102+
if (Test-Path -LiteralPath $target) {
103+
Remove-Item -LiteralPath $target -Force
104+
}
105+
Move-Item -LiteralPath $page.FullName -Destination $target
106+
}
107+
}
108+
}
109+
}
110+
111+
$elapsed = (Get-Date) - $start
112+
$status = if ($odlExitCode -eq 0 -and $pngExitCode -eq 0) { 'OK' } else { "FAIL (odl=$odlExitCode png=$pngExitCode)" }
113+
$results += [PSCustomObject]@{
114+
Name = $name
115+
Status = $status
116+
Seconds = [math]::Round($elapsed.TotalSeconds, 2)
117+
Json = if (Test-Path -LiteralPath (Join-Path $odlOutputDir "$name.json")) { (Get-Item -LiteralPath (Join-Path $odlOutputDir "$name.json")).Length } else { 0 }
118+
Markdown = if (Test-Path -LiteralPath (Join-Path $odlOutputDir "$name.md")) { (Get-Item -LiteralPath (Join-Path $odlOutputDir "$name.md")).Length } else { 0 }
119+
Pages = (Get-ChildItem -LiteralPath $odlOutputDir -Filter 'page-*.png' -File -ErrorAction SilentlyContinue).Count
120+
Images = (Get-ChildItem -LiteralPath $imageDir -File -ErrorAction SilentlyContinue).Count
121+
}
122+
}
123+
124+
$totalElapsed = (Get-Date) - $totalStart
125+
126+
Write-Host "`nSummary:" -ForegroundColor Cyan
127+
$results | Format-Table -AutoSize
128+
129+
$failed = @($results | Where-Object Status -ne 'OK').Count
130+
Write-Host ("Total: {0} files in {1:F1}s, {2} failed" -f $results.Count, $totalElapsed.TotalSeconds, $failed) -ForegroundColor $(if ($failed -eq 0) { 'Green' } else { 'Red' })
131+
132+
if ($failed -gt 0) { exit 1 }

0 commit comments

Comments
 (0)