Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion app/api/extract-pdf/route.ts
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ function normalizeMetadata(value: Record<string, unknown>, fallback: ExtractedMe
doi: cleanNullable(value.doi) ?? fallback.doi,
url: cleanNullable(value.url) ?? fallback.url,
category: cleanNullable(value.category) ?? fallback.category,
preprintId: cleanNullable(value.preprintId || value.preprint_id || value.arxivId) ?? fallback.preprintId,
preprintId: cleanNullable(value.preprintId || value.preprint_id) ?? fallback.preprintId,
};
}

Expand Down
3 changes: 3 additions & 0 deletions app/api/feed/snippets/[id]/events/route.ts
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,9 @@ export async function GET(
content: message.content,
toolUseId: message.toolUseId,
attachments: message.attachments,
inputTokens: message.inputTokens,
outputTokens: message.outputTokens,
durationMs: message.durationMs,
createdAt: message.createdAt,
})),
proposals,
Expand Down
5 changes: 3 additions & 2 deletions app/api/import/route.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import { captureWebpageSnapshot } from "@/app/lib/webpage-snapshot";
import { parseRequest } from "@/app/lib/schemas/parse";
import { ImportUrlRequestSchema } from "@/app/lib/schemas/requests";
import { canonicalPreprintId } from "@/app/lib/preprint-id";

export const dynamic = "force-dynamic";
export const runtime = "nodejs";
Expand Down Expand Up @@ -43,7 +44,7 @@ export async function POST(request: Request): Promise<Response> {
abstract: "",
url: parsed.toString(),
pdfUrl: parsed.toString(),
arxivId: arxivMatch?.[1]?.replace(/\.pdf$/i, "") ?? null,
preprintId: canonicalPreprintId(arxivMatch?.[1]?.replace(/\.pdf$/i, "") ?? null),
readerContent: "",
});
}
Expand All @@ -70,7 +71,7 @@ export async function POST(request: Request): Promise<Response> {
abstract: snapshot.text.slice(0, 1200),
url: snapshot.finalUrl || sourceUrl,
pdfUrl: isPdfPath ? resolved.toString() : null,
arxivId: arxivMatch?.[1]?.replace(/\.pdf$/i, "") ?? null,
preprintId: canonicalPreprintId(arxivMatch?.[1]?.replace(/\.pdf$/i, "") ?? null),
readerContent: snapshot.text.slice(0, 14000),
});
} catch (error) {
Expand Down
44 changes: 12 additions & 32 deletions app/api/library/route.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import { ensureDatabase } from "@/db/bootstrap";
import type { LibraryQuerier } from "@/db/client";
import { removeStoredFile, resolveStoredFile } from "@/app/lib/local-files";
import { normalizeAbstract, normalizeAuthorNames, normalizePages, normalizeTitle } from "@/app/lib/metadata-normalize";
import { canonicalPreprintId } from "@/app/lib/preprint-id";
import { scheduleAutoSync } from "@/app/lib/local-settings";
import { idList, LibraryMutationSchema } from "@/app/lib/schemas/library";
import { parseRequest } from "@/app/lib/schemas/parse";
Expand Down Expand Up @@ -30,7 +31,7 @@ function jsonError(message: string, status = 400): Response {
/** Turn raw SQLite errors into user-facing messages, hiding internal detail. */
/** Thrown when a paper create is a duplicate of an existing library record. The
* check runs inside the insert transaction (so it can't race a concurrent
* create), and the unique indexes on doi/arxiv_id/semantic_scholar_id are the
* create), and the unique indexes on doi/preprint_id/semantic_scholar_id are the
* final backstop if two transactions still interleave. */
class DuplicatePaperError extends Error {
constructor() {
Expand All @@ -43,7 +44,7 @@ function describeDbError(error: unknown): string {
const raw = error instanceof Error ? error.message : String(error);
if (/UNIQUE constraint failed/i.test(raw)) {
if (/papers\.doi/i.test(raw)) return "A paper with this DOI is already in your library.";
if (/papers\.arxiv_id/i.test(raw)) return "A paper with this arXiv id is already in your library.";
if (/papers\.preprint_id/i.test(raw)) return "A paper with this preprint ID is already in your library.";
if (/papers\.semantic_scholar_id/i.test(raw)) return "A paper with this Semantic Scholar id is already in your library.";
return "This record already exists in your library.";
}
Expand Down Expand Up @@ -212,7 +213,6 @@ async function readSnapshot() {
pages: paper.pages,
category: paper.category,
doi: paper.doi,
arxivId: paper.arxivId,
preprintId: paper.preprintId,
semanticScholarId: paper.semanticScholarId,
url: paper.url,
Expand Down Expand Up @@ -420,31 +420,9 @@ function resolveCollectionIdsByName(querier: LibraryQuerier, collectionNames: un

/**
* Return the id of an existing paper that matches this record by a strong
* identifier (DOI, arXiv id, or Semantic Scholar id), used to skip duplicates on
* identifier (DOI, preprint id, or Semantic Scholar id), used to skip duplicates on
* import. Title is intentionally not matched here — it is too noisy for dedup.
*/
/**
* Canonical form of an arXiv id: the bare identifier, lowercased.
*
* The app's own producers disagree: the seed data writes "arXiv:2605.09104", the
* scholarly providers write the bare id, and a BibTeX import can carry either
* plus a full URL. Since dedup compares this column exactly, the same paper
* imported from two sources produced two rows (and the unique index never fired).
*/
function canonicalArxivId(value: unknown): string | null {
const raw = cleanString(value);
if (!raw) {
return null;
}
const stripped = raw
.replace(/^https?:\/\/(?:www\.)?arxiv\.org\/(?:abs|pdf)\//i, "")
.replace(/^arxiv[:\s]*/i, "")
.replace(/v\d+$/i, "")
.replace(/\.pdf$/i, "")
.trim();
return stripped.toLowerCase() || null;
}

/**
* Canonical form of a DOI: lowercased, with any resolver prefix removed. DOIs are
* case-insensitive by spec, so comparing them raw let "10.1000/ABC" and
Expand All @@ -464,11 +442,11 @@ function canonicalDoi(value: unknown): string | null {

function findDuplicatePaper(querier: LibraryQuerier, data: Record<string, unknown>): string | null {
const doi = canonicalDoi(data.doi);
const arxivId = canonicalArxivId(data.arxivId);
const preprintId = canonicalPreprintId(data.preprintId);
const semanticScholarId = cleanString(data.semanticScholarId);
const checks: Array<ReturnType<typeof eq>> = [];
if (doi) checks.push(eq(papers.doi, doi));
if (arxivId) checks.push(eq(papers.arxivId, arxivId));
if (preprintId) checks.push(eq(papers.preprintId, preprintId));
if (semanticScholarId) checks.push(eq(papers.semanticScholarId, semanticScholarId));
for (const condition of checks) {
const existing = querier.select({ id: papers.id }).from(papers).where(condition).limit(1).get();
Expand Down Expand Up @@ -519,8 +497,7 @@ async function createPaper(data: Record<string, unknown>): Promise<void> {
pages: normalizedPages,
category: cleanString(data.category),
doi: canonicalDoi(data.doi),
arxivId: canonicalArxivId(data.arxivId),
preprintId: cleanString(data.preprintId),
preprintId: canonicalPreprintId(data.preprintId),
semanticScholarId: cleanString(data.semanticScholarId),
url: cleanString(data.url),
pdfUrl: cleanString(data.pdfUrl),
Expand Down Expand Up @@ -742,7 +719,6 @@ const paperTextFields = {
pages: papers.pages,
category: papers.category,
doi: papers.doi,
arxivId: papers.arxivId,
preprintId: papers.preprintId,
url: papers.url,
pdfUrl: papers.pdfUrl,
Expand Down Expand Up @@ -780,7 +756,11 @@ async function updatePaper(id: string, data: Record<string, unknown>): Promise<v
for (const key of Object.keys(paperTextFields) as Array<keyof typeof paperTextFields>) {
if (key in data) {
const value = textValue(data[key]);
assignments[key] = key === "pages" && value ? normalizePages(value) : value;
assignments[key] = key === "pages" && value
? normalizePages(value)
: key === "preprintId"
? canonicalPreprintId(value)
: value;
}
}

Expand Down
4 changes: 2 additions & 2 deletions app/components/FeedRichContent.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ export function FeedImage({ feedId, feedName, alt = "", src = "", ...props }: Co
const source = typeof src === "string" ? src : "";
const id = useMemo(() => feedVisualizationId("image", feedId, source), [feedId, source]);
return (
<span className="feed-rich-content feed-rich-image">
<span className="feed-rich-content markdown-media markdown-image">
<span className="feed-rich-actions">
<OpenInNewWindow
id={id}
Expand Down Expand Up @@ -123,7 +123,7 @@ export function FeedTable({ feedId, feedName, children, ...props }: ComponentPro
return (
<div className="feed-rich-content feed-rich-table">
<div className="feed-rich-actions"><OpenInNewWindow id={id} prepare={prepare} /></div>
<div className="feed-rich-table-scroll">
<div className="markdown-media markdown-table-scroll">
<table ref={tableRef} {...props}>{children}</table>
</div>
</div>
Expand Down
Loading