- onSelectDataset(
- dataset.dataset_id.toString(),
- dataset.dataset_name,
- )
- }
- >
-
-
-
-
-
- {dataset.dataset_name}
-
-
- {dataset.description && (
-
- )}
-
-
{dataset.total_items} items
- {dataset.original_items > 0 &&
- dataset.original_items !== dataset.total_items && (
- <>
-
·
-
{dataset.original_items} original
- >
- )}
+ {datasets.map((dataset) => (
+
+
+
+
+
+
+ {dataset.dataset_name}
-
-
{
- event.stopPropagation();
- onViewDataset(
- dataset.dataset_id,
- dataset.dataset_name,
- );
- }}
- disabled={viewingId === dataset.dataset_id}
- className={` ${
- viewingId === dataset.dataset_id ? "opacity-50" : ""
- }`}
- >
- {viewingId === dataset.dataset_id
- ? "Loading..."
- : "View"}
-
-
{
- event.stopPropagation();
- onRequestDelete(dataset.dataset_id);
- }}
- aria-label={`Delete ${dataset.dataset_name}`}
- >
- Delete
-
+ {dataset.description && (
+
+ )}
+
+ {dataset.total_items} items
+ {dataset.original_items > 0 &&
+ dataset.original_items !== dataset.total_items && (
+ <>
+ ·
+ {dataset.original_items} original
+ >
+ )}
+
+
+ onViewDataset(
+ dataset.dataset_id,
+ dataset.dataset_name,
+ )
+ }
+ disabled={viewingId === dataset.dataset_id}
+ className={` ${
+ viewingId === dataset.dataset_id ? "opacity-50" : ""
+ }`}
+ >
+ {viewingId === dataset.dataset_id
+ ? "Loading..."
+ : "View"}
+
+ onRequestDelete(dataset.dataset_id)}
+ aria-label={`Delete ${dataset.dataset_name}`}
+ >
+ Delete
+
+
- );
- })}
+
+ ))}
)}
-
-
-
-
- {canProceed
- ? "Dataset selected. Continue to AI configuration."
- : "Select a dataset to continue."}
-
-
- Next: AI Configuration
-
-
-
);
}
diff --git a/app/components/assessment/prompt-config/AssessmentConfiguration.tsx b/app/components/assessment/prompt-config/AssessmentConfiguration.tsx
index a0465778..49b23a25 100644
--- a/app/components/assessment/prompt-config/AssessmentConfiguration.tsx
+++ b/app/components/assessment/prompt-config/AssessmentConfiguration.tsx
@@ -1,32 +1,9 @@
"use client";
-import { RadioGroup } from "@/app/components/ui";
import type { AssessmentConfigurationProps } from "@/app/lib/types/assessment";
-import type { ConfigMode } from "@/app/lib/types/assessment/config";
import ConfigCreator from "./ConfigCreator";
-import SavedConfigs from "./SavedConfigs";
-import SelectedConfigs from "./SelectedConfigs";
export default function AssessmentConfiguration({
- configMode,
- setConfigMode,
- configs,
- onRemoveConfig,
- configCards,
- searchQuery,
- setSearchQuery,
- isLoadingConfigs,
- hasMoreConfigs,
- nextConfigSkip,
- expandedConfigId,
- versionStateByConfig,
- latestModelByConfig,
- loadingSelectionKeys,
- isSelected,
- onLoadMoreConfigs,
- onLoadVersions,
- onToggleConfigExpansion,
- onToggleVersionSelection,
currentProvider,
currentModel,
providerModels,
@@ -35,6 +12,13 @@ export default function AssessmentConfiguration({
configName,
commitMessage,
isSaving,
+ isSaveModalOpen,
+ setIsSaveModalOpen,
+ saveMode,
+ setSaveMode,
+ versionConfigId,
+ setVersionConfigId,
+ existingConfigs,
setConfigName,
setCommitMessage,
onProviderChange,
@@ -47,73 +31,38 @@ export default function AssessmentConfiguration({
- AI Configuration
+ Model Selection
- {configs.length > 0
- ? `${configs.length} selected`
- : "Choose at least one configuration"}
+ Choose the provider and model this configuration runs on.
- {configs.length > 0 && (
-
-
-
- )}
-
-
-
- value={configMode}
- onChange={setConfigMode}
- ariaLabel="Config source"
- options={[
- { value: "existing", label: "Saved" },
- { value: "create", label: "New" },
- ]}
- />
-
-
- {configMode === "existing" && (
-
- )}
-
- {configMode === "create" && (
-
- )}
+
);
diff --git a/app/components/assessment/prompt-config/ConfigCreator.tsx b/app/components/assessment/prompt-config/ConfigCreator.tsx
index 524f9d58..605fdedd 100644
--- a/app/components/assessment/prompt-config/ConfigCreator.tsx
+++ b/app/components/assessment/prompt-config/ConfigCreator.tsx
@@ -1,6 +1,11 @@
-import { Button, Field, Select } from "@/app/components/ui";
+"use client";
+
+import { Button, Field, Modal, RadioGroup, Select } from "@/app/components/ui";
import { PROVIDER_OPTIONS } from "@/app/lib/data/assessmentModels";
-import type { ConfigCreatorProps } from "@/app/lib/types/assessment";
+import type {
+ ConfigCreatorProps,
+ ConfigSaveMode,
+} from "@/app/lib/types/assessment";
import type { CompletionConfig } from "@/app/lib/types/configs";
import ConfigParamControl from "./ConfigParamControl";
@@ -16,6 +21,13 @@ export default function ConfigCreator({
configName,
commitMessage,
isSaving,
+ isSaveModalOpen,
+ setIsSaveModalOpen,
+ saveMode,
+ setSaveMode,
+ versionConfigId,
+ setVersionConfigId,
+ existingConfigs,
setConfigName,
setCommitMessage,
onProviderChange,
@@ -23,7 +35,14 @@ export default function ConfigCreator({
onParamChange,
onSave,
}: ConfigCreatorProps) {
- const saveDisabled = isSaving || !configName.trim();
+ const confirmDisabled =
+ isSaving ||
+ (saveMode === "version" ? !versionConfigId : !configName.trim());
+
+ const handleConfirm = () => {
+ void onSave();
+ setIsSaveModalOpen(false);
+ };
return (
@@ -87,29 +106,73 @@ export default function ConfigCreator({
-
-
-
-
-
-
void onSave()}
- disabled={saveDisabled}
+ setIsSaveModalOpen(false)}
+ title="Save configuration"
+ maxWidth="max-w-md"
+ maxHeight="max-h-[90vh]"
>
- {isSaving ? "Saving..." : "Save behavior"}
-
+
+
+ value={saveMode}
+ onChange={setSaveMode}
+ ariaLabel="Save mode"
+ options={[
+ { value: "new", label: "New configuration" },
+ { value: "version", label: "New version of existing" },
+ ]}
+ />
+
+ {saveMode === "new" ? (
+
+ ) : (
+
+
+ Existing configuration
+
+ ({
+ value: config.id,
+ label: config.name,
+ }))}
+ onChange={(event) => setVersionConfigId(event.target.value)}
+ className={selectClass}
+ />
+
+ )}
+
+
+
+
+ setIsSaveModalOpen(false)}
+ >
+ Cancel
+
+
+ {isSaving ? "Saving..." : "Save"}
+
+
+
);
}
diff --git a/app/components/assessment/prompt-config/PromptEditor.tsx b/app/components/assessment/prompt-config/PromptEditor.tsx
index 93b86d9f..34bbb380 100644
--- a/app/components/assessment/prompt-config/PromptEditor.tsx
+++ b/app/components/assessment/prompt-config/PromptEditor.tsx
@@ -2,7 +2,7 @@
import { Button } from "@/app/components/ui";
import { usePromptPlaceholderEditor } from "@/app/hooks/usePromptPlaceholderEditor";
-import type { SampleRow, ValueSetter } from "@/app/lib/types/assessment";
+import type { ValueSetter } from "@/app/lib/types/assessment";
interface PromptEditorProps {
value: string;
@@ -11,7 +11,6 @@ interface PromptEditorProps {
placeholder: string;
emptyPreviewText: string;
textColumns?: string[];
- sampleRow?: SampleRow;
enablePlaceholders?: boolean;
}
@@ -22,7 +21,6 @@ export default function PromptEditor({
placeholder,
emptyPreviewText,
textColumns = [],
- sampleRow = {},
enablePlaceholders = true,
}: PromptEditorProps) {
const {
@@ -37,33 +35,24 @@ export default function PromptEditor({
handleInput,
handleKeyDown,
insertMention,
- insertPlaceholder,
- usedColumns,
- orderedColumns,
previewText,
} = usePromptPlaceholderEditor({
value,
onChange,
previewMode,
- textColumns,
- sampleRow,
+ // Drop blank/whitespace column names so no empty `{}` placeholder or
+ // duplicate-empty-key mention option is ever produced.
+ textColumns: textColumns.filter((col) => col.trim() !== ""),
enablePlaceholders,
});
if (previewMode) {
return (
- {!value.trim() ? (
- {emptyPreviewText}
- ) : enablePlaceholders && Object.keys(sampleRow).length === 0 ? (
-
- Sample data not available. Go back to Datasets and choose a row with
- values.
-
+ {previewText ? (
+ previewText
) : (
- previewText || (
- {emptyPreviewText}
- )
+ {emptyPreviewText}
)}
);
@@ -72,31 +61,10 @@ export default function PromptEditor({
return (
<>
{enablePlaceholders && (
-
-
- Use `@` or tap a column chip to insert placeholders.
-
-
- {orderedColumns.map((col) => {
- const isUsed = usedColumns.includes(col);
- return (
- insertPlaceholder(col)}
- className={`!rounded-full !px-3 !py-1.5 !font-mono !text-xs ${
- isUsed
- ? "!border-status-success-border !bg-status-success-bg !text-status-success-text"
- : "!bg-bg-primary"
- }`}
- >
- {`{${col}}`}
-
- );
- })}
-
+
+ Type @ to mention a column and
+ insert it as a {"{column}"} {" "}
+ placeholder.
)}
diff --git a/app/components/assessment/prompt-config/PromptPanel.tsx b/app/components/assessment/prompt-config/PromptPanel.tsx
index 1c96ec3f..fe91f9a0 100644
--- a/app/components/assessment/prompt-config/PromptPanel.tsx
+++ b/app/components/assessment/prompt-config/PromptPanel.tsx
@@ -1,49 +1,18 @@
"use client";
-import { useState } from "react";
-import { RadioGroup } from "@/app/components/ui";
import type { PromptPanelProps } from "@/app/lib/types/assessment";
-
-type PromptViewMode = "edit" | "preview";
-import UserPrompt from "./UserPrompt";
import SystemPrompt from "./SystemPrompt";
export default function PromptPanel({
- textColumns,
- sampleRow,
systemInstruction,
setSystemInstruction,
- promptTemplate,
- setPromptTemplate,
}: PromptPanelProps) {
- const [viewMode, setViewMode] = useState
("edit");
- const previewMode = viewMode === "preview";
-
return (
-
-
- value={viewMode}
- onChange={setViewMode}
- ariaLabel="Prompt view mode"
- options={[
- { value: "edit", label: "Edit" },
- { value: "preview", label: "Preview" },
- ]}
- />
-
-
-
);
diff --git a/app/components/assessment/prompt-config/SystemPrompt.tsx b/app/components/assessment/prompt-config/SystemPrompt.tsx
index 596cbc2c..f77944b3 100644
--- a/app/components/assessment/prompt-config/SystemPrompt.tsx
+++ b/app/components/assessment/prompt-config/SystemPrompt.tsx
@@ -39,7 +39,7 @@ export default function SystemPrompt({
- Set the evaluation rules. Example: judge every answer fairly,
+ Set the assessment rules. Example: judge every answer fairly,
follow the scoring format, and explain the result briefly.
}
diff --git a/app/components/assessment/prompt-config/UserPrompt.tsx b/app/components/assessment/prompt-config/UserPrompt.tsx
index 1c5101ae..1392bf42 100644
--- a/app/components/assessment/prompt-config/UserPrompt.tsx
+++ b/app/components/assessment/prompt-config/UserPrompt.tsx
@@ -3,12 +3,11 @@
import { useState } from "react";
import { Button, InfoTooltip } from "@/app/components/ui";
import { ChevronDownIcon } from "@/app/components/icons";
-import type { SampleRow, ValueSetter } from "@/app/lib/types/assessment";
+import type { ValueSetter } from "@/app/lib/types/assessment";
import PromptEditor from "./PromptEditor";
interface UserPromptProps {
textColumns: string[];
- sampleRow: SampleRow;
promptTemplate: string;
setPromptTemplate: ValueSetter;
previewMode: boolean;
@@ -16,7 +15,6 @@ interface UserPromptProps {
export default function UserPrompt({
textColumns,
- sampleRow,
promptTemplate,
setPromptTemplate,
previewMode,
@@ -62,7 +60,6 @@ export default function UserPrompt({
onChange={setPromptTemplate}
previewMode={previewMode}
textColumns={textColumns}
- sampleRow={sampleRow}
enablePlaceholders
emptyPreviewText="Preview will appear here."
placeholder={`Describe what the AI should do.\n\nExample:\nEvaluate the student's answer.\nQuestion: {question}\nAnswer: {answer}\nContext: {context}`}
diff --git a/app/components/assessment/review/ExperimentReview.tsx b/app/components/assessment/review/ExperimentReview.tsx
index 8ed347fb..b7ad6ff9 100644
--- a/app/components/assessment/review/ExperimentReview.tsx
+++ b/app/components/assessment/review/ExperimentReview.tsx
@@ -22,7 +22,7 @@ export default function ExperimentReview({
className="!rounded-md !bg-neutral-50 !px-3 !py-2"
/>
- A descriptive name to identify this evaluation run.
+ A descriptive name to identify this assessment run.
);
diff --git a/app/components/assessment/review/PrefilterReview.tsx b/app/components/assessment/review/PrefilterReview.tsx
index 542d4e0b..f8218711 100644
--- a/app/components/assessment/review/PrefilterReview.tsx
+++ b/app/components/assessment/review/PrefilterReview.tsx
@@ -38,7 +38,7 @@ export default function PrefilterReview({
return (
- Submit Evaluation
+ Submit Assessment
>
)}
diff --git a/app/hooks/useAssessmentDatasetsTab.ts b/app/hooks/useAssessmentDatasetsTab.ts
index 319f3281..f37dc184 100644
--- a/app/hooks/useAssessmentDatasetsTab.ts
+++ b/app/hooks/useAssessmentDatasetsTab.ts
@@ -10,36 +10,31 @@ import {
fetchDatasetPreview,
handleForbiddenError,
isAllowedDatasetFile,
- isBlankCell,
} from "@/app/lib/utils/assessment";
import { PREVIEW_ROW_LIMIT } from "@/app/lib/assessment/results";
-import {
- DATASET_SAMPLE_ROW_LIMIT,
- MAX_DATASET_FILE_BYTES,
-} from "@/app/lib/assessment/constants";
+import { MAX_DATASET_FILE_BYTES } from "@/app/lib/assessment/constants";
import type {
CreateDatasetResponse,
DatasetPreview,
DatasetResponse,
- DatasetsTabProps,
DatasetViewModalData,
+ ValueSetter,
+ WithForbiddenHandler,
} from "@/app/lib/types/assessment";
-type UseAssessmentDatasetsTabParams = Pick<
- DatasetsTabProps,
- | "onForbidden"
- | "datasetId"
- | "setDatasetId"
- | "setSelectedDatasetName"
- | "onColumnsLoaded"
->;
+// Shared by the Datasets tab (library management) and the Experiment tab
+// (dataset selection at run time). Selection only records the dataset id/name;
+// the run's input_binding is derived from the chosen config's input_schema.
+interface UseAssessmentDatasetsTabParams extends WithForbiddenHandler {
+ datasetId: string;
+ setDatasetId: ValueSetter;
+ setSelectedDatasetName: ValueSetter;
+}
export interface UseAssessmentDatasetsTabResult {
datasets: Dataset[];
isLoading: boolean;
- isLoadingColumns: boolean;
viewingId: number | null;
- canProceed: boolean;
datasetName: string;
datasetDescription: string;
uploadedFile: File | null;
@@ -59,7 +54,7 @@ export interface UseAssessmentDatasetsTabResult {
handleFileSelect: (event: React.ChangeEvent) => void;
resetForm: () => void;
handleCreateDataset: () => Promise;
- handleDatasetSelect: (id: string, name?: string) => Promise;
+ handleDatasetSelect: (id: string, name?: string) => void;
handleViewDataset: (datasetId: number, name: string) => Promise;
handleDeleteDataset: (id: number) => Promise;
handleDrop: (event: React.DragEvent) => void;
@@ -70,7 +65,6 @@ export function useAssessmentDatasetsTab({
datasetId,
setDatasetId,
setSelectedDatasetName,
- onColumnsLoaded,
}: UseAssessmentDatasetsTabParams): UseAssessmentDatasetsTabResult {
const toast = useToast();
const { activeKey, isAuthenticated } = useAuth();
@@ -79,7 +73,6 @@ export function useAssessmentDatasetsTab({
const [datasets, setDatasets] = useState([]);
const [isLoading, setIsLoading] = useState(false);
- const [isLoadingColumns, setIsLoadingColumns] = useState(false);
const [datasetName, setDatasetName] = useState("");
const [datasetDescription, setDatasetDescription] = useState("");
const [uploadedFile, setUploadedFile] = useState(null);
@@ -156,58 +149,20 @@ export function useAssessmentDatasetsTab({
if (fileInputRef.current) fileInputRef.current.value = "";
};
- const handleDatasetSelect = async (id: string, name?: string) => {
+ // Records which dataset a run targets; no column loading — the run's
+ // input_binding comes from the selected config's input_schema.
+ const handleDatasetSelect = (id: string, name?: string) => {
setDatasetId(id);
if (!id) {
setSelectedDatasetName("");
- onColumnsLoaded([]);
return;
}
-
const resolvedName =
name ??
datasets.find((dataset) => dataset.dataset_id.toString() === id)
?.dataset_name ??
"";
setSelectedDatasetName(resolvedName);
-
- setIsLoadingColumns(true);
- try {
- const cached = previewCache[id];
- const parsed =
- cached ??
- (await fetchDatasetPreview(id, apiKey, DATASET_SAMPLE_ROW_LIMIT));
- if (!cached) {
- setPreviewCache((prev) => ({ ...prev, [id]: parsed }));
- }
- const keptIdx = parsed.headers
- .map((_, colIdx) => colIdx)
- .filter((colIdx) =>
- parsed.rows.some((row) => !isBlankCell(row[colIdx])),
- );
-
- const filteredHeaders = keptIdx.map((idx) => parsed.headers[idx]);
- const firstRow = parsed.rows[0] || [];
- const sampleRow = Object.fromEntries(
- keptIdx.map((idx) => [
- parsed.headers[idx],
- String(firstRow[idx] ?? ""),
- ]),
- );
- onColumnsLoaded(filteredHeaders, sampleRow);
- } catch (error) {
- if (handleForbiddenError(error, onForbidden)) return;
- const message =
- error instanceof Error
- ? error.message
- : "Failed to fetch dataset columns.";
- onColumnsLoaded([]);
- setDatasetId("");
- setSelectedDatasetName("");
- toast.error(message);
- } finally {
- setIsLoadingColumns(false);
- }
};
const handleCreateDataset = async () => {
@@ -323,7 +278,6 @@ export function useAssessmentDatasetsTab({
fileInputRef.current.dispatchEvent(new Event("change", { bubbles: true }));
};
- const canProceed = Boolean(datasetId) && !isLoadingColumns;
const datasetPendingDelete = datasets.find(
(dataset) => dataset.dataset_id === confirmDeleteId,
);
@@ -331,9 +285,7 @@ export function useAssessmentDatasetsTab({
return {
datasets,
isLoading,
- isLoadingColumns,
viewingId,
- canProceed,
datasetName,
datasetDescription,
uploadedFile,
diff --git a/app/hooks/useAssessmentWorkflow.ts b/app/hooks/useAssessmentWorkflow.ts
index 10e8ce30..f45466d7 100644
--- a/app/hooks/useAssessmentWorkflow.ts
+++ b/app/hooks/useAssessmentWorkflow.ts
@@ -9,29 +9,61 @@ import {
getAssessmentSubmitBlocker,
getAssessmentSubmitError,
handleForbiddenError,
- schemaToJsonSchema,
} from "@/app/lib/utils/assessment";
import { removeFeatureFromClient } from "@/app/lib/utils/features";
import { FeatureFlag } from "@/app/lib/constants";
-import { PAGE_TABS } from "@/app/lib/assessment/constants";
+import {
+ ASSESSMENT_TAB_ROUTES,
+ PAGE_TABS,
+} from "@/app/lib/assessment/constants";
import { useAssessmentDatasetStore } from "@/app/lib/store/assessment";
+import { assessmentBlobToBuilderState } from "@/app/lib/utils/assessmentFetcher";
import type {
- AssessmentFormState,
+ AssessmentRunConfigRef,
AssessmentTabId,
+ ColumnMapping,
+ ConfigDraftSeed,
ConfigSelection,
PageLayoutProps,
PrefilterConfig,
PostProcessingConfig,
SchemaProperty,
} from "@/app/lib/types/assessment";
+import type {
+ AssessmentColumnType,
+ AssessmentConfigBlob,
+} from "@/app/lib/types/configs";
+
+const EMPTY_COLUMN_MAPPING: ColumnMapping = {
+ textColumns: [],
+ attachments: [],
+ groundTruthColumns: [],
+};
export type UseAssessmentWorkflowResult = PageLayoutProps;
-export function useAssessmentWorkflow(): UseAssessmentWorkflowResult {
+export function useAssessmentWorkflow(
+ initialTab: AssessmentTabId = "datasets",
+): UseAssessmentWorkflowResult {
const router = useRouter();
const { error: showToastError, success: showToastSuccess } = useToast();
const { activeKey } = useAuth();
- const [activeTab, setActiveTab] = useState("datasets");
+ const [activeTab, setActiveTab] = useState(initialTab);
+
+ // The route (sidebar sub-tab) is the source of truth; keep state in sync when
+ // the page mounts/remounts on a different route.
+ useEffect(() => {
+ setActiveTab(initialTab);
+ }, [initialTab]);
+
+ // Switch tab AND navigate so the sidebar sub-item highlights.
+ const goToTab = useCallback(
+ (tab: AssessmentTabId) => {
+ setActiveTab(tab);
+ router.push(ASSESSMENT_TAB_ROUTES[tab]);
+ },
+ [router],
+ );
const [configStep, setConfigStep] = useState(1);
const [completedConfigSteps, setCompletedConfigSteps] = useState>(
new Set(),
@@ -39,18 +71,12 @@ export function useAssessmentWorkflow(): UseAssessmentWorkflowResult {
const [isSubmitting, setIsSubmitting] = useState(false);
const [experimentName, setExperimentName] = useState("");
const featureRedirectingRef = useRef(false);
- const {
- datasetId,
- datasetName,
- columns,
- sampleRow,
- columnMapping,
- setDatasetId,
- setDatasetName,
- setDataset,
- setColumnMapping,
- clearDataset,
- } = useAssessmentDatasetStore();
+ const { datasetId, datasetName, setDatasetId, setDatasetName, clearDataset } =
+ useAssessmentDatasetStore();
+ // Config input_schema is authored directly (dataset-independent) via the
+ // manual field editor, so the mapping starts empty and lives in the workflow.
+ const [columnMapping, setColumnMapping] =
+ useState(EMPTY_COLUMN_MAPPING);
const [promptTemplate, setPromptTemplate] = useState("");
const [systemInstruction, setSystemInstruction] = useState("");
const [outputSchema, setOutputSchema] = useState([]);
@@ -59,6 +85,45 @@ export function useAssessmentWorkflow(): UseAssessmentWorkflowResult {
useState(null);
const [postProcessingConfig, setPostProcessingConfig] =
useState(null);
+ // Seeds the Assessment step's provider/model draft + save mode when the
+ // Configuration step starts fresh or loads an existing version.
+ const [configSeed, setConfigSeed] = useState(null);
+ const seedNonce = useRef(0);
+
+ const startNewConfig = useCallback(() => {
+ setColumnMapping(EMPTY_COLUMN_MAPPING);
+ setSystemInstruction("");
+ setPromptTemplate("");
+ setOutputSchema([]);
+ setPrefilterConfig(null);
+ seedNonce.current += 1;
+ setConfigSeed({
+ blob: null,
+ saveMode: "new",
+ configId: "",
+ configName: "",
+ nonce: seedNonce.current,
+ });
+ }, []);
+
+ const loadExistingConfig = useCallback(
+ (blob: AssessmentConfigBlob, configId: string, configName: string) => {
+ const state = assessmentBlobToBuilderState(blob);
+ setColumnMapping(state.columnMapping);
+ setSystemInstruction(state.systemInstruction);
+ setOutputSchema(state.outputSchema);
+ setPrefilterConfig(state.prefilterConfig);
+ seedNonce.current += 1;
+ setConfigSeed({
+ blob: state.draft,
+ saveMode: "version",
+ configId,
+ configName,
+ nonce: seedNonce.current,
+ });
+ },
+ [],
+ );
const handleForbidden = useCallback(
(options?: { notify?: boolean }) => {
@@ -98,33 +163,19 @@ export function useAssessmentWorkflow(): UseAssessmentWorkflowResult {
[markConfigCompleted],
);
- const handleColumnsLoaded = useCallback(
- (loadedColumns: string[], firstRow: Record = {}) => {
- const currentId = useAssessmentDatasetStore.getState().datasetId;
- setDataset(currentId, loadedColumns, firstRow);
- setPromptTemplate("");
- },
- [setDataset],
- );
-
- useEffect(() => {
- setPrefilterConfig(null);
- }, [datasetId]);
-
- const outputSchemaJson = useMemo(
- () => schemaToJsonSchema(outputSchema),
- [outputSchema],
- );
+ // input_binding is derived from the selected config's stored input_schema so a
+ // saved config can run without re-authoring: text-typed keys become
+ // text_columns, image/pdf keys become url attachments.
+ const runInputSchema = configs[0]?.input_schema ?? {};
const handleSubmit = useCallback(async () => {
+ const inputSchemaEntries = Object.entries(runInputSchema);
const validationError = getAssessmentSubmitError({
datasetId,
- hasMapperSelection:
- columnMapping.textColumns.length > 0 ||
- columnMapping.attachments.length > 0,
- hasResponseFormat: outputSchema.some((field) => field.name.trim()),
+ hasInputSchema: inputSchemaEntries.length > 0,
configCount: configs.length,
experimentName,
+ hasPrompt: promptTemplate.trim().length > 0,
});
if (validationError) {
showToastError(validationError);
@@ -133,28 +184,35 @@ export function useAssessmentWorkflow(): UseAssessmentWorkflowResult {
setIsSubmitting(true);
try {
+ const runConfigs: AssessmentRunConfigRef[] = configs.map(
+ ({ config_id, config_version }) => ({
+ id: config_id,
+ version: config_version,
+ }),
+ );
+ const textColumns = inputSchemaEntries
+ .filter(([, column]) => column.type === "text")
+ .map(([name]) => name);
+ const attachments = inputSchemaEntries
+ .filter(([, column]) => column.type !== "text")
+ .map(([name, column]) => ({
+ column: name,
+ type: column.type as Exclude,
+ format: column.format ?? "url",
+ }));
await apiFetch("/api/assessment/runs", activeKey?.key ?? "", {
method: "POST",
body: JSON.stringify({
experiment_name: experimentName.trim(),
dataset_id: parseInt(datasetId, 10),
- prompt_template: promptTemplate || null,
- system_instruction: systemInstruction.trim() || null,
- text_columns: columnMapping.textColumns,
- attachments: columnMapping.attachments.map(
- ({ column, type, format, type_column, type_value_map }) => ({
- column,
- type,
- format,
- ...(type_column ? { type_column, type_value_map } : {}),
- }),
- ),
- output_schema: outputSchemaJson,
- configs: configs.map(({ config_id, config_version }) => ({
- config_id,
- config_version,
- })),
- prefilter_config: prefilterConfig ?? null,
+ input_binding: {
+ // Authored in the Experiment run step (not stored in the config);
+ // required non-empty because the legacy binding needs min_length 1.
+ prompt: promptTemplate,
+ text_columns: textColumns,
+ attachments,
+ },
+ configs: runConfigs,
post_processing_config: postProcessingConfig ?? null,
}),
});
@@ -164,13 +222,15 @@ export function useAssessmentWorkflow(): UseAssessmentWorkflowResult {
setCompletedConfigSteps(new Set());
setExperimentName("");
clearDataset();
+ setColumnMapping(EMPTY_COLUMN_MAPPING);
setSystemInstruction("");
setPromptTemplate("");
setOutputSchema([]);
setConfigs([]);
setPrefilterConfig(null);
+ setConfigSeed(null);
setPostProcessingConfig(null);
- setActiveTab("results");
+ goToTab("results");
} catch (error) {
if (handleForbiddenError(error, handleForbiddenWithNotify)) return;
showToastError(
@@ -181,38 +241,21 @@ export function useAssessmentWorkflow(): UseAssessmentWorkflowResult {
}
}, [
clearDataset,
- columnMapping,
configs,
datasetId,
experimentName,
+ goToTab,
handleForbiddenWithNotify,
- prefilterConfig,
- outputSchema,
- outputSchemaJson,
postProcessingConfig,
promptTemplate,
+ runInputSchema,
activeKey,
- systemInstruction,
showToastError,
showToastSuccess,
]);
- const formState: AssessmentFormState = {
- experimentName,
- datasetId,
- datasetName,
- columns,
- sampleRow,
- columnMapping,
- systemInstruction,
- promptTemplate,
- outputSchema,
- configs,
- prefilterConfig,
- postProcessingConfig,
- };
-
- const hasDataset = !!datasetId && columns.length > 0;
+ // Config-authoring progress indicators (Mapper builds input fields, Evaluation
+ // sets the response format).
const hasMapperSelection =
columnMapping.textColumns.length > 0 ||
columnMapping.attachments.length > 0;
@@ -220,75 +263,88 @@ export function useAssessmentWorkflow(): UseAssessmentWorkflowResult {
field.name.trim(),
);
const canReachReview = configs.length > 0 && hasConfiguredResponseFormat;
+
+ // Run readiness is driven by the selected config's stored input_schema.
+ const hasRunInputSchema = Object.keys(runInputSchema).length > 0;
+ const hasRunPrompt = promptTemplate.trim().length > 0;
const canSubmitAssessment =
!!datasetId &&
- hasMapperSelection &&
- hasConfiguredResponseFormat &&
configs.length > 0 &&
+ hasRunInputSchema &&
experimentName.trim().length > 0 &&
+ hasRunPrompt &&
!isSubmitting;
const submitBlockerMessage = getAssessmentSubmitBlocker({
datasetId,
- hasMapperSelection,
- hasResponseFormat: hasConfiguredResponseFormat,
+ hasInputSchema: hasRunInputSchema,
configCount: configs.length,
experimentName,
+ hasPrompt: hasRunPrompt,
});
const effectiveCompletedConfigSteps = useMemo(() => {
+ // Steps: 1 Configuration, 2 Input Schema, 3 Pre-filter (optional), 4 Assessment.
const merged = new Set(completedConfigSteps);
- if (hasMapperSelection) merged.add(1);
- if (hasMapperSelection) merged.add(2); // Prefilter is optional and always passable
- if (canReachReview) merged.add(3);
- if (canReachReview) merged.add(4); // Post Processing is optional and always passable
+ if (hasMapperSelection) {
+ merged.add(2);
+ merged.add(3); // Pre-filter is optional and always passable
+ }
+ if (canReachReview) merged.add(4);
return merged;
}, [canReachReview, completedConfigSteps, hasMapperSelection]);
return {
activeTab,
tabs: [...PAGE_TABS],
- onTabSwitch: setActiveTab,
+ onTabSwitch: goToTab,
datasetsTabProps: {
onForbidden: handleForbiddenWithNotify,
datasetId,
setDatasetId,
setSelectedDatasetName: setDatasetName,
- onColumnsLoaded: handleColumnsLoaded,
- onNext: () => {
- setActiveTab("config");
- setConfigStep(1);
- },
},
configPanelProps: {
- canSubmitAssessment,
- columns,
columnMapping,
completedSteps: effectiveCompletedConfigSteps,
configStep,
configs,
- datasetId,
- experimentName,
- formState,
- hasDataset,
- isSubmitting,
prefilterConfig,
outputSchema,
systemInstruction,
promptTemplate,
- sampleRow,
- setActiveTabToDatasets: () => setActiveTab("datasets"),
setColumnMapping,
setConfigStep,
setConfigs,
- setExperimentName,
setPrefilterConfig,
setOutputSchema,
setSystemInstruction,
setPromptTemplate,
- postProcessingConfig,
- setPostProcessingConfig,
+ onStepComplete: handleConfigNext,
+ configSeed,
+ onStartNewConfig: startNewConfig,
+ onLoadExistingConfig: loadExistingConfig,
+ onForbidden: handleForbiddenWithNotify,
+ },
+ experimentTabProps: {
+ onForbidden: handleForbiddenWithNotify,
+ textColumns: columnMapping.textColumns,
+ promptTemplate,
+ setPromptTemplate,
+ configs,
+ setConfigs,
+ outputSchema,
+ systemInstruction,
+ columnMapping,
+ prefilterConfig,
+ datasetId,
+ datasetName,
+ setDatasetId,
+ setSelectedDatasetName: setDatasetName,
+ experimentName,
+ setExperimentName,
+ isSubmitting,
+ canSubmit: canSubmitAssessment,
submitBlockerMessage,
onSubmit: handleSubmit,
- onStepComplete: handleConfigNext,
},
evaluationsTabProps: {
onForbidden: handleForbiddenWithNotify,
diff --git a/app/hooks/useConfigRunDetails.ts b/app/hooks/useConfigRunDetails.ts
index f742e0fc..759f058e 100644
--- a/app/hooks/useConfigRunDetails.ts
+++ b/app/hooks/useConfigRunDetails.ts
@@ -8,6 +8,7 @@ import {
} from "@/app/lib/utils/assessment";
import { ASSESSMENT_TAG } from "@/app/lib/assessment/constants";
import type {
+ AssessmentConfigBlob,
ConfigResponse,
ConfigVersionResponse,
} from "@/app/lib/types/configs";
@@ -83,16 +84,16 @@ export default function useConfigRunDetails({
);
}
+ const assessmentBlob = versionResponse.data
+ .config_blob as unknown as AssessmentConfigBlob | null;
const detail: ConfigRunDetail = {
configId,
version,
name: configResponse.data.name,
description: configResponse.data.description,
commitMessage: versionResponse.data.commit_message,
- provider:
- versionResponse.data.config_blob?.completion?.provider || null,
- model:
- versionResponse.data.config_blob?.completion?.params?.model || null,
+ provider: assessmentBlob?.assessment?.provider || null,
+ model: assessmentBlob?.assessment?.params?.model || null,
};
setConfigDetailsByKey((prev) => ({ ...prev, [key]: detail }));
diff --git a/app/hooks/usePromptAndConfigStep.ts b/app/hooks/usePromptAndConfigStep.ts
index a65f931d..206d075c 100644
--- a/app/hooks/usePromptAndConfigStep.ts
+++ b/app/hooks/usePromptAndConfigStep.ts
@@ -9,6 +9,7 @@ import {
MAX_CONFIGS,
} from "@/app/lib/assessment/constants";
import {
+ buildAssessmentConfigBlob,
buildDefaultParams,
buildInitialAssessmentConfigDraft,
buildInitialAssessmentVersionState,
@@ -22,21 +23,30 @@ import {
} from "@/app/lib/utils/assessmentFetcher";
import {
type ConfigMode,
+ type ConfigSaveMode,
type ConfigSelection,
type PromptAndConfigStepProps,
type UsePromptAndConfigStepResult,
type VersionListState,
} from "@/app/lib/types/assessment";
import type {
+ AssessmentConfigBlob,
CompletionConfig,
- ConfigBlob,
ConfigPublic,
} from "@/app/lib/types/configs";
import useLatestConfigModels from "@/app/hooks/useLatestConfigModels";
type UsePromptAndConfigStepParams = Pick<
PromptAndConfigStepProps,
- "textColumns" | "promptTemplate" | "configs" | "setConfigs" | "outputSchema"
+ | "textColumns"
+ | "promptTemplate"
+ | "configs"
+ | "setConfigs"
+ | "outputSchema"
+ | "systemInstruction"
+ | "columnMapping"
+ | "prefilterConfig"
+ | "configSeed"
>;
export function usePromptAndConfigStep({
@@ -45,6 +55,10 @@ export function usePromptAndConfigStep({
configs,
setConfigs,
outputSchema,
+ systemInstruction,
+ columnMapping,
+ prefilterConfig,
+ configSeed,
}: UsePromptAndConfigStepParams): UsePromptAndConfigStepResult {
const toast = useToast();
const { activeKey, isAuthenticated } = useAuth();
@@ -65,18 +79,32 @@ export function usePromptAndConfigStep({
>({});
const hasLoadedInitialConfigsRef = useRef(false);
- const [draft, setDraft] = useState(() =>
+ const [draft, setDraft] = useState(() =>
buildInitialAssessmentConfigDraft(),
);
const [configName, setConfigName] = useState("");
const [commitMessage, setCommitMessage] = useState("");
+ const [saveMode, setSaveMode] = useState("new");
+ const [versionConfigId, setVersionConfigId] = useState("");
const [isSaving, setIsSaving] = useState(false);
+ const appliedSeedNonce = useRef(null);
- const draftParams = draft.completion.params as Record<
+ // Apply a config seed from the Configuration step once per (re)load: hydrate
+ // the provider/model draft and preset the save mode (new vs new-version).
+ useEffect(() => {
+ if (!configSeed || appliedSeedNonce.current === configSeed.nonce) return;
+ appliedSeedNonce.current = configSeed.nonce;
+ if (configSeed.blob) setDraft(configSeed.blob);
+ setSaveMode(configSeed.saveMode);
+ setVersionConfigId(configSeed.configId);
+ setConfigName(configSeed.configName);
+ }, [configSeed]);
+
+ const draftParams = draft.assessment.params as Record<
string,
string | number | undefined
>;
- const currentProvider = draft.completion.provider ?? "openai";
+ const currentProvider = draft.assessment.provider ?? "openai";
const providerModels = useMemo(
() => getModelsByProvider(currentProvider),
[currentProvider],
@@ -93,6 +121,17 @@ export function usePromptAndConfigStep({
);
const namedSchemaFields = outputSchema.filter((field) => field.name.trim());
const hasConfiguredResponseFormat = namedSchemaFields.length > 0;
+ const configBlob = useMemo(
+ () =>
+ buildAssessmentConfigBlob({
+ draft,
+ systemInstruction,
+ outputSchema,
+ columnMapping,
+ prefilterConfig,
+ }),
+ [draft, systemInstruction, outputSchema, columnMapping, prefilterConfig],
+ );
const canProceed = configs.length > 0 && hasConfiguredResponseFormat;
const nextBlockerMessage =
configs.length === 0
@@ -288,9 +327,9 @@ export function usePromptAndConfigStep({
const updateDraftParam = (key: string, value: string | number) => {
setDraft((prev) => ({
...prev,
- completion: {
- ...prev.completion,
- params: { ...prev.completion.params, [key]: value },
+ assessment: {
+ ...prev.assessment,
+ params: { ...prev.assessment.params, [key]: value },
},
}));
};
@@ -299,11 +338,12 @@ export function usePromptAndConfigStep({
const defaultModel = getDefaultModelForProvider(provider);
setDraft((prev) => ({
...prev,
- completion: {
- ...prev.completion,
+ assessment: {
+ ...prev.assessment,
provider,
params: {
- instructions: String(prev.completion.params.instructions || ""),
+ instructions: String(prev.assessment.params.instructions || ""),
+ input_schema: prev.assessment.params.input_schema,
model: defaultModel,
...buildDefaultParams(defaultModel),
},
@@ -314,10 +354,11 @@ export function usePromptAndConfigStep({
const handleModelChange = (modelName: string) => {
setDraft((prev) => ({
...prev,
- completion: {
- ...prev.completion,
+ assessment: {
+ ...prev.assessment,
params: {
- instructions: String(prev.completion.params.instructions || ""),
+ instructions: String(prev.assessment.params.instructions || ""),
+ input_schema: prev.assessment.params.input_schema,
model: modelName,
...buildDefaultParams(modelName),
},
@@ -330,38 +371,55 @@ export function usePromptAndConfigStep({
toast.error("Please sign in to create configurations");
return;
}
- if (!configName.trim()) {
+ const isVersion = saveMode === "version";
+ const targetConfig = isVersion
+ ? (configCards.find((c) => c.id === versionConfigId) ?? null)
+ : null;
+ if (isVersion) {
+ if (!targetConfig) {
+ toast.error("Select a configuration to version");
+ return;
+ }
+ } else if (!configName.trim()) {
toast.error("Configuration name is required");
return;
}
setIsSaving(true);
try {
- const existingConfig =
- configCards.find(
- (c) =>
- c.name.trim().toLowerCase() === configName.trim().toLowerCase(),
- ) ?? null;
+ const configBlob = buildAssessmentConfigBlob({
+ draft,
+ systemInstruction,
+ outputSchema,
+ columnMapping,
+ prefilterConfig,
+ });
const saved = await saveAssessmentConfig({
apiKey,
- configName: configName.trim(),
+ configName: isVersion ? targetConfig!.name : configName.trim(),
commitMessage: commitMessage.trim(),
- configBlob: draft,
- existingConfig: existingConfig
- ? { id: existingConfig.id, name: existingConfig.name }
+ configBlob,
+ existingConfig: isVersion
+ ? { id: targetConfig!.id, name: targetConfig!.name }
: null,
});
addSelection({
config_id: saved.config_id,
config_version: saved.config_version,
- name: configName.trim(),
- provider: draft.completion.provider,
+ name: saved.name ?? configName.trim(),
+ provider: draft.assessment.provider,
model: currentModel,
});
setDraft(buildInitialAssessmentConfigDraft());
setConfigName("");
setCommitMessage("");
+ setVersionConfigId("");
+ setSaveMode("new");
setConfigMode("existing");
- toast.success("Configuration saved and added!");
+ toast.success(
+ isVersion
+ ? "New version saved and added!"
+ : "Configuration saved and added!",
+ );
void loadConfigs(0, true);
} catch (err) {
toast.error(
@@ -404,11 +462,16 @@ export function usePromptAndConfigStep({
configName,
commitMessage,
isSaving,
+ saveMode,
+ setSaveMode,
+ versionConfigId,
+ setVersionConfigId,
setConfigName,
setCommitMessage,
handleProviderChange,
handleModelChange,
updateDraftParam,
handleCreateAndAdd,
+ configBlob,
};
}
diff --git a/app/hooks/usePromptPlaceholderEditor.ts b/app/hooks/usePromptPlaceholderEditor.ts
index 2ffd69bc..7b5d5691 100644
--- a/app/hooks/usePromptPlaceholderEditor.ts
+++ b/app/hooks/usePromptPlaceholderEditor.ts
@@ -9,14 +9,13 @@ import {
useRef,
useState,
} from "react";
-import type { SampleRow, ValueSetter } from "@/app/lib/types/assessment";
+import type { ValueSetter } from "@/app/lib/types/assessment";
interface UsePromptPlaceholderEditorParams {
value: string;
onChange: ValueSetter;
previewMode: boolean;
textColumns: string[];
- sampleRow: SampleRow;
enablePlaceholders: boolean;
}
@@ -43,7 +42,6 @@ export function usePromptPlaceholderEditor({
onChange,
previewMode,
textColumns,
- sampleRow,
enablePlaceholders,
}: UsePromptPlaceholderEditorParams): UsePromptPlaceholderEditorResult {
const [mentionQuery, setMentionQuery] = useState(null);
@@ -257,20 +255,9 @@ export function usePromptPlaceholderEditor({
return [...used, ...unused];
}, [textColumns, usedColumns]);
- const previewText = useMemo(() => {
- if (!value.trim()) return "";
- if (!enablePlaceholders) return value;
-
- let next = value;
- textColumns.forEach((col) => {
- const safe = col.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
- next = next.replace(
- new RegExp(`\\{${safe}\\}`, "g"),
- sampleRow[col] || "",
- );
- });
- return next;
- }, [enablePlaceholders, sampleRow, textColumns, value]);
+ // No dataset sample data in the dataset-independent flow, so the preview shows
+ // the composed prompt as written (placeholders remain visible).
+ const previewText = useMemo(() => (value.trim() ? value : ""), [value]);
return {
textareaRef,
diff --git a/app/lib/assessment/constants.ts b/app/lib/assessment/constants.ts
index a3bae083..86a9fb1a 100644
--- a/app/lib/assessment/constants.ts
+++ b/app/lib/assessment/constants.ts
@@ -1,5 +1,6 @@
import type {
AssessmentTab,
+ AssessmentTabId,
ColumnRole,
PostProcessingConfig,
PostProcessingFilterRule,
@@ -91,11 +92,10 @@ export const SUMMARY_BADGE_CLASSES: Record = {
};
export const ASSESSMENT_CONFIG_STEPS: Step[] = [
- { id: 1, label: "Mapper" },
- { id: 2, label: "Eliminatory (opt.)" },
- { id: 3, label: "Evaluation" },
- { id: 4, label: "Post Processing (opt.)" },
- { id: 5, label: "Review" },
+ { id: 1, label: "Configuration" },
+ { id: 2, label: "Input Schema" },
+ { id: 3, label: "Pre-filter (opt.)" },
+ { id: 4, label: "Assessment" },
];
export const SCHEMA_TYPE_OPTIONS: Array<{
@@ -250,11 +250,29 @@ export const CONFIGS_VISIBLE_BATCH_SIZE = 2;
export const PAGE_TABS: ReadonlyArray = [
{ id: "datasets", label: "Datasets" },
{ id: "config", label: "Config" },
- { id: "results", label: "Result" },
+ { id: "experiment", label: "Experiment" },
+ { id: "results", label: "Runs" },
];
export const MAX_CONFIGS = 4;
+// Each tab is its own route (sidebar sub-item). The `results` tab id keeps its
+// name for churn reasons but lives at the /assessment/runs URL.
+export const ASSESSMENT_TAB_ROUTES: Record = {
+ datasets: "/assessment",
+ config: "/assessment/config",
+ experiment: "/assessment/experiment",
+ results: "/assessment/runs",
+};
+
+// Reverse map from the /assessment/[tab] URL segment to the tab id.
+export const ASSESSMENT_ROUTE_SEGMENT_TO_TAB: Record =
+ {
+ config: "config",
+ experiment: "experiment",
+ runs: "results",
+ };
+
export const ATTACHMENT_FORMATS: Record = {
mixed: ["url", "base64"],
image: ["url", "base64"],
diff --git a/app/lib/assessment/results.ts b/app/lib/assessment/results.ts
index 91c1b808..90aacb34 100644
--- a/app/lib/assessment/results.ts
+++ b/app/lib/assessment/results.ts
@@ -67,16 +67,18 @@ export function hasViewableResults(run: AssessmentChildRun): boolean {
return getStageProgress(run).some((s) => s.status === "completed");
}
+// Backend status enums serialize UPPERCASE (e.g. "COMPLETED"); the status sets
+// are lowercase — compare case-insensitively so tones/badges resolve correctly.
export function isActiveStatus(status: string): boolean {
- return ACTIVE_ASSESSMENT_STATUSES.has(status);
+ return ACTIVE_ASSESSMENT_STATUSES.has(status.toLowerCase());
}
export function isFailedStatus(status: string): boolean {
- return FAILED_ASSESSMENT_STATUSES.has(status);
+ return FAILED_ASSESSMENT_STATUSES.has(status.toLowerCase());
}
export function isCompletedStatus(status: string): boolean {
- return COMPLETED_ASSESSMENT_STATUSES.has(status);
+ return COMPLETED_ASSESSMENT_STATUSES.has(status.toLowerCase());
}
export function canRetryStatus(status: string): boolean {
@@ -127,9 +129,10 @@ export function normalizeAssessmentRun(run: AssessmentRun): AssessmentRun {
}
export function getResultTone(status: string): ResultTone {
- if (isCompletedStatus(status)) return "success";
- if (status === "failed" || status === "prefilter_failed") return "error";
- if (isActiveStatus(status) || status === "completed_with_errors") {
+ const s = status.toLowerCase();
+ if (isCompletedStatus(s)) return "success";
+ if (s === "failed" || s === "prefilter_failed") return "error";
+ if (isActiveStatus(s) || s === "completed_with_errors") {
return "warning";
}
return "default";
@@ -162,7 +165,7 @@ export function filterAssessments(
return assessments.filter((run) => {
if (statusFilter === "processing") return isActiveStatus(run.status);
if (statusFilter === "failed") return isFailedStatus(run.status);
- return run.status === statusFilter;
+ return run.status.toLowerCase() === statusFilter;
});
}
diff --git a/app/lib/data/assessmentModels.ts b/app/lib/data/assessmentModels.ts
index f695ca04..b950f59b 100644
--- a/app/lib/data/assessmentModels.ts
+++ b/app/lib/data/assessmentModels.ts
@@ -3,7 +3,7 @@ import type {
ConfigParamDefinition,
ModelOption,
} from "@/app/lib/types/assessment";
-import type { ConfigBlob } from "@/app/lib/types/configs";
+import type { AssessmentConfigBlob } from "@/app/lib/types/configs";
export const GPT4_STYLE_CONFIG = {
top_p: {
@@ -376,13 +376,14 @@ export function buildDefaultParams(
);
}
-export const ASSESSMENT_DEFAULT_CONFIG: ConfigBlob = {
- completion: {
+export const ASSESSMENT_DEFAULT_CONFIG: AssessmentConfigBlob = {
+ assessment: {
provider: "openai",
type: "text",
params: {
model: "gpt-4o-mini",
instructions: "",
+ input_schema: {},
...buildDefaultParams("gpt-4o-mini"),
},
},
diff --git a/app/lib/navConfig.ts b/app/lib/navConfig.ts
index 6ddad7d3..e0a87404 100644
--- a/app/lib/navConfig.ts
+++ b/app/lib/navConfig.ts
@@ -59,9 +59,14 @@ export const NAV_ITEMS: NavItemConfig[] = [
},
{
name: "Assessment",
- route: "/assessment",
icon: "assessment",
featureFlag: FeatureFlag.ASSESSMENT,
+ submenu: [
+ { name: "Datasets", route: "/assessment" },
+ { name: "Config", route: "/assessment/config" },
+ { name: "Experiment", route: "/assessment/experiment" },
+ { name: "Runs", route: "/assessment/runs" },
+ ],
gateDescription: "Log in to run assessments.",
},
{
diff --git a/app/lib/store/assessment.ts b/app/lib/store/assessment.ts
index d29864be..bbb8da7c 100644
--- a/app/lib/store/assessment.ts
+++ b/app/lib/store/assessment.ts
@@ -1,46 +1,21 @@
-// Zustand store for the selected assessment dataset: id, columns, sample row, and column mapping.
+// Zustand store for the selected assessment dataset: id + name only. A dataset
+// is chosen in the Experiment tab to run a config against; the config itself is
+// authored dataset-independently, so no columns/mapping live here.
import { create } from "zustand";
-import type {
- AssessmentDatasetState,
- ColumnMapping,
-} from "@/app/lib/types/assessment";
-
-const DEFAULT_MAPPING: ColumnMapping = {
- textColumns: [],
- attachments: [],
- groundTruthColumns: [],
-};
+import type { AssessmentDatasetState } from "@/app/lib/types/assessment";
export const useAssessmentDatasetStore = create()(
(set) => ({
datasetId: "",
datasetName: "",
- columns: [],
- sampleRow: {},
- columnMapping: DEFAULT_MAPPING,
setDatasetId: (id) => set({ datasetId: id }),
setDatasetName: (name) => set({ datasetName: name }),
- setDataset: (datasetId, columns, sampleRow, datasetName) =>
- set((state) => ({
- datasetId,
- datasetName: datasetName ?? state.datasetName,
- columns,
- sampleRow,
- columnMapping:
- datasetId !== state.datasetId ? DEFAULT_MAPPING : state.columnMapping,
- })),
-
- setColumnMapping: (mapping) => set({ columnMapping: mapping }),
-
clearDataset: () =>
set({
datasetId: "",
datasetName: "",
- columns: [],
- sampleRow: {},
- columnMapping: DEFAULT_MAPPING,
}),
}),
);
diff --git a/app/lib/types/assessment/config.ts b/app/lib/types/assessment/config.ts
index 72ecd8b5..2140e37d 100644
--- a/app/lib/types/assessment/config.ts
+++ b/app/lib/types/assessment/config.ts
@@ -1,5 +1,7 @@
// Assessment types: model configurations, versions, and config-selection UI.
import type {
+ AssessmentConfigBlob,
+ AssessmentInputSchemaColumn,
CompletionConfig,
ConfigPublic,
ConfigVersionItems,
@@ -16,6 +18,14 @@ export interface ConfigSelection extends ConfigRef {
name?: string;
provider?: string;
model?: string;
+ // Stored input_schema of the selected config version, used to build the run's
+ // input_binding without re-authoring a dataset mapping.
+ input_schema?: Record;
+}
+
+export interface AssessmentRunConfigRef {
+ id: string;
+ version: number;
}
export type ConfigParamType = "float" | "int" | "enum";
@@ -39,6 +49,9 @@ export type ModelOption = LabeledValue;
export type ConfigMode = "existing" | "create";
+// Config-tab save modal: create a brand-new config vs. a new version of one.
+export type ConfigSaveMode = "new" | "version";
+
export interface VersionListState {
items: ConfigVersionItems[];
isLoading: boolean;
@@ -83,6 +96,16 @@ export interface ConfigCreatorProps {
configName: string;
commitMessage: string;
isSaving: boolean;
+ // Save modal open-state is lifted so the trigger button can live in the
+ // step-level footer while the modal renders here.
+ isSaveModalOpen: boolean;
+ setIsSaveModalOpen: ValueSetter;
+ // Save modal: new config vs. new version of an existing config.
+ saveMode: ConfigSaveMode;
+ setSaveMode: ValueSetter;
+ versionConfigId: string;
+ setVersionConfigId: ValueSetter;
+ existingConfigs: ConfigPublic[];
setConfigName: ValueSetter;
setCommitMessage: ValueSetter;
onProviderChange: ValueSetter;
@@ -126,39 +149,24 @@ export interface UsePromptAndConfigStepResult {
configName: string;
commitMessage: string;
isSaving: boolean;
+ saveMode: ConfigSaveMode;
+ setSaveMode: ValueSetter;
+ versionConfigId: string;
+ setVersionConfigId: ValueSetter;
setConfigName: ValueSetter;
setCommitMessage: ValueSetter;
handleProviderChange: (provider: CompletionConfig["provider"]) => void;
handleModelChange: (modelName: string) => void;
updateDraftParam: (key: string, value: string | number) => void;
handleCreateAndAdd: () => Promise;
+ configBlob: AssessmentConfigBlob;
}
+// The Config tab renders only the "new configuration" builder (ConfigCreator);
+// selecting an existing saved config to run lives in the Experiment tab.
export interface AssessmentConfigurationProps extends Omit<
ConfigCreatorProps,
"onSave"
> {
- configMode: ConfigMode;
- setConfigMode: ValueSetter;
- configs: ConfigSelection[];
- onRemoveConfig: (configId: string, version: number) => void;
- configCards: ConfigPublic[];
- searchQuery: string;
- setSearchQuery: ValueSetter;
- isLoadingConfigs: boolean;
- hasMoreConfigs: boolean;
- nextConfigSkip: number;
- expandedConfigId: string | null;
- versionStateByConfig: Record;
- latestModelByConfig: Record;
- loadingSelectionKeys: Record;
- isSelected: (configId: string, version: number) => boolean;
- onLoadMoreConfigs: (skip: number) => void | Promise;
- onLoadVersions: (configId: string, skip: number) => void;
- onToggleConfigExpansion: ValueSetter;
- onToggleVersionSelection: (
- config: ConfigPublic,
- version: number,
- ) => void | Promise;
onSaveConfig: () => void | Promise;
}
diff --git a/app/lib/types/assessment/core.ts b/app/lib/types/assessment/core.ts
index e4675fd6..c75f5889 100644
--- a/app/lib/types/assessment/core.ts
+++ b/app/lib/types/assessment/core.ts
@@ -21,7 +21,7 @@ export interface PagedResult {
nextSkip: number;
}
-export type AssessmentTabId = "datasets" | "config" | "results";
+export type AssessmentTabId = "datasets" | "config" | "experiment" | "results";
export interface AssessmentTab {
id: AssessmentTabId;
label: string;
diff --git a/app/lib/types/assessment/dataset.ts b/app/lib/types/assessment/dataset.ts
index ded47968..b26cab1e 100644
--- a/app/lib/types/assessment/dataset.ts
+++ b/app/lib/types/assessment/dataset.ts
@@ -4,7 +4,6 @@ import type {
CreateResponse,
LabeledValue,
ListResponse,
- SampleRow,
StepNavigationProps,
ValueSetter,
} from "./core";
@@ -22,6 +21,9 @@ export interface ColumnMapping {
textColumns: string[];
attachments: Attachment[];
groundTruthColumns: string[];
+ // Names of columns marked strict (required in every submission row). Any
+ // column not listed is optional (strict: false).
+ strictColumns?: string[];
}
export type ColumnRole = "unmapped" | "text" | "attachment" | "ground_truth";
@@ -65,6 +67,8 @@ export interface PrefilterStepProps extends StepNavigationProps {
attachmentColumns?: string[];
prefilterConfig: PrefilterConfig | null;
setPrefilterConfig: ValueSetter;
+ // Changes when a config is (re)loaded, so local state re-syncs from props.
+ syncToken?: number;
}
export interface ColumnConfig {
@@ -83,27 +87,21 @@ export interface RoleVisuals {
activeButtonClass: string;
}
-export interface ColumnMapperStepProps extends StepNavigationProps {
- columns: string[];
+// The Mapper authors the config's input_schema directly as a manual field list;
+// it no longer maps a dataset's columns, so it takes no `columns` and no back nav.
+export interface ColumnMapperStepProps {
columnMapping: ColumnMapping;
setColumnMapping: ValueSetter;
+ onNext: () => void;
+ // Changes when a config is (re)loaded, so the local field list re-syncs.
+ syncToken?: number;
}
export interface AssessmentDatasetState {
datasetId: string;
datasetName: string;
- columns: string[];
- sampleRow: SampleRow;
- columnMapping: ColumnMapping;
setDatasetId: ValueSetter;
setDatasetName: ValueSetter;
- setDataset: (
- datasetId: string,
- columns: string[],
- sampleRow: SampleRow,
- datasetName?: string,
- ) => void;
- setColumnMapping: ValueSetter;
clearDataset: () => void;
}
@@ -144,6 +142,13 @@ export interface DatasetViewModalData {
rows: string[][];
}
+// GET /api/assessment/datasets/{id}/rows — all rows, column-keyed.
+export interface AssessmentDatasetRows {
+ headers: string[];
+ rows: Record[];
+ total_rows: number;
+}
+
export type ReviewColumnRole = "text" | "attachment" | "ground truth";
export interface ReviewColumn {
diff --git a/app/lib/types/assessment/results.ts b/app/lib/types/assessment/results.ts
index 13f1cc9d..2e9a4aea 100644
--- a/app/lib/types/assessment/results.ts
+++ b/app/lib/types/assessment/results.ts
@@ -26,6 +26,19 @@ export interface AssessmentRunStat {
stage_status: string | null;
}
+export interface AssessmentPreFilterCost {
+ topic_relevance?: number | null;
+ duplicate_detection?: number | null;
+ total: number;
+}
+
+export interface AssessmentCost {
+ pre_filter: AssessmentPreFilterCost;
+ assessment: number;
+ total: number;
+ currency: string;
+}
+
export interface AssessmentRun {
id: number;
experiment_name: string;
@@ -37,6 +50,7 @@ export interface AssessmentRun {
processing_runs: number;
completed_runs: number;
failed_runs: number;
+ counts?: ResultsCounts | null;
run_stats: AssessmentRunStat[];
error_message: string | null;
inserted_at: string;
@@ -79,11 +93,14 @@ export interface AssessmentChildRun {
id: number;
assessment_id: number | null;
run_name: string;
+ experiment_name?: string | null;
dataset_name: string | null;
dataset_id: number | null;
config_id: string | null;
config_version: number | null;
status: string;
+ input?: Record | null;
+ cost?: AssessmentCost | null;
total_items: number;
error_message: string | null;
organization_id: number;
diff --git a/app/lib/types/assessment/workflow.ts b/app/lib/types/assessment/workflow.ts
index 64e2293b..e4a41111 100644
--- a/app/lib/types/assessment/workflow.ts
+++ b/app/lib/types/assessment/workflow.ts
@@ -8,9 +8,37 @@ import type {
ValueSetter,
WithForbiddenHandler,
} from "./core";
-import type { ConfigSelection } from "./config";
+import type { ConfigSelection, ConfigSaveMode } from "./config";
import type { ColumnMapping, PrefilterConfig, SchemaProperty } from "./dataset";
import type { PostProcessingConfig } from "./results";
+import type { AssessmentConfigBlob } from "@/app/lib/types/configs";
+
+// One-shot seed passed from the Configuration step to the Assessment step so a
+// loaded config version hydrates the provider/model draft + save mode. `nonce`
+// changes on every (re)load so the effect that applies it fires once per load.
+export interface ConfigDraftSeed {
+ blob: AssessmentConfigBlob | null;
+ saveMode: ConfigSaveMode;
+ configId: string;
+ configName: string;
+ nonce: number;
+}
+
+// Config flow step 1: start a new configuration or load an existing one (and a
+// specific version) to edit into a new version.
+export interface ConfigSelectStepProps {
+ onStartNew: () => void;
+ onLoadExisting: (
+ blob: AssessmentConfigBlob,
+ configId: string,
+ configName: string,
+ ) => void;
+ onForbidden: () => void;
+ onNext: () => void;
+ // Which config scope to list/load. ASSESSMENT when rendered from the
+ // assessment flow; default when reused from the configuration library.
+ tag?: "default" | "ASSESSMENT";
+}
export interface AssessmentFormState {
experimentName: string;
@@ -28,12 +56,8 @@ export interface AssessmentFormState {
}
export interface PromptPanelProps {
- textColumns: string[];
- sampleRow: SampleRow;
systemInstruction: string;
setSystemInstruction: ValueSetter;
- promptTemplate: string;
- setPromptTemplate: ValueSetter;
}
export interface ResponseSchemaProps {
@@ -43,47 +67,70 @@ export interface ResponseSchemaProps {
hasFields: boolean;
}
+// Config tab authors + saves a dataset-independent config (Mapper builds the
+// input_schema directly -> Eliminatory -> Evaluation). Run submission lives in
+// the Experiment tab, so no dataset/submit/postprocessing here.
export interface ConfigPanelProps {
- canSubmitAssessment: boolean;
- columns: string[];
columnMapping: ColumnMapping;
completedSteps: Set;
configStep: number;
configs: ConfigSelection[];
- datasetId: string | null;
- experimentName: string;
- formState: AssessmentFormState;
- hasDataset: boolean;
- isSubmitting: boolean;
prefilterConfig: PrefilterConfig | null;
outputSchema: SchemaProperty[];
systemInstruction: string;
promptTemplate: string;
- sampleRow: SampleRow;
- setActiveTabToDatasets: () => void;
setColumnMapping: ValueSetter;
setConfigStep: ValueSetter;
setConfigs: StateSetter;
- setExperimentName: ValueSetter;
setPrefilterConfig: ValueSetter;
setOutputSchema: ValueSetter;
setSystemInstruction: ValueSetter;
setPromptTemplate: ValueSetter;
- postProcessingConfig: PostProcessingConfig | null;
- setPostProcessingConfig: ValueSetter;
+ onStepComplete: ValueSetter;
+ configSeed: ConfigDraftSeed | null;
+ onStartNewConfig: () => void;
+ onLoadExistingConfig: (
+ blob: AssessmentConfigBlob,
+ configId: string,
+ configName: string,
+ ) => void;
+ onForbidden: () => void;
+}
+
+// Experiment tab: pick a saved config + a dataset, then dispatch a run.
+export interface ExperimentTabProps extends WithForbiddenHandler {
+ // Saved-config picker (fed to usePromptAndConfigStep).
+ textColumns: string[];
+ promptTemplate: string;
+ setPromptTemplate: ValueSetter;
+ configs: ConfigSelection[];
+ setConfigs: StateSetter;
+ outputSchema: SchemaProperty[];
+ systemInstruction: string;
+ columnMapping: ColumnMapping;
+ prefilterConfig: PrefilterConfig | null;
+ // Dataset picker (fed to useAssessmentDatasetsTab).
+ datasetId: string;
+ datasetName: string;
+ setDatasetId: ValueSetter;
+ setSelectedDatasetName: ValueSetter;
+ // Run dispatch.
+ experimentName: string;
+ setExperimentName: ValueSetter;
+ isSubmitting: boolean;
+ canSubmit: boolean;
submitBlockerMessage: string;
onSubmit: () => void;
- onStepComplete: ValueSetter;
}
export type EvaluationsTabProps = WithForbiddenHandler;
+// The Datasets tab manages the dataset library (list / view / create / delete).
+// Dataset selection for a run happens in the Experiment tab, not here.
export interface DatasetsTabProps extends WithForbiddenHandler {
datasetId: string;
setDatasetId: ValueSetter;
setSelectedDatasetName: ValueSetter;
- onColumnsLoaded: (columns: string[], sampleRow?: SampleRow) => void;
- onNext: () => void;
}
export interface PageLayoutProps {
@@ -92,6 +139,7 @@ export interface PageLayoutProps {
onTabSwitch: ValueSetter;
datasetsTabProps: DatasetsTabProps;
configPanelProps: ConfigPanelProps;
+ experimentTabProps: ExperimentTabProps;
evaluationsTabProps: EvaluationsTabProps;
}
@@ -111,7 +159,6 @@ export interface PostProcessingPanelProps {
export interface PromptAndConfigStepProps extends StepNavigationProps {
textColumns: string[];
- sampleRow: SampleRow;
systemInstruction: string;
setSystemInstruction: ValueSetter;
promptTemplate: string;
@@ -120,4 +167,7 @@ export interface PromptAndConfigStepProps extends StepNavigationProps {
setConfigs: StateSetter;
outputSchema: SchemaProperty[];
setOutputSchema: ValueSetter;
+ columnMapping: ColumnMapping;
+ prefilterConfig: PrefilterConfig | null;
+ configSeed?: ConfigDraftSeed | null;
}
diff --git a/app/lib/types/configs.ts b/app/lib/types/configs.ts
index 1ab8a890..e08368cf 100644
--- a/app/lib/types/configs.ts
+++ b/app/lib/types/configs.ts
@@ -90,10 +90,56 @@ export interface ConfigBlob {
output_guardrails?: GuardrailRef[];
}
+export type AssessmentColumnType = "text" | "image" | "pdf";
+
+export interface AssessmentInputSchemaColumn {
+ type: AssessmentColumnType;
+ format?: "url";
+ // Whether the column must be present in every submission row (default false).
+ strict?: boolean;
+}
+
+export interface AssessmentParams {
+ model: string;
+ instructions: string;
+ input_schema: Record;
+ json_output_schema?: object | null;
+ [key: string]: unknown;
+}
+
+export interface AssessmentModelBlock {
+ provider: ProviderType;
+ type: "text";
+ params: AssessmentParams;
+}
+
+export interface AssessmentPreFilterParams {
+ model?: string;
+ instructions: string;
+ [key: string]: unknown;
+}
+
+export interface AssessmentPreFilter {
+ provider: ProviderType;
+ params: AssessmentPreFilterParams;
+ stop_on_fail: boolean;
+ knowledge_base_id?: string;
+}
+
+export interface AssessmentPreFilters {
+ topic_relevance?: AssessmentPreFilter;
+ duplicate_detection?: AssessmentPreFilter;
+}
+
+export interface AssessmentConfigBlob {
+ pre_filters?: AssessmentPreFilters;
+ assessment: AssessmentModelBlock;
+}
+
export interface ConfigCreate {
name: string;
description?: string | null;
- config_blob: ConfigBlob;
+ config_blob: ConfigBlob | AssessmentConfigBlob;
commit_message?: string | null;
tag?: ConfigTag | null;
}
@@ -105,7 +151,7 @@ export interface ConfigUpdate {
}
export interface ConfigVersionCreate {
- config_blob: ConfigBlob;
+ config_blob: ConfigBlob | AssessmentConfigBlob;
commit_message?: string | null;
}
diff --git a/app/lib/utils/assessment.ts b/app/lib/utils/assessment.ts
index 383dbb4a..40b3d11d 100644
--- a/app/lib/utils/assessment.ts
+++ b/app/lib/utils/assessment.ts
@@ -239,45 +239,40 @@ export function isBlankCell(cell: string | undefined): boolean {
return cell == null || String(cell).trim() === "";
}
+// A run pairs a saved config with a dataset. The config already carries its
+// input_schema + output schema, so the run only validates dataset, config
+// selection, that the chosen config defines input fields, and a run name.
interface AssessmentSubmitChecks {
datasetId: string | null;
- hasMapperSelection: boolean;
- hasResponseFormat: boolean;
+ hasInputSchema: boolean;
configCount: number;
experimentName: string;
+ hasPrompt: boolean;
}
export function getAssessmentSubmitError(
checks: AssessmentSubmitChecks,
): string | null {
if (!checks.datasetId) return "Dataset is required";
- if (!checks.hasMapperSelection)
- return "Map at least one text or attachment column";
- if (!checks.hasResponseFormat) return "Response format is required";
if (checks.configCount === 0) return "Select at least one configuration";
+ if (!checks.hasInputSchema)
+ return "Selected configuration has no input fields";
if (!checks.experimentName.trim()) return "Experiment name is required";
+ if (!checks.hasPrompt) return "Enter a user prompt";
return null;
}
-interface AssessmentSubmitBlockerChecks {
- datasetId: string | null;
- hasMapperSelection: boolean;
- hasResponseFormat: boolean;
- configCount: number;
- experimentName: string;
-}
-
export function getAssessmentSubmitBlocker(
- checks: AssessmentSubmitBlockerChecks,
+ checks: AssessmentSubmitChecks,
): string {
if (!checks.datasetId) return "Select a dataset to submit";
- if (!checks.hasMapperSelection)
- return "Map at least one text or attachment column to submit";
- if (!checks.hasResponseFormat) return "Set response format to submit";
if (checks.configCount === 0)
return "Select at least one configuration to submit";
+ if (!checks.hasInputSchema)
+ return "Selected configuration has no input fields";
if (!checks.experimentName.trim())
return "Enter an experiment name to submit";
+ if (!checks.hasPrompt) return "Enter a user prompt to submit";
return "";
}
diff --git a/app/lib/utils/assessmentFetcher.ts b/app/lib/utils/assessmentFetcher.ts
index 46635e72..e8110224 100644
--- a/app/lib/utils/assessmentFetcher.ts
+++ b/app/lib/utils/assessmentFetcher.ts
@@ -8,16 +8,25 @@ import {
GPT4_STYLE_CONFIG,
} from "@/app/lib/data/assessmentModels";
import { DEFAULT_PAGE_LIMIT } from "@/app/lib/constants";
+import { schemaToJsonSchema } from "@/app/lib/utils/assessment";
+import { fromJsonSchema } from "@/app/lib/utils/outputSchema";
import type {
+ AssessmentDatasetRows,
+ Attachment,
+ ColumnMapping,
ConfigParamDefinition,
ConfigSelection,
ModelOption,
PagedResult,
+ PrefilterConfig,
+ SchemaProperty,
VersionListState,
} from "@/app/lib/types/assessment";
import type {
- CompletionParams,
- ConfigBlob,
+ AssessmentConfigBlob,
+ AssessmentInputSchemaColumn,
+ AssessmentParams,
+ AssessmentPreFilters,
ConfigCreate,
ConfigListResponse,
ConfigPublic,
@@ -27,6 +36,7 @@ import type {
ConfigVersionPublic,
ConfigVersionResponse,
ConfigWithVersionResponse,
+ ProviderType,
SavedConfig,
} from "@/app/lib/types/configs";
@@ -61,8 +71,10 @@ export function buildDefaultParams(
);
}
-export function buildInitialAssessmentConfigDraft(): ConfigBlob {
- return JSON.parse(JSON.stringify(ASSESSMENT_DEFAULT_CONFIG)) as ConfigBlob;
+export function buildInitialAssessmentConfigDraft(): AssessmentConfigBlob {
+ return JSON.parse(
+ JSON.stringify(ASSESSMENT_DEFAULT_CONFIG),
+ ) as AssessmentConfigBlob;
}
export function buildInitialAssessmentVersionState(): VersionListState {
@@ -97,33 +109,254 @@ function buildPageResult(
};
}
-function normalizeConfigBlobForApi(configBlob: ConfigBlob): ConfigBlob {
- const nextParams: Partial = {};
- Object.entries(configBlob.completion.params).forEach(([key, value]) => {
+const PRESERVED_ASSESSMENT_PARAM_KEYS = new Set([
+ "model",
+ "instructions",
+ "input_schema",
+ "json_output_schema",
+]);
+
+// Assessment config_blob provider must be a backend TextProvider
+// (openai | google | anthropic). The model catalog uses "google-aistudio" for
+// Gemini (the credential provider), so narrow it to "google" at the API boundary.
+function toTextProvider(provider: ProviderType): ProviderType {
+ return (provider === "google-aistudio" ? "google" : provider) as ProviderType;
+}
+
+function normalizeConfigBlobForApi(
+ configBlob: AssessmentConfigBlob,
+): AssessmentConfigBlob {
+ const src = configBlob.assessment.params;
+ const nextParams: AssessmentParams = {
+ model: src.model,
+ instructions: src.instructions,
+ input_schema: src.input_schema,
+ };
+ if (src.json_output_schema != null) {
+ nextParams.json_output_schema = src.json_output_schema;
+ }
+ Object.entries(src).forEach(([key, value]) => {
+ if (PRESERVED_ASSESSMENT_PARAM_KEYS.has(key)) return;
if (value !== undefined && value !== "") {
nextParams[key] = value;
}
});
- return {
- completion: {
- provider: configBlob.completion.provider,
+
+ const normalized: AssessmentConfigBlob = {
+ assessment: {
+ provider: toTextProvider(configBlob.assessment.provider),
type: "text",
- params: nextParams as CompletionParams,
+ params: nextParams,
},
};
+ if (
+ configBlob.pre_filters &&
+ Object.keys(configBlob.pre_filters).length > 0
+ ) {
+ const preFilters = { ...configBlob.pre_filters };
+ if (preFilters.topic_relevance) {
+ preFilters.topic_relevance = {
+ ...preFilters.topic_relevance,
+ provider: toTextProvider(preFilters.topic_relevance.provider),
+ };
+ }
+ if (preFilters.duplicate_detection) {
+ preFilters.duplicate_detection = {
+ ...preFilters.duplicate_detection,
+ provider: toTextProvider(preFilters.duplicate_detection.provider),
+ };
+ }
+ normalized.pre_filters = preFilters;
+ }
+ return normalized;
+}
+
+export function buildAssessmentInputSchema(
+ columnMapping: ColumnMapping,
+): Record {
+ const strictColumns = new Set(columnMapping.strictColumns ?? []);
+ const schema: Record = {};
+ for (const column of columnMapping.textColumns) {
+ schema[column] = { type: "text", strict: strictColumns.has(column) };
+ }
+ for (const attachment of columnMapping.attachments) {
+ const type = attachment.type === "pdf" ? "pdf" : "image";
+ schema[attachment.column] = {
+ type,
+ format: "url",
+ strict: strictColumns.has(attachment.column),
+ };
+ }
+ return schema;
+}
+
+function buildPreFilters(
+ prefilterConfig: PrefilterConfig | null,
+ provider: ProviderType,
+ model: string,
+): AssessmentPreFilters | undefined {
+ if (!prefilterConfig) return undefined;
+ const preFilters: AssessmentPreFilters = {};
+ if (prefilterConfig.topic_relevance?.prompt?.trim()) {
+ preFilters.topic_relevance = {
+ provider,
+ params: {
+ model,
+ instructions: prefilterConfig.topic_relevance.prompt.trim(),
+ },
+ stop_on_fail: true,
+ };
+ }
+ if (prefilterConfig.duplicate_detection) {
+ preFilters.duplicate_detection = {
+ provider,
+ params: {
+ model,
+ instructions: "Flag rows that duplicate an earlier row.",
+ },
+ stop_on_fail: false,
+ };
+ }
+ return Object.keys(preFilters).length > 0 ? preFilters : undefined;
+}
+
+export function buildAssessmentConfigBlob(params: {
+ draft: AssessmentConfigBlob;
+ systemInstruction: string;
+ outputSchema: SchemaProperty[];
+ columnMapping: ColumnMapping;
+ prefilterConfig: PrefilterConfig | null;
+}): AssessmentConfigBlob {
+ const { draft, systemInstruction, outputSchema, columnMapping } = params;
+ const provider = draft.assessment.provider;
+ const model = String(draft.assessment.params.model || "");
+ const jsonOutputSchema = schemaToJsonSchema(outputSchema);
+
+ const nextParams: AssessmentParams = {
+ ...draft.assessment.params,
+ model,
+ instructions: systemInstruction.trim(),
+ input_schema: buildAssessmentInputSchema(columnMapping),
+ };
+ if (jsonOutputSchema) {
+ nextParams.json_output_schema = jsonOutputSchema;
+ } else {
+ delete nextParams.json_output_schema;
+ }
+
+ const blob: AssessmentConfigBlob = {
+ assessment: { provider, type: "text", params: nextParams },
+ };
+ const preFilters = buildPreFilters(params.prefilterConfig, provider, model);
+ if (preFilters) blob.pre_filters = preFilters;
+ return blob;
+}
+
+export interface AssessmentBuilderState {
+ draft: AssessmentConfigBlob;
+ systemInstruction: string;
+ outputSchema: SchemaProperty[];
+ columnMapping: ColumnMapping;
+ prefilterConfig: PrefilterConfig | null;
+}
+
+// Inverse of buildAssessmentConfigBlob: hydrate the builder from a saved
+// config version's blob so an existing config can be loaded and edited into a
+// new version. Pre-filter column selections are not stored in the blob, so
+// they come back empty (the criteria/prompt is preserved).
+export function assessmentBlobToBuilderState(
+ blob: AssessmentConfigBlob,
+): AssessmentBuilderState {
+ if (!blob?.assessment?.params) {
+ throw new Error(
+ "This configuration is not a valid assessment config (no assessment block).",
+ );
+ }
+ const params = blob.assessment.params as Record;
+ const inputSchema = (params.input_schema ?? {}) as Record<
+ string,
+ AssessmentInputSchemaColumn
+ >;
+
+ const textColumns: string[] = [];
+ const attachments: Attachment[] = [];
+ const strictColumns: string[] = [];
+ for (const [name, column] of Object.entries(inputSchema)) {
+ if (column.type === "text") {
+ textColumns.push(name);
+ } else {
+ attachments.push({
+ column: name,
+ type: column.type,
+ format: column.format ?? "url",
+ });
+ }
+ if (column.strict) strictColumns.push(name);
+ }
+
+ const jsonOutputSchema = params.json_output_schema as
+ | Record
+ | undefined;
+
+ let prefilterConfig: PrefilterConfig | null = null;
+ if (blob.pre_filters) {
+ prefilterConfig = {};
+ const tr = blob.pre_filters.topic_relevance;
+ if (tr) {
+ const trParams = (tr.params ?? {}) as Record;
+ prefilterConfig.topic_relevance = {
+ columns: [],
+ prompt: String(trParams.instructions ?? ""),
+ };
+ }
+ if (blob.pre_filters.duplicate_detection) {
+ prefilterConfig.duplicate_detection = { columns: [] };
+ }
+ }
+
+ return {
+ draft: blob,
+ systemInstruction: String(params.instructions ?? ""),
+ outputSchema: jsonOutputSchema ? fromJsonSchema(jsonOutputSchema) : [],
+ columnMapping: {
+ textColumns,
+ attachments,
+ groundTruthColumns: [],
+ strictColumns,
+ },
+ prefilterConfig,
+ };
+}
+
+export async function fetchAssessmentDatasetRows(
+ datasetId: string,
+ apiKey: string,
+): Promise {
+ // Endpoint returns APIResponse[AssessmentDatasetRows] — unwrap the envelope
+ // so callers get { headers, rows, total_rows } directly.
+ const res = await apiFetch<{
+ success: boolean;
+ data: AssessmentDatasetRows | null;
+ error?: string;
+ }>(`/api/assessment/datasets/${datasetId}/rows`, apiKey);
+ if (!res.success || !res.data) {
+ throw new Error(res.error || "Failed to fetch dataset rows");
+ }
+ return res.data;
}
export async function fetchConfigPage(params: {
apiKey: string;
skip?: number;
limit?: number;
+ tag?: string;
}): Promise> {
const skip = params.skip ?? 0;
const limit = params.limit ?? DEFAULT_PAGE_LIMIT;
const query = new URLSearchParams({
skip: String(skip),
limit: String(limit),
- tag: ASSESSMENT_TAG,
+ tag: params.tag ?? ASSESSMENT_TAG,
});
const data = await apiFetch(
`/api/configs?${query.toString()}`,
@@ -138,14 +371,14 @@ export async function fetchConfigPage(params: {
export async function fetchConfigVersionsPage(
apiKey: string,
configId: string,
- params: { skip?: number; limit?: number },
+ params: { skip?: number; limit?: number; tag?: string },
): Promise> {
const skip = params.skip ?? 0;
const limit = params.limit ?? DEFAULT_PAGE_LIMIT;
const query = new URLSearchParams({
skip: String(skip),
limit: String(limit),
- tag: ASSESSMENT_TAG,
+ tag: params.tag ?? ASSESSMENT_TAG,
});
const data = await apiFetch(
`/api/configs/${configId}/versions?${query.toString()}`,
@@ -161,8 +394,9 @@ export async function fetchConfigVersionDetail(
apiKey: string,
configId: string,
versionNumber: number,
+ tag: string = ASSESSMENT_TAG,
): Promise {
- const query = new URLSearchParams({ tag: ASSESSMENT_TAG });
+ const query = new URLSearchParams({ tag });
const data = await apiFetch(
`/api/configs/${configId}/versions/${versionNumber}?${query.toString()}`,
apiKey,
@@ -183,13 +417,15 @@ export async function fetchConfigSelection(
config.id,
versionNumber,
);
- const completion = version.config_blob.completion;
+ const blob = version.config_blob as unknown as AssessmentConfigBlob;
+ const assessment = blob.assessment;
return {
config_id: config.id,
config_version: version.version,
name: config.name,
- provider: completion.provider,
- model: String(completion.params.model || ""),
+ provider: assessment?.provider ?? "",
+ model: String(assessment?.params?.model || ""),
+ input_schema: assessment?.params?.input_schema ?? {},
};
}
@@ -197,23 +433,19 @@ export async function saveAssessmentConfig(params: {
apiKey: string;
configName: string;
commitMessage: string;
- configBlob: ConfigBlob;
+ configBlob: AssessmentConfigBlob;
existingConfig: { id: string; name: string } | null;
}): Promise {
const { apiKey, existingConfig } = params;
- if (!apiKey) {
- throw new Error("No API key selected. Please choose one in the Keystore.");
- }
-
const trimmedName = params.configName.trim();
if (!trimmedName) {
throw new Error("Configuration name is required");
}
const normalizedBlob = normalizeConfigBlobForApi(params.configBlob);
- const provider = normalizedBlob.completion.provider;
- const model = String(normalizedBlob.completion.params.model || "");
+ const provider = normalizedBlob.assessment.provider;
+ const model = String(normalizedBlob.assessment.params.model || "");
if (existingConfig) {
const versionCreate: ConfigVersionCreate = {