diff --git a/backend/app/api/docs/evaluation/list_evaluations.md b/backend/app/api/docs/evaluation/list_evaluations.md index 24ab51623..662b2cee4 100644 --- a/backend/app/api/docs/evaluation/list_evaluations.md +++ b/backend/app/api/docs/evaluation/list_evaluations.md @@ -1,3 +1,5 @@ List all evaluation runs for the current organization and project. Returns a paginated list of evaluation runs ordered by most recent first. Each run includes metadata (ID, name, dataset info, timestamps), configuration details, batch job ID, status tracking (pending/running/completed/failed), progress metrics (total/completed items), and results when available. + +Optional query param `dataset_id` restricts the list to runs of a single evaluation dataset. Omit it to return runs across all datasets in the project. diff --git a/backend/app/api/routes/evaluations/evaluation.py b/backend/app/api/routes/evaluations/evaluation.py index ff3ca7f3b..b6560eae2 100644 --- a/backend/app/api/routes/evaluations/evaluation.py +++ b/backend/app/api/routes/evaluations/evaluation.py @@ -1,6 +1,7 @@ """Evaluation run API routes.""" import logging +from typing import Annotated from uuid import UUID from asgi_correlation_id import correlation_id @@ -114,14 +115,11 @@ def list_evaluation_runs( auth_context: AuthContextDep, limit: int = 50, offset: int = 0, + dataset_id: Annotated[ + int | None, Query(description="Filter runs by evaluation dataset ID") + ] = None, ) -> APIResponse[list[EvaluationRunPublic]]: """List evaluation runs.""" - logger.info( - f"[list_evaluation_runs] Listing evaluation runs | " - f"org_id={auth_context.organization_.id} | " - f"project_id={auth_context.project_.id} | limit={limit} | offset={offset}" - ) - return APIResponse.success_response( data=list_evaluation_runs_crud( session=session, @@ -129,6 +127,7 @@ def list_evaluation_runs( project_id=auth_context.project_.id, limit=limit, offset=offset, + dataset_id=dataset_id, ) ) diff --git a/backend/app/crud/evaluations/core.py b/backend/app/crud/evaluations/core.py index 5300fa090..5fa886dbd 100644 --- a/backend/app/crud/evaluations/core.py +++ b/backend/app/crud/evaluations/core.py @@ -119,6 +119,7 @@ def list_evaluation_runs( project_id: int, limit: int = 50, offset: int = 0, + dataset_id: int | None = None, ) -> list[EvaluationRun]: """ List all evaluation runs for an organization and project. @@ -129,6 +130,7 @@ def list_evaluation_runs( project_id: Project ID to filter by limit: Maximum number of runs to return (default 50) offset: Number of runs to skip (for pagination) + dataset_id: Optional evaluation dataset ID to filter by Returns: List of EvaluationRun objects, ordered by most recent first @@ -138,18 +140,17 @@ def list_evaluation_runs( .where(EvaluationRun.organization_id == organization_id) .where(EvaluationRun.project_id == project_id) .where(EvaluationRun.type == EvaluationType.TEXT.value) - .order_by(EvaluationRun.inserted_at.desc()) - .limit(limit) - .offset(offset) ) - runs = session.exec(statement).all() + if dataset_id is not None: + statement = statement.where(EvaluationRun.dataset_id == dataset_id) - logger.info( - f"Found {len(runs)} evaluation runs for org_id={organization_id}, " - f"project_id={project_id}" + statement = ( + statement.order_by(EvaluationRun.inserted_at.desc()).limit(limit).offset(offset) ) + runs = session.exec(statement).all() + return runs diff --git a/backend/app/tests/api/routes/test_evaluation.py b/backend/app/tests/api/routes/test_evaluation.py index 18f0dfa72..51e3c4686 100644 --- a/backend/app/tests/api/routes/test_evaluation.py +++ b/backend/app/tests/api/routes/test_evaluation.py @@ -1680,6 +1680,94 @@ def test_list_evaluation_runs_without_authentication( response = client.get("/api/v1/evaluations") assert response.status_code == 401 + def test_list_evaluation_runs_filtered_by_dataset_id( + self, + client: TestClient, + user_api_key_header: dict[str, str], + db: Session, + user_api_key: TestAuthContext, + create_test_dataset: EvaluationDataset, + ) -> None: + other_dataset = create_test_evaluation_dataset( + db=db, + organization_id=user_api_key.organization_id, + project_id=user_api_key.project_id, + name="other_dataset_for_list_runs", + ) + config = create_test_config(db, project_id=user_api_key.project_id) + + run = create_evaluation_run( + session=db, + run_name="run_on_filtered_dataset", + dataset_name=create_test_dataset.name, + dataset_id=create_test_dataset.id, + config_id=config.id, + config_version=1, + organization_id=user_api_key.organization_id, + project_id=user_api_key.project_id, + ) + create_evaluation_run( + session=db, + run_name="run_on_other_dataset", + dataset_name=other_dataset.name, + dataset_id=other_dataset.id, + config_id=config.id, + config_version=1, + organization_id=user_api_key.organization_id, + project_id=user_api_key.project_id, + ) + + response = client.get( + "/api/v1/evaluations", + params={"dataset_id": create_test_dataset.id}, + headers=user_api_key_header, + ) + + assert response.status_code == 200, response.text + data = response.json()["data"] + assert [r["run_name"] for r in data] == [run.run_name] + assert data[0]["dataset_id"] == create_test_dataset.id + + def test_list_evaluation_runs_without_dataset_id_returns_all_datasets( + self, + client: TestClient, + user_api_key_header: dict[str, str], + db: Session, + user_api_key: TestAuthContext, + create_test_dataset: EvaluationDataset, + ) -> None: + other_dataset = create_test_evaluation_dataset( + db=db, + organization_id=user_api_key.organization_id, + project_id=user_api_key.project_id, + name="other_dataset_for_unfiltered_list", + ) + config = create_test_config(db, project_id=user_api_key.project_id) + + for run_name, dataset in ( + ("run_unfiltered_a", create_test_dataset), + ("run_unfiltered_b", other_dataset), + ): + create_evaluation_run( + session=db, + run_name=run_name, + dataset_name=dataset.name, + dataset_id=dataset.id, + config_id=config.id, + config_version=1, + organization_id=user_api_key.organization_id, + project_id=user_api_key.project_id, + ) + + response = client.get( + "/api/v1/evaluations", + headers=user_api_key_header, + ) + + assert response.status_code == 200, response.text + data = response.json()["data"] + assert {r["run_name"] for r in data} == {"run_unfiltered_a", "run_unfiltered_b"} + class TestEvaluationRouterOrdering: """Regression tests for router ordering (evaluation vs dataset/stt routes). diff --git a/backend/app/tests/crud/evaluations/test_core.py b/backend/app/tests/crud/evaluations/test_core.py index bcedb8a3b..ead0caaa2 100644 --- a/backend/app/tests/crud/evaluations/test_core.py +++ b/backend/app/tests/crud/evaluations/test_core.py @@ -1,4 +1,5 @@ -from uuid import uuid4 +from datetime import datetime, timedelta +from uuid import UUID, uuid4 from sqlmodel import Session, select @@ -10,8 +11,15 @@ list_evaluation_runs, ) from app.crud.evaluations.dataset import create_evaluation_dataset -from app.models import EvaluationRun, Organization, Project +from app.models import EvaluationDataset, EvaluationRun, Organization, Project from app.models.stt_evaluation import EvaluationType +from app.tests.utils.auth import ( + TestAuthContext, + get_superuser_test_auth_context, + get_user_test_auth_context, +) +from app.tests.utils.test_data import create_test_evaluation_dataset +from app.tests.utils.utils import random_lower_string def _create_config(db: Session, project_id: int) -> tuple: @@ -42,6 +50,37 @@ def _create_config(db: Session, project_id: int) -> tuple: return config.id, config_version.version +def _create_run( + db: Session, + auth: TestAuthContext, + dataset: EvaluationDataset, + config: tuple[UUID, int], + run_name: str | None = None, + inserted_at: datetime | None = None, + run_type: str = EvaluationType.TEXT.value, +) -> EvaluationRun: + """Helper to create an evaluation run, optionally overriding type/inserted_at.""" + config_id, config_version = config + run = create_evaluation_run( + session=db, + run_name=run_name or f"run_{random_lower_string()}", + dataset_name=dataset.name, + dataset_id=dataset.id, + config_id=config_id, + config_version=config_version, + organization_id=auth.organization_id, + project_id=auth.project_id, + ) + + run.type = run_type + if inserted_at is not None: + run.inserted_at = inserted_at + db.add(run) + db.commit() + db.refresh(run) + return run + + class TestCreateEvaluationRun: """Test creating evaluation runs.""" @@ -247,3 +286,141 @@ def test_list_evaluation_runs_excludes_non_text_type(self, db: Session) -> None: assert len(runs) == 3 assert all(r.type == EvaluationType.TEXT.value for r in runs) + + def test_list_evaluation_runs_filters_by_dataset_id(self, db: Session) -> None: + auth = get_user_test_auth_context(db) + config = _create_config(db, auth.project_id) + dataset_a = create_test_evaluation_dataset( + db=db, organization_id=auth.organization_id, project_id=auth.project_id + ) + dataset_b = create_test_evaluation_dataset( + db=db, organization_id=auth.organization_id, project_id=auth.project_id + ) + + run_a = _create_run(db, auth, dataset_a, config) + _create_run(db, auth, dataset_b, config) + + runs = list_evaluation_runs( + session=db, + organization_id=auth.organization_id, + project_id=auth.project_id, + dataset_id=dataset_a.id, + ) + + assert [r.run_name for r in runs] == [run_a.run_name] + + def test_list_evaluation_runs_without_dataset_id_returns_all_datasets( + self, db: Session + ) -> None: + auth = get_user_test_auth_context(db) + config = _create_config(db, auth.project_id) + dataset_a = create_test_evaluation_dataset( + db=db, organization_id=auth.organization_id, project_id=auth.project_id + ) + dataset_b = create_test_evaluation_dataset( + db=db, organization_id=auth.organization_id, project_id=auth.project_id + ) + + run_a = _create_run(db, auth, dataset_a, config) + run_b = _create_run(db, auth, dataset_b, config) + + runs = list_evaluation_runs( + session=db, + organization_id=auth.organization_id, + project_id=auth.project_id, + ) + + assert {r.run_name for r in runs} == {run_a.run_name, run_b.run_name} + + def test_list_evaluation_runs_dataset_from_other_project_excluded( + self, db: Session + ) -> None: + user_auth = get_user_test_auth_context(db) + other_auth = get_superuser_test_auth_context(db) + + other_dataset = create_test_evaluation_dataset( + db=db, + organization_id=other_auth.organization_id, + project_id=other_auth.project_id, + ) + other_run = _create_run( + db, other_auth, other_dataset, _create_config(db, other_auth.project_id) + ) + + user_dataset = create_test_evaluation_dataset( + db=db, + organization_id=user_auth.organization_id, + project_id=user_auth.project_id, + ) + _create_run( + db, user_auth, user_dataset, _create_config(db, user_auth.project_id) + ) + + runs = list_evaluation_runs( + session=db, + organization_id=user_auth.organization_id, + project_id=user_auth.project_id, + dataset_id=other_dataset.id, + ) + + assert runs == [] + + # positive control: the same dataset_id resolves in its owning project + owner_runs = list_evaluation_runs( + session=db, + organization_id=other_auth.organization_id, + project_id=other_auth.project_id, + dataset_id=other_dataset.id, + ) + assert [r.run_name for r in owner_runs] == [other_run.run_name] + + def test_list_evaluation_runs_dataset_id_composes_with_type_and_pagination( + self, db: Session + ) -> None: + auth = get_user_test_auth_context(db) + config = _create_config(db, auth.project_id) + dataset_a = create_test_evaluation_dataset( + db=db, organization_id=auth.organization_id, project_id=auth.project_id + ) + dataset_b = create_test_evaluation_dataset( + db=db, organization_id=auth.organization_id, project_id=auth.project_id + ) + + base = now() + oldest = _create_run( + db, auth, dataset_a, config, inserted_at=base - timedelta(minutes=3) + ) + middle = _create_run( + db, auth, dataset_a, config, inserted_at=base - timedelta(minutes=2) + ) + newest = _create_run( + db, auth, dataset_a, config, inserted_at=base - timedelta(minutes=1) + ) + _create_run( + db, + auth, + dataset_a, + config, + inserted_at=base, + run_type=EvaluationType.STT.value, + ) + _create_run(db, auth, dataset_b, config, inserted_at=base) + + first_page = list_evaluation_runs( + session=db, + organization_id=auth.organization_id, + project_id=auth.project_id, + dataset_id=dataset_a.id, + limit=2, + ) + second_page = list_evaluation_runs( + session=db, + organization_id=auth.organization_id, + project_id=auth.project_id, + dataset_id=dataset_a.id, + limit=2, + offset=2, + ) + + assert [r.run_name for r in first_page] == [newest.run_name, middle.run_name] + assert [r.run_name for r in second_page] == [oldest.run_name]