From e222617f98b177081ae1cd0aea8932d6d99164af Mon Sep 17 00:00:00 2001 From: Prajna1999 Date: Tue, 7 Jul 2026 11:16:41 +0530 Subject: [PATCH 01/16] feat: basic stats queries --- backend/app/api/routes/cron.py | 38 ++++++++++++ backend/app/crud/stats.py | 105 +++++++++++++++++++++++++++++++++ backend/app/services/stats.py | 33 +++++++++++ 3 files changed, 176 insertions(+) create mode 100644 backend/app/crud/stats.py create mode 100644 backend/app/services/stats.py diff --git a/backend/app/api/routes/cron.py b/backend/app/api/routes/cron.py index c31288ce8..305b9a738 100644 --- a/backend/app/api/routes/cron.py +++ b/backend/app/api/routes/cron.py @@ -9,6 +9,7 @@ from app.core.config import settings from app.crud.evaluations import process_all_pending_evaluations from app.services.job_monitoring import monitor_pending_jobs +from app.services.stats import collect_daily_stats logger = logging.getLogger(__name__) @@ -33,6 +34,16 @@ "recovery_threshold": 1, } +DAILY_STATS_CRON_MONITOR_CONFIG: MonitorConfig = { + "schedule": {"type": "crontab", "value": "0 0 * * *"}, + "timezone": "UTC", + "checkin_margin": 5, + "max_runtime": 10, + "failure_issue_threshold": 1, + "recovery_threshold": 1, +} + + PENDING_JOBS_CRON_MONITOR_CONFIG: MonitorConfig = { "schedule": { "type": "interval", @@ -123,6 +134,33 @@ async def evaluation_cron_job( raise +@router.get( + "/cron/daily-stats", + include_in_schema=False, + dependencies=[Depends(require_permission(Permission.SUPERUSER))], +) +@sentry_sdk.monitor( + monitor_slug="daily-stats-cron-job", + monitor_config=DAILY_STATS_CRON_MONITOR_CONFIG, +) +def daily_stats_cron_job(session: SessionDep) -> dict: + logger.info("[daily_stats_cron_job] Cron job invoked") + try: + result = collect_daily_stats(session=session) + logger.info( + "[daily_stats_cron_job] Completed | window: %s", + result["window"], + ) + return result + except Exception as e: + logger.error( + f"[daily_stats_cron_job] Error executing cron job: {e}", + exc_info=True, + ) + sentry_sdk.capture_exception(e) + raise + + @router.get( "/cron/pending-jobs", include_in_schema=False, diff --git a/backend/app/crud/stats.py b/backend/app/crud/stats.py new file mode 100644 index 000000000..b54b2f95a --- /dev/null +++ b/backend/app/crud/stats.py @@ -0,0 +1,105 @@ +import logging +from datetime import datetime +from typing import Any + +from sqlalchemy import text +from sqlmodel import Session + +logger = logging.getLogger(__name__) + + +_LLM_TOKEN_SUMMARY_SQL = text( + """ + SELECT + o.name AS organization_name, + l.model AS model, + COALESCE(SUM((l.usage->>'total_tokens')::INTEGER), 0) AS sum_total_tokens, + COALESCE(SUM((l.usage->>'input_tokens')::INTEGER), 0) AS sum_input_tokens, + COALESCE(SUM((l.usage->>'output_tokens')::INTEGER), 0) AS sum_output_tokens + FROM llm_call l + INNER JOIN organization o ON l.organization_id = o.id + WHERE l.inserted_at BETWEEN :start_at AND :end_at + AND l.deleted_at IS NULL + GROUP BY o.name, l.model + ORDER BY sum_total_tokens DESC + """ +) + +_LLM_MODALITY_COUNTS_SQL = text( + """ + SELECT + o.name AS organization_name, + CASE + WHEN l.input_type = 'text' AND l.output_type = 'text' THEN 'TEXT' + WHEN l.input_type = 'audio' AND l.output_type = 'text' THEN 'STT' + WHEN l.input_type = 'text' AND l.output_type = 'audio' THEN 'TTS' + ELSE 'OTHER' + END AS modality, + COUNT(*) AS call_count + FROM llm_call l + INNER JOIN organization o ON l.organization_id = o.id + WHERE l.inserted_at BETWEEN :start_at AND :end_at + AND l.deleted_at IS NULL + GROUP BY o.name, modality + ORDER BY o.name, modality + """ +) + +_JOB_COUNTS_SQL = text( + """ + SELECT + o.name AS organization_name, + j.job_type AS job_type, + COUNT(*) AS job_count + FROM job j + INNER JOIN project p ON j.project_id = p.id + INNER JOIN organization o ON p.organization_id = o.id + WHERE j.inserted_at BETWEEN :start_at AND :end_at + GROUP BY o.name, j.job_type + ORDER BY o.name, j.job_type + """ +) + + +def _org_count_sql(table: str) -> Any: + return text( + f""" + SELECT + o.name AS organization_name, + COUNT(*) AS row_count + FROM {table} t + INNER JOIN organization o ON t.organization_id = o.id + WHERE t.inserted_at BETWEEN :start_at AND :end_at + GROUP BY o.name + ORDER BY row_count DESC + """ + ) + + +_EVAL_RUN_COUNTS_SQL = _org_count_sql("evaluation_run") +_STT_RESULT_COUNTS_SQL = _org_count_sql("stt_result") +_TTS_RESULT_COUNTS_SQL = _org_count_sql("tts_result") +_ASSESSMENT_COUNTS_SQL = _org_count_sql("assessment") + + +def _rows(session: Session, stmt: Any, params: dict[str, Any]) -> list[dict[str, Any]]: + return [dict(row) for row in session.execute(stmt, params).mappings().all()] + + +def get_daily_stats( + *, session: Session, start_at: datetime, end_at: datetime +) -> dict[str, list[dict[str, Any]]]: + params = {"start_at": start_at, "end_at": end_at} + logger.info( + f"[get_daily_stats] Collecting stats | start_at: {start_at.isoformat()}, " + f"end_at: {end_at.isoformat()}" + ) + return { + "llm_call_token_summary": _rows(session, _LLM_TOKEN_SUMMARY_SQL, params), + "llm_call_modality_counts": _rows(session, _LLM_MODALITY_COUNTS_SQL, params), + "job_type_counts": _rows(session, _JOB_COUNTS_SQL, params), + "evaluation_run_counts": _rows(session, _EVAL_RUN_COUNTS_SQL, params), + "stt_result_counts": _rows(session, _STT_RESULT_COUNTS_SQL, params), + "tts_result_counts": _rows(session, _TTS_RESULT_COUNTS_SQL, params), + "assessment_counts": _rows(session, _ASSESSMENT_COUNTS_SQL, params), + } diff --git a/backend/app/services/stats.py b/backend/app/services/stats.py new file mode 100644 index 000000000..2f0e65355 --- /dev/null +++ b/backend/app/services/stats.py @@ -0,0 +1,33 @@ +import logging +from datetime import timedelta +from typing import Any + +from sqlmodel import Session + +from app.core.util import now +from app.crud.stats import get_daily_stats + +logger = logging.getLogger(__name__) + +DAILY_WINDOW = timedelta(hours=24) + + +def collect_daily_stats(*, session: Session) -> dict[str, Any]: + end_at = now() + start_at = end_at - DAILY_WINDOW + logger.info( + f"[collect_daily_stats] Starting | start_at: {start_at.isoformat()}, " + f"end_at: {end_at.isoformat()}" + ) + stats = get_daily_stats(session=session, start_at=start_at, end_at=end_at) + logger.info( + f"[collect_daily_stats] Completed | sections: {len(stats)}, " + f"llm_calls: {len(stats['llm_call_token_summary'])}" + ) + return { + "window": { + "start_at": start_at.isoformat(), + "end_at": end_at.isoformat(), + }, + "stats": stats, + } From db92ae610b6cde9d84fb3d282436351ef4e1b00e Mon Sep 17 00:00:00 2001 From: Prajna1999 Date: Tue, 7 Jul 2026 12:05:21 +0530 Subject: [PATCH 02/16] feat: routes for health_probe --- backend/app/api/routes/cron.py | 43 +++++ backend/app/core/config.py | 4 + backend/app/services/health_probes.py | 219 ++++++++++++++++++++++++++ 3 files changed, 266 insertions(+) create mode 100644 backend/app/services/health_probes.py diff --git a/backend/app/api/routes/cron.py b/backend/app/api/routes/cron.py index 305b9a738..f6f68937e 100644 --- a/backend/app/api/routes/cron.py +++ b/backend/app/api/routes/cron.py @@ -9,6 +9,7 @@ from app.core.config import settings from app.crud.evaluations import process_all_pending_evaluations from app.services.job_monitoring import monitor_pending_jobs +from app.services.health_probes import run_probes from app.services.stats import collect_daily_stats logger = logging.getLogger(__name__) @@ -34,6 +35,20 @@ "recovery_threshold": 1, } +HEALTH_PROBES_CRON_MONITOR_CONFIG: MonitorConfig = { + "schedule": { + "type": "interval", + "value": settings.HEALTH_PROBE_INTERVAL_MINUTES, + "unit": "minute", + }, + "timezone": "UTC", + "checkin_margin": 2, + "max_runtime": 2 * settings.HEALTH_PROBE_INTERVAL_MINUTES, + "failure_issue_threshold": 1, + "recovery_threshold": 1, +} + + DAILY_STATS_CRON_MONITOR_CONFIG: MonitorConfig = { "schedule": {"type": "crontab", "value": "0 0 * * *"}, "timezone": "UTC", @@ -161,6 +176,34 @@ def daily_stats_cron_job(session: SessionDep) -> dict: raise +@router.get( + "/cron/health-probes", + include_in_schema=False, + dependencies=[Depends(require_permission(Permission.SUPERUSER))], +) +@sentry_sdk.monitor( + monitor_slug="health-probes-cron-job", + monitor_config=HEALTH_PROBES_CRON_MONITOR_CONFIG, +) +def health_probes_cron_job(session: SessionDep) -> dict: + logger.info("[health_probes_cron_job] Cron job invoked") + try: + result = run_probes(session=session) + logger.info( + "[health_probes_cron_job] Completed | ok: %s, failed: %s", + result.get("ok"), + result.get("failed"), + ) + return result + except Exception as e: + logger.error( + f"[health_probes_cron_job] Error executing cron job: {e}", + exc_info=True, + ) + sentry_sdk.capture_exception(e) + raise + + @router.get( "/cron/pending-jobs", include_in_schema=False, diff --git a/backend/app/core/config.py b/backend/app/core/config.py index 8d938d4e3..483088ae8 100644 --- a/backend/app/core/config.py +++ b/backend/app/core/config.py @@ -187,6 +187,10 @@ def AWS_S3_BUCKET(self) -> str: DOC_TRANSFORMATION_PENDING_THRESHOLD_MINUTES: int = 30 PENDING_JOB_QUERY_TIMEOUT_MS: int = 1000 + HEALTH_PROBE_ORG_ID: int | None = None + HEALTH_PROBE_PROJECT_ID: int | None = None + HEALTH_PROBE_INTERVAL_MINUTES: int = 5 + # AI-assisted prompt improvement settings. # See docs/srd-ai-prompt-improvement.md for the full design rationale. # Platform-owned Anthropic key shared by every org/project for this feature, diff --git a/backend/app/services/health_probes.py b/backend/app/services/health_probes.py new file mode 100644 index 000000000..d5f3ec70f --- /dev/null +++ b/backend/app/services/health_probes.py @@ -0,0 +1,219 @@ +import base64 +import logging +import time +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass +from typing import Any, Literal + +from sqlmodel import Session + +from app.core.audio_utils import AudioRef +from app.core.config import settings +from app.models.llm import KaapiCompletionConfig, QueryParams +from app.services.llm.providers.base import BaseProvider +from app.services.llm.providers.registry import get_llm_provider + +logger = logging.getLogger(__name__) + +_PROBE_INPUT = "ping" +_PROBE_MAX_TOKENS = 1 +_PROBE_WORKERS = 4 + +# "Hello" ~1sec +_STT_AUDIO_B64: str = "T2dnUwACAAAAAAAAAACWAyvBAAAAAK8Ly3sBE09wdXNIZWFkAQI4AYC7AAAAAABPZ2dTAAAAAAAAAAAAAJYDK8EBAAAADsN7gwE2T3B1c1RhZ3MNAAAATGF2ZjYwLjE2LjEwMQEAAAAVAAAAZW5jb2Rlcj1MYXZmNjAuMTYuMTAxT2dnUwAAgLsAAAAAAACWAyvBAgAAAKnTMEhb//9A/wX/dv8o/v8F/wP/G/8e+/8S/zz/LP8J/wf3+/Hu/wT/Gv8l/w//DP8F7/8E/wr/Of9L/1L/QP8+/w7/SP8x/1n/IP8A9/8Y9v8C/wb/Kf8L+v8p/yD/4/x/cqF3yWnKJA+FRE/nSs5qstg30xlSAKfIW9e0H4hx095LUdI3HKNmbgjnKiDv3IlFUdpy7t2wPk1WZuuVX0o3MGAITvlC8ZVdG7R890ps2WXgJatlobIk8kSFl57Ispzj2WlqDg3BIwKyuQDIXFY49jbiiaJJ/2ln3N9+uTUe6pEr4GgTJeQOP5PM3wW8HJ8+2Fcccx6J7nQ9Fwwe+ye8YBWsHE6KHkKslGKqDuslGnQWiK9TdNnQuWro8YSrvnQ6K7VDSp/6fSj89BzyuLJ+VuyhAA5Sp3dYtytIBnR8OZKDR9rHG9Xk1gdZc6FX8p7Iwx23M3Kgg+dALdI1SVhHzoU1yNhlhz8wjqWiTUDwAAAAAAAB4tu7QcxPl1EsJBQi/OEnNcntPEWmTJUMnz1p+YbN2pKP7QughtAJibW3IjF8xMiQo+aDSo3/60XaFjProF4w44EheWxDL2v4SxPVmU0h/qBG4syaVwDXrUO0pbvw+lAVrp34fSoN0T01XerHqRRpwmlbbitWcauaQFtvYVzqUL0xkqQia7v/HGLA+5dmvAroSk4hWT8QOqq66DtdTyAJiWP2basqDIUwxBVVOnkMbHcrFScvAR1A9zayzA66G73Cyrbd2I/LL+BlRjuJ2pi8c9uBosB8HBytcvJ7DlSChPxIR7Iqtiq5uN5cKakTc03BNHlEOaeDm1EfkOfOkRmn3r28ggDJl0otTIWVz7wyznTfmvH6HTiyWe4AKlIJ3TNH8gdEJEvmQOv8SzXtqGlUeA77FwnWhtCsi1FMLlJNxA0tKIjyXNXwtJLvNd2sTfu0Qz1EnyjMOd16LqbYdHcvuLxLiTurviijKYa37+KQlpa0JyJClur5EfVHsFWpQC/VqGxFY0YNXVpCL8KV2slPthFj/5NwzOEaTPZPGTnJXe7fO6PtDPsUY/iz4q0gM798WKrPR803lqxcCgqZ+mCjGsbF+DCCYj3fwISbj6d/rbE5b0lWVzDNM/HkTCS7+exWY4iEdkh6gMRF+u+5rMBZ8BDBafDNPwHlEnllSdJKkRUSoXSvT3SZ6FqQkcq7cJT9afqEaHECNvb2DL8pg4xR0LQEruNgCaWg8LWvXPx8r9vzrAjfHOuuxMXH4DOEcCNj+Cciy/FIrFatJiyS21xuavTiZnH1SvavxOgolhAqKBDX6pnaaoAGkbeMi+kxGO7xS6zjG9vNzN8cI5qHOfr2LNGcvAMkvtC5qt6oyvW/OmUFyquyDN5Zb+m6CHGvnbeuvS2O72KxPhlWSQTYhp61XQljeUhhYKcMN2zOXwXY4QYWskt5YXgeexpM/NxMwzNSgawoojMtwrDi91z6VcuJVmIcCg0Z75ls0gQ+CfP+jbkTJWxiHKfs8dX2FHMyw58uPyu6jAG3tlNBeuWkh1/JfyAT5oZ84uXUA02zLDJTyUpHPhjEVBKlF2CFxiJEXVOlclAbL7idHaQkqTy6RlkwnjrGzygajqZ3nkftGHp0sapHwEDBZqURvTZV51V4kIycKmj5AtUBQESyCUJj5w06zFqxius8flGa3bIkFjHXitdl0ttsIrs0ww/FMZ+0HzRVzrlIHZuS1qwErW14/iJFJUv8wpnbXGNI07NclvoqRa64UVzpcVxEvvzaKdxRUbBTj2nrkRMM2mIrY5vXHTr6fRkF7M+Re8SvwmiFY/s6X6JaR6LqmFheGJkBwpbz8SpC/9d2BRl3VtC/GfWMnnZ8pvzSSU1swebkQb6tMY0wgzxeRn7gGxuCJcUoe8C75ThqIlYUvCbvoRKW13owiQlIcuTQOgk56VeqrHq04O5kyS6FGjenHbM6gKNq0iLzb5kD5r3WRNyZ5FTAy9VWVGzuezfHc2SnHPJWj+KaxNd8hWFdYoUclS3G7fjNqggRv8wQ5kQy/do8Vcl4Gwtq4A7XtitY1ZKYVv0iOyUtq00b1XLTbzA+ZZNLsplGSqI4DSTbzPOpeScWilFgNoUe0DUC0KD5pQqh0HGO/MICFpXbOtSv8pvNyQqsnY+gTCTRv77BseSsQbyT5AescVq+xBtVhs8t/JxmwpZ+JLNptgYhPGNHooiuAtgvESv5wkCqP7SjzpoMF9NblUFYkkCY9dG0qyJfhgW6o98eSFgyUqL+3YusDQFxogKz+GkSNXVxzHRuBT6QPShnRkPgDiBkL3usUezDEAEIPw2Wa3bBZRqryM8BCFKU9jRfHhGPLA3DU8YIwx4rdJcNiSil3LCqcmmY8QGrog4gBV+cVLoMqjevksEQNqYadQniIssNGkV97wW+RQrgdNTCyOdGx25b8O+FQsB42Ha7tmEQSFxyttlqgAoKnK97YQ78wClklb3lYxavDs1jF8lIB0w8lrV1i9MRvLEEJKpMdAhkB8ZSJ5ZlMqhuewjMxlrutNCMREjXLQAmQn5bLxFchjbNhphMBPXpfZUc+j83KPEr3m1IMNV5wgi2AFlW0uQ0hGQt7PJnvvbrVQKVi9xMQH1YkccfS/qVoZaGefvr9Ss+4HLGavpheS/X0QKavu8+g9eWkL5JoY75Der87JCguq7rnyBYBCd7Mpi7mRHr8ibi0FdXyzUAWc/pB2LVuI84GZ6oqfQUHmYCzMmMhHTrM9BY8BY8BIi7G/YhUDrQ9fYp0M7Ejpv5b21llJ2dN59dwGUQRv6S22xJJVQkIrM2/7y2EPzAK8XkliTK7UZxVSrrMCmnCijk90WYa+YUFoHTi7It5PNQppOCWDURdDbQIxs62BxmEBBhAUhvX6QrYJmunX1nMAHpFLK4K51HpqjFBLZVRhX1kUUnwEmJxp4v7mHM7n9rAVaJTcVZVZhKK3qfrJ5FtE5f/zFQdzNntux3vBzgpVdLkXgaEqvgc2jud1oWTH0NA6CBFPW2JUanJe9oKHTibX9mrVEBxYoPUHsUOh/tusIZo88NF5kHbJqIFvjmecSDi6XPixay+ZuWcrVtuHkxRjCF1mY4Az4Rc9aYrHOC/gGyxY0s4NgqKAtJpkKTHmOFcACNu2RsBQDYCVUxrrVOk/zANIxJy0Jai68H2KXedNd8eE8OKtFS8v5PnuknbCx+AMKmuU/9IHH0eoV06ka4c1orihLo1y/Vu+EOWGvkkbrAVlxODNDEvNdXkG+3Y+y4Y8o+aLhBXNcqiWbzDUtuXRsjIn2UtLofb6MaqMAKCkmfhpQ3ipCd5gXjsVcj65TE/kDrwCmvmLRayjgazg6IpSKnd0PUtnUjedc2bG5qMsLQqCPElJV9LMiQ1/2GihTjlS1U36FWIDgLiMuITOHccjLhF/phI5gU2K1zc9oJuSFZSKdOrplmTkK3oqj7ERBa84PUZXw5LvQq7SXmZVws4gQvHmq+eoxJBv0OeTATMzNmYadCWs4J65Jy7buSk2tj8P2/3UVElzTFFPzAKq1LPhtqUhArK5hA+Oj+I46RU6aaf3YPIiaFaamq1fKKoBWwMfA8sfKxFCYfEJncvXOIfc/PNUNNtceBtsm1pl2DhgNPQrYSreid/niE/xQwUiUXEfJ3CVR02srh8HaDfSLDmcj2e4bnmKsaG8fFBPIDY2nAt6skIRZacfMvQvK1IRXlRDkAenS3fge3EkIEYQKUdpQEr2oK8Q+F5SvtH5r6Iinesha2Jf9JC6TvpwgcSoHEsWM6LWyJV+NntV2laJVsg2yHAmju290pH3yobZZrkvY/yXoXjidIKcURTgYD/b+G1cJ6P1aTqdKuPn+KpegTgfgfAsaRov4eW9HuN7m6moq+QTx3WS48b3//Jb2StWtSc/AMva22mPzCAhSn87TC7XMR1HfFO5RJVCsXj4Vcw9ceoegZNWhn2sh4HXZCA61AKyUJRBq5zNOSVCzIYafumbPwf4nieScQ06T34ykr7Ne6qg5AFf57OPN8OMXKwiTX/2zAXQg9VpsaZ8YOk9cz/EDV4RAzNJCRhELI16AGRONtWjyNAoOsdAWbK0P4dxfj+2I38TiKwHSwzUbwozOijf3iLEBSPRuSZ5dZP8jz+HluMQZRJNtKB+yiO2o2LJLuy7yHA6K10Es/KJIP1czSBudKm+QEa9tNOk+jWzXxLY/AQWVGAcRIQ2PcVtRs7Hw9CZCEwBEDX+OOM/CvoLkVarKb/MApcmlzlkoe7rLEh8udVvRFkJnolTjpKqYGWiTo4hDCxqFL56Zw/tETKmDTunDK8uc2v1tqMpCgogGg4BGvZYHYaT1+wgzi9MsnyldgpZxWIVq7DxrM4LmqNdLaYfSkiDAAgmru7Zzve0FSlTlhcAcLa/dYUzMBzXbSYpsPobyj9078iSqE5UEJZaBv0QuZKpHVYEicsgK5CK63EBIKVO4TyLfyT0oEPFUc5W9YeZkiL5inPvhQ9URUM0SWRL3YO5KlrgRRPt3oJzOyxjNvObXm7qt4ALe3BWeZGl6qFliKDuHRdG/aS6S7Dbv2MLXMpLPQ+0+fqCXHbEQPfoLNnzyMud79TjVJr/BZ2k4xoBqh/MICUyL40CwGH1oVRRcCG0NrZn7vcx9GDFAgBc0YVW3ZyODEwmawCzkzfPJmmlwG7pU9+I7uu7RWb+jT6GF6Hvfo/WXY4/XzpGc2/lE+ACkSquJ/fIvBFu1pRUwj4RPPCrqPIvlhRhuxPaaml/IfJbTMsDbUMPDrWtYPO+RGmjake3vi9ilM3O4cVD24KbDntdTa6Y/wS0R8+Qng5543xadRteBBy4i0YHuOHQQbed0vhLcqZR2h0FGjK70R6yGT0THsDQqIVpioUBrBGF09XcO2IdjhnpTcqZRcdk/5/9inwem8+SD5bpd4NtrTf1HZoY6IoyIZS9WOAG5sFCTbcja889p6krbof6V0Tzi+jGGXsQs14wMCfEEdYPz/6914e3Y7F/PjCGXraQkAJA1aAB2CQkiT1kbRM26i/MGdRjc9j/u50ZEirMdylXWu7KohwjfNk9ZlKfTr93IvU4XE5Hli5m+ukf+a8+LGs3jKCSqxglwymSRMoebFx7rixa7HrinW09L7385dSXs2W1U7xuNYa6kdOeGyfpQ6zPGcXrrUyPTlbg30lMxYgbb/0Pq/HEWVNs1x60hqeF5BFVPkMGktxfIhUpiAoHPVjh6MTmToRK/Y//fa6TXBgKaKc6qjT4A1QgQ+/cRpMKpybRsSt+77ZB5Jp7X8IK24uodcbwZLbYBn9UVu7y8iijuQY4lUQCKNA5bDm/VGgycLGMfU5NCagjeX4lw9X+mdSEjXn5bKb4kKPxbttRO0Lxprlz/TPWCXzCHtDfSPNH5bv0DTdx3rEDgs8EkixFJJsnagACWvSd/UjqH8wCtwXqSDfcYfVFx2tJGvdnBccxJF8WUXtV+e9TD0wK6U3+OAwPqOMFCXxbQf/5QH5AmGyp0bN/g5nrFAGfCMQfRhCrpPLQDMRPCtKRk2OzX07bxgcwKI56f4DOG20nhlEkw+273tA3w2Jin16RMDz8l/Omg13C95V+zuSavcElGimo5QYoQejdcGqHUTzVhjTT8OqDh15eJni8fGWUlNJWWBPKK6ZZano2yPBjrG7sYEl6raf+gidczMorJlFFA+AbYEUB7IXjb+SmFiap6jOZrEr3wTYGZMfKqlrGL5ldc46cmirpTfV5MFOkbA7x0YxwKimW4Htqa/AEjj+1JsAFBZoF8Uppz8wClyQsy2b1ZOuztjy0XnGUe+yzTkuLbaP+CRYKouZwlCAsvVyqd5mqLbQCQj5Yv3VyouHC4SbpDHtSfFq+fjRCGfNUrj9NxFrWC5EqF7YaX0JtiwA2yGajQMdPN67hRjhIGAZAhjydmKNotq4lRJJoKMDZfIwcFKo3ez46sgO4tjouCO+KMcR7vA0GXo2kn9Nsr6dTCSXfTRavAiqaGvIsXawo1VKy5uILNNp6UK9AzSH6P0UnRc5t57hw0I8NSO3ckz193P8+ydKShJ4MxQIv9twjm5oFoSKTnxDPhRWKfN3uzkX/QTWK1ZwIzYd2f1K0qjfMsdhDNCLdkrf186Qz8maoIW/MICFpOVfSrKI8sCefkY49jcsDXBreZ3L5i8FL4JlgdmkVJDMxkf8nTycFz/AwzMRa8Smly9LsOD5uWCoTxcs3iG+tfvHe9Ugr5JUzMaDDAVkxM0WM8XCRqbMZk9rmuQh5VXYW9mALTST6SkvYkTl4KqJ61BEwbfSoUzUhJsouSrxAXu7ftBgCRLdzm5L1L75dz8bCP4+odOD8pJPT3yyYgTtqWDq/Oqd9ibq1zk0xkaPWcJGRXfL9LKPcq62OJrWPSRuAuWgSQk2stWRa1pXFM914JsVQ6FYrN/GFrzSHCl5bRXGqEFFiKltsADa0lZ9cqwIq69jfzAKXJa2D4hhpzLI7Wq3xvFGEKmw1+53J3OVuD29BZ7f+8gCDI9tOkFubOJGiyzxvkgi3W1YVWaYM88FLHkZWs9QBbS2qFyMR0lp8pUELAlTNCCFkaPpzT7oZ6Is9qJ5oWqNt9HAOM4sjpyAiLAxI6fTL9K2dc9rFjlfxKC5ghfIMAk3Y2uPesCen4fWkHn8qGHhVYF7ItPK3PLAGRylcBd8PnQaK8JLGwvhIz19y9n0MrpEnq8mdtqIaNhmvLquuKpjHQZM9DFIdDInZSgJ5A1Dt2AQQdT0ClRp4/Sj9V/CpRQg7EfmVIWLPqAG3vTB2a/bUAD4JARWOmH/KxHyp7SG+u2ZNqGwRMWnBwc/4AS+tUG8ouoq5+TQ87jW/jv7RxGea2q2QWvhXGNsACF9lsR8VsM+k7IFrYcm4KdwtXq1njasNGO++3qO93OEuBjMYN1vpIoSJThuxOXVd52i+RhBQ8p5ZcfhzxNhawnZh0o3Tu6dL8eqM8sZyPjGfgF8mSJAG4lIgPwROjh8TVRB4XzMNnmF/caiZRT00rkLa1R7j4zCwu4T5eeFrN7UszqwEY4XCV3sJExOwYhcKY0hJ8lM2vnLa95z6Hfu++PS+hWd/7QPJTyCa4TfvmgNZmuwAHwPDa3tWq1AeRHPvyq1kavbDHxTapblL0HctLTO/lT2IpXo8vPoHJHudE08B3TncZGTd24v5gbBwerFHg7ADfmJ3Tpbk2c2V0+h8cCWw8i5k449OXX7kOa9eVKGuDyxwTRWt08a/hgZa1W8gLK8LakLECZrwVXU/XmWu8Bp4tJT6A2pMpkDVSIzgVg9xfmrr5n+b0hJ9v/LW0bzmzgmwRxdXAA7o61U0enlGiBJq3naL3xtln9wl91uUxVWwMcDBDedUrGLE8XzCHT5ChQX8tLaujIwoUDj8McDh+XmV7ABlIa3itb2xeX3dqm+rj5j2ASXvyKklVQITX8rJ17eTLtKwAhYwjLvAUCqhlRO9X2bebMMfQJAxcINtBewkl095Cz8ysqjy3dYPaHyIBG7Sl0bca5G9UMghtRgNB46SVp3CwKMsIkj+HwGTHA1LpQ5dDTndX+CG6TqmxHzLCcEewPV0FmJLYrATKsngFv69325fdOZAIFgmlZ2Qw7oALWSzifCyDCh7RwdZAmD3ju8tLB10x2vHB4WgZcLfG1jzTVx6IrsPYQQwTpXHzqjWFfJ2CkstjtQcokAkKG/lGhvJm5gvGR1CeuqFZ8k2fKTB2sOeYtBJGB12GjbburEr+ZM75oD6KKyFNsi9GFAkcwOMRflJ/gWdYsrdbqkp0u/Kyc38ehe6mw09CetgdISbO2506aApuD9ZrhSeAFQ2k5lsU51F+n+8vsgh8MEyKeB9TFgO08b+oKhFtaSzo/ruUNcO52WHhnIierL3rgCRz3dx+iAwuYtFyyu/C4FSlfXLV8hBN867N2rE60Yb+4uNm9qYjxLxah3qrXRa9+R3XaVYVf6dqtmCIMhW7YlgVLNDRbyPIKJ0HCRRF8OHy24V703cxi2zZa/nx13644SBZLFGnMpmamU/nQL7KeuYrOS1m9usiIQRoF+xMmyb6avNIJF2XDeFVhPQ3TVi7a2ZDpLngIsHD8Mu32FAd363Bs58IClwg2z0QsmIpfGKLOT9JKhHPmwPxSp5Q1VRMbI7VwcR/QP+Gs3638l1gL2UY1Vd/MYucNoKsj4a8+VnNIZoZ4ay8nfl1Ft4J3B9y0+CArixiUkeuY7kyUn9cFe7LqxKQlNbWk9fDD9S6tjoF/uWXsPMxxAjGUyBczKj5H3p/D+6l7Z1e+VJheS2xjRuOGDHTuYVKPmwA0bplQRXQwEC1xyftAxsmLAG2NUAvU3+pF152MUuZ06mXOb1c+LlB99Fi7AoaPfE5vgokwLJOkMHT9VZbwc4Ox2HtlbFKJr5Tu5SmMmIPtTUCS/vNn/VJuuB8uhZPNZZOYBhPGWQokDS0K46CU1EMSeMtmc1Ka9A7hSMB8wyv6Jp7+aLdsoI0R5t6SLoyT61X2aQDHTIBJWBIYNcfmjl7NYKgsFoCIySJSktRXwX2t2GKpzS83/JVcH0TytNPo1uUDEy7rUgiRFrdMZfSln/rtxpaZw/tg3EhTwxdhT0n2ZxcpbKVFFWzr0NsJCZAiPVMaqbtVwSFNb9WhJrWMsAQwlzglBUSASWgGtlvXR7uA1fFxoERp1NXdzw2Lxhq4X2wOlGzeNYSo4GEVdkiR/ueWBZ63c7ovNwDKO9IMc/kvQ9pF+ntrfIGsjdVRAk7C0ueegMEYkAw+AVmrXxLU7+g3KxiJuwI8iR2sGJS/2UGgKhoSOSVW6TJ6YEHzH4TKRtYsMqGgtd6oCSpW0kdxjKhTGK4PqNbJpPMThPchrsaQCFpZRcJPKc189yNDWPb0OBg7XV0E/qcCTfhpr/WvS+2ghkB4/JYf7Afhh+slkUxzQQu2x+nE7YK0BvPqoDxLWta+6QQZrxO7/iVMd1tMlJlFaiNhZutgC4RcwLdrc4rUlRgJSUe/hE5dd2SLrh6GMxsh1NXali0NnOd9Sp1v0tW8Cm/S8zCKTvnxpdY0XWXRxJVANsYv2ID7oaqCJMfC7PMKyZN5eV0pfpdaMcaLHAM+1uZB4VlBSzk5gtMnuqP20TwbYNZ+8qpK3OwPnVXrv4CmP745fQCWZgmIt50IdSTZ5e3gtHMYY03ek1z71cmMWtSiOfIBpngBbVs7vzB5XlzzX0X/sYrPUeRFQ4Ud8mC35T0/+1DkC4zpH1jAx2zQyTAakQ/t4zUP6g+QhhN5/JdbaI2oP+fQHTUJQ2LSYfM8eFuDSNIGS+0hvw2BaBYxThOFIwODKmkA3akBGDv1da4MyvWJaOLJPj9xV4ErJJUS2YY8mUixQekVw/j8WEtVIDjMTb0qGpL9iwILVg0yYQvaJMpzRKhyFVudcW/3Bx48qVjikUelnTXmGrRMmCw/B742LlE+a3TuciHD24vo1sP9HLXiIzeFYG6x0cw2tUof+efCEeuCtdRp7OHtsUNNTtlVyy/U1vae1s344pf8Yc83knx3mzXyvEKOrn3RZQB8ZZRzpmXf1coJ933xrSuLk1H10fSOA3hS2i2WUz2E6BbT87yp9t3bRSJ9fqrQu0GID1r8qq/2Ub6x1axewXj/GvbFqCirz36nV+MGHA9WxgSXmX4jn2czfjk392upqxoKlkjXFyP22JjoOSh/2uz77z6R0gp2CsH+G/v6utoBGlHffk1yCY0nSEGtCs9UTucTjIS93xjxqiv1FKF3FTmO03OHBCYr+YCzlCcwuV3wTPmrEpeeHOxCguilQb+EEel0aNDU+QX74n06StK3cT2+wkG0Yz8gJsK1yn37iX/NxltXiFG1vncpGQOiAMyGvKUyjJin6U/+GWBgoUDOWV9hn2vKQNHNKSHD33WrvtSu5HOdDOfw8mxIEpAH0HVSrUi/tfyv7CaBYSp83PQzx2WfOawYmVAZlc25c9qcgTZROX2ix29iILPi1Blva53VC59MIOvENqpJqhpwdpRDyVZWqzTOID0hS/GM6Unk0MH5CpU3nu8PEkccnTKoNhH5suHdDr9rI6LmZlbTU57HqcyfSwgEwkJlZQkXclwHxgiRfzApllT80HmGCxdbkoFl4hertlwRYcZKP9y/TsPiYp7P5Rc2VBK8MWto9zITPE/S3BbKUgS5r9f9LQrBwJUf+d+gOEyBUs37HO6jnHe4VZ3TYOdJUY3wjXz3mJhpchz8TSjFQF3wYow2roxVaa5sV6LLYrZ+EcZjuaASEoyJLNAhweEnx3f8tlXFgxgjeBgiuxuwy+x5aZKnfn74VEhoYWnKYxLC7JlI1J0SvSyZmeDTJQY4R0Vl9dpB5IqHZpoYWb9NdDmbbkWjegN8dZ1oF6c070adEa2VvQQRgEl3b2bO6d7rava2qYN26MyoLjQNVwUxj9ypirw4boceWUXeKxDYGK6uqlwq2bOJvH6YvgIKoFayL2276A1H49noKLMpd4tKh2pDse98LNyk5WBsicNegdtXun47vU+dCsoFbUcFS0FYZROxkQYP/Lx8w8a7X2x0Z5FTRhH77CSBtE2FO8r3JeahF+pqjTUqCuEjI75eEPr5/AfgTJmq9cfVo5byj7K3b9EP+q/av/QGaTt5/LZVoNw2PhnqSjUu/tickN9hH3wXPp0xdiubL40NTZSt6Xnjr6/U00+N5GUH76oP5vpGf9j2WOtGnqV6BCLV9P9nV92tEuVPS2YPAK5D8nvlvj6Qx83RkAqlpmqyOKcEUoa0aOnjARNPP2DI/2M6nGSLye33EfnBGDunnNsLG59cRLfk+BbZoHVaZAbT83G6R30vhBGTx8B+HBNj1ccn3wsbaXZ5P+fCG61mVDSS+vNx0W8Q6qwANz4zmfKL7ML4QVqEBm+Y8dOXIoqfRe5Q344wWNnI1R+81XpYZyfauxVCOp3O3bGe3ptmO31FZQX5DW5Dx+VNnBzHgpm9PAhJIlj6UhKU3cBPc3z6LNdn5V1hUP473zX+TWligW7ZGWR2R/b6in0vM8iiPslJ+Rb6ZLaW3gFbTP95/LZVxuh7uPD0oPjhy6FfQkmvGTC/J39OK7K6CGvZXnZHemXfU7gwx39EvO2HAY7W9p+t4WdCY5YOhnvCWLA5G7NpT3cw657ULyorMvnQChXPmDQOQCwaWcFOaLwuhS8w4QKEH/Cr1mMfgOncXMZ/sx4M6nOBM50Af1rYPX37PN++oqX6ogm3iaDLdFnjq+6V4oMQmg0zQn3VgivvSLHBSmw2aW1lkXzLz672y//pUo3jAgGVdeZGNP40//f6JOQZUPWYL7stsNThKLXAkB+WTuKFzGXwTq1Xa6pINOLHA0mC9itmoytE4+5AoCJcHY5s+AsTThzSf3Lvf9XUcpaFn+XkMl99QOZsSKcFQlsBrgsaKV3WgRzYJIW9blXVdgM09Smx8lwaoplQ3ws98TVuo+mZ79LOz9Mw8dpEWWaDkJR+yPUYAeAi1rN5/LgW9EL0PBQElq3eQRKk2O6M6Q1IVt+9ZGECZieYd6z94vUh7wWAV0Y1ukVPHWLMXWgBXuuHeFsaMF4ZXYH1pnuOBiYKXSgcdpqgp4+tps9arx8SCNdmGt7NRm76sGwj53rA7Ec1xTs0p/YbWQ9pdU1xTkWkQwPAOGzYaHLPZc0cSbhMydd6p2FIcU5GfcGERmc3uuK1KEzpy2LjveWqwGl60ONUj7SBFLypXCDGDsOeIPf/5CvHXzrjY5IbRX/GZL23w6RETRXL13P+0CELll4QwKxNP1uzEZmshCXC92A6F4i5lVagq4pL4K3RG9hiqHhvue5LfGOzw0beJFAskxdqvXxQkAv8ID23Jl8RQIY2blYfHi9Wj8X6u9pfUcYLtF7RyClWIBSmMs8QHKsm3KM370HW+N/twewdhDVAPsffb+klZIlIABNv/LwCyA+Rd/y6v5V7L3ltuFSHn1sAwAbtrJNpPeM50fVYgAXKoTDkx/w10PadOZJBypkC6DC1/gptj7fWfzgg3ihqQUTNv9C1Y9o25CTkiyo7ijlggDVpOCnnSwULCPyiEG75plqGwCo/f6CqTHD4RtkkZpPHpmHSAAZQZ6wHgyXMIA0szqSDbmKWvXrS5z5AMOYJKt/py3ucZsTmvRPuf4i1WZ6bq4+rchsKy/S1QKvsf4AR00oc9iKK6853PUqTmbbtWFcMzZFFibJBO8fwar7XKNw7w7Mj0twh1ksivIOGBlRH5EkmVUDGKMqM6Eerf0tl7TRhCRpDCsYFoybUOonuoV61jglTFKAiVuJiVjzxuaeqJ//PiRuWxoFoTbPrLcd1tpOqFYT+99lGgWmCCAVKZ+V5M7xIGpOyJCkauQ6gUpXNy3b8usQULLB/1YCR1c5SNY+n2nYR7egj90fiBqIIIcgPlbfIji9iUpXmRmhjDTS+Up66L8aVdKmt3D4dwbWIJ3aeqgBYy0pv942aLOZyY/kd9UT4s3RSuGVMDOSX1OoC1zd2YUMTaDX+awpYUvq4N08ICA9R78f7xLjbDk61k4SMONxkUZ+9JCYScUn6oXJq7sqrHKdUjGuFK+xlTNBpPapQpUk6MK0a2nHeKXiHzavdhTDeYHFNlZo0e6VutQjgZTzzIF9LTJuZQu50TcS2v/rJ6d4cdRgfZpXhkYn4hE2jileIqFp1KOPU2BjlRLHuzln3Ba0JCkzIxzsEIlsxs9VHywu26YuXJn5hsxHDtjiQWvkEQnQiGGbCYAeQWp48v0fbKYNn46e27TWa+azn2H6kTfpMqQLcQTPq2xl3d/y7qxWbtm2XZt0Wex5JZk78vOJLSm56+iTW1sDZgSPg4I8PvP3u5+IcOVp8M8oEVq4xT0XaDpONODHimMc3y6vvmVrviE+LuGTfq6rfITzQxLNDFyAUb9U4neXnGMrZnxBJTvZYzHHAoV1+x/gXyX9BO8xbTpnOKCoGOfCTDXy2K8aTDZLAXr+OR+jv/DShcONwrYV0f8u9Ss0EOdukW8ZM6EluUM4+LN14VyL/Og784lLGb7vAvGmt7jHxReIFWCWWySIdzPSJDgDI/9KqKMrldOlvhG9rvvfdd8geBXwED1XvJFWA0OTB13r+kLVpFHJv2KeWOdYCxTBzhC8ttv8ZLJ/6QH+qhc/+ADl5/OgTB0YiLoOPvRnfGhCbbHkfKTRDnWTwlsvrZgQgKL77RIScKjlq0YL1fodOoP1WqzbdslR0rDCob1RflwWLc0ytciGVS/HsMwDenDmnZD2jinJTBSYO/FzO5i1wA1bu6UBuUCQGcTDx1napT/H/9goIVP7UxPUrC9xbJ+tRnvaRa/dqMQ1IIQ8YwMWj2UX8ILC8/2diNcUCAjSTNzh0c9etZhG5gLrVr6l/JrlyN17b4DQvwMENDy6HlBY3J3Jo8BI8v40j+2jbcPQveg+3dwa0tD6UGvG+TFoGvlA8QDi9iyVfQr4zo/KX/jDErfd0vanukCG0uSsAlxOxr2PB/f/vkHo5cDWcg3KdokFw9WhruNPyzx6yRAYxN0/4IaEZU07VwiDjyLrxVqZx/9ofW49okN6yGMyZspfBBO6UlvSbf9eFWVkR/GB7XWsmNllQs+Kw1XoRjr4P3oeIKLfcy1OIfOvVdw0h/loTEFXtFZJ4r8UDcyyQylelYVQCAQxSLJZLpprOBaiFeW7D+Zdtdd2KbDWyD7DvQsuxcNnHs3pdVLNgRPYGiVtnTjSFdOCnlLMh5zbAsVThPBzkQ9cXrNMN6AY2W0glhtprfaHplUOTbJ/SFmaEVYSf3kC1Z6VmX5g4D52gY+6xgY+qQQ9Uj4hDoNelpwElqDmTwmqXaoKAdwwmq3NplA++LE4hwjqRQVTN6TF/j2ppsITs5j7MLQdxftZjiC63rLONVX1UJHbffGzpgDpvq6bKZ2ExV+ykueC/CL0u9knTCoUE9903zQ615CVBp+MVVPJHIQ7OxUgqc2+McOdR99P0gbaObktAQCrWkVEkr/zSuOAFVwdAH+LgwDP3F4a0dQmAh6glnAoHiexe/9hmsgjKgBVixeAUImpXH2wdTyyGy/qnF5hcOEJTt/mF+PLDiGm+gv8UVaaldk9XvHXaf3MsQGbE2ZoXCNUEaiQsqvGyx6HZh25psy8NhnwaU2EzN/mwkLnE7GjrJoB0o+Svsr5ByBsaD5wzz3ogkFxOnhqL6CJ5pcETeQW+vbnG9CmLE8xbrX1SupAcsI5XQyn1Y0v+6yX95gZnChiR/gBhAqmGdB48TINXIWeKeydWb22bLu2Tp2SDvyFor2bj061eVJW5+TDgcV7PyJV74glr0xtHScSHkMJ/5XW1HmKo2IWI9BuGQGdijsRRQJtnqFevANbs3E7rzf4ZqyQGuWoKmfgSltH9KY0GKSK/BDJ3jIW8IzcfqVi0iJ7a5zTqQYIajyXNKib+eVa1BHUoKVEADVvX/+iWwCmB/LlH4AUbKM9FP1j4yyw5fyx3/zax2v5TJ7WmFtHJJ8wcdqjdU+CvxTdPbFBM0Jw37oVJNVwht9HgelkoBbQWwKp/ZZoAhOvIr+ORYvgFQItodx1X2RCkW0P5eCy+b3nVb6D4NILYXeHZYZqt1jJlF0fXkTX69HIZqfKOsPH8yRL8UPtRdXdKQiAz9cscbRz9XN5kav5SorORKm4ysYfN7fuM4CZ8Fs9H1QDeKwPxJWo3HC4st0hARI7Ex76DmseDhepub1o9ultq2kebGz/TBfjsL5csXsTQdCL7VyG+T5+BUcEe8UzltHm/eK7BRLKnmur677sG7OWconLR5/QHItNRS0wXd6VPFdxQQ5NGIKQ3446ZfMwfg6A3flpj/Pz8Q/VKAmsxUg6Jg4C8ozY+W0mUbbMvGCBT78SJK8CFIfu/EjUpJ3N15b1JsADwCyjjPTZ6mm06CrE8bEewtbZquqhUxh4vVYBMQRKeUZqF4pfgc7yIJoTHfhapMd6EDxIGdU83JFIhMEdEQPsDA1sFGYnQQc/jhMHmmZaJ3Ca8LNRPMIBYb4FOwKKkrCFKhTKuN7XGk8G2cmXoz0TMYOzwv0j0rxz0Jixar3xd+6xSw5BKkzICFy3SA70l9oQpjya8ioQbd8T6hQXCayYPABuohwT1JtzzF5qPmsf29nzruA34xSyMqZSZ0rOw9UStU3LYjKeXJZJG2wLfAZZ86l38U8bUlAVw6q7JK8WfXgqtzd6cdz7GlZ78kHhJV87yQS/LKozwfBkBVvyMoORCrwCIqBJyK7FWfHeV7k/YmJdqP+hE2IIDbzI5J+mU9Twc8yTUFm+E9txHOXbyH1IIXqwt90pprIU2TRnSPGnyngvvj1jE6S8HyaJZJAQL+bBoOB03whLUaI5HJUmM49h8tsFC1XkThOJHrVurKDNPT2TAl1YjDoUAjFOoijjf16QE5cEXT569814DlZnxUUmI/xMimhMFOU7oOO7wG3geWEQqELIdPgMd0PEBHi+hOldqK62XdAth+mTi1oGreJHJm/9/qvtXxaiD/FOpi942aL1zJXeJxXHMfNjA9Sxxt9xYrvIx3R5mu03JYpgc3sY2YzZ3UVpuQMdK+fg6gzGkTOVUpEs/BbicYMc5rNL/Kv+PH2JgpvlihwVpYTcgoP9oXXUhewFGBf3txEDJsrz2iRMtvWiOQkvzsXGbTiBfPZMTimm+kGIOp5a/bzM51394M+g2IK6bYYYFxxvE+oL4wsy+IJFve4R/rwJ69mJ2WxMXMqC22wxQw2AhV/s3VxCydTCDXMl5xLvhyjVcA64kJrgZ+UDhiTWM7A9CkCNXQd6XA63SmM9z+HdsA2a/QsLs8x9ovtdm0m9jkO4KnLwy6AZWnGEafvx6ROlhNlUDRCGXf69K1Iua2UG/KHemQ9Ba/FO8J7HV0MA5RGj+M2OoMwyYOR9gJkc87Iweue8pk/74lRFs2g7QhlvBnYtQLZzjFxQDexoPG0cCOxHAhg6piexP1XTR593x/pWphRaaN3DCJy1M1MxprTM0mT9AR1WxkDGPkBEQEycyA2oUykTLyqB24+JUzHQqhNrhtwYf0sJwrh1KH8cGEsvQlZgqknJT6QL8jOLSsjdcPqS7NwPil/cmNv9zimdKhGQBOewF6JAOJmfhdqiOl+Gs9yZsYiXJgMXkZlGZBKHOYpFW1YkFkY3ZDVnoRKhIaCF+cvUcE4vvYPscf6UrV6SnZnLbrpFAl7EX+do1/FtjLldAMqrGYPClYm3+QcZ1lOCP87CjQ3uGPSa1FfdWOxJfmzAW+OThvGFQAscuL2l5lTCNwSZJVoS5Xdb6SpDbqZ+9x/C5mA5q+N7QTA+bYGXX0wqEWZMPlCoJz4uwCpNbWvcUBZyeTJIrxHW9QEE24ZEvXcukslM9hS2PfmfRS/Vi0Jn3yMmuJ/+nASD2IT/JZmKcUjk43Q8psOF1z8ukV63S1a33JKJYBt6dBRwkh+RdgwCblow22WOdal7IhNrQR1FA1pJ2YTqOEKW+9xIuT49wHl8XBVBnyQrMkkjd5MhHuk1aoiJD5XZkD3BLGOYIrFWRqWMD7rqytussvnT8WzkF5Ft0+jo+8Jq9JLPats/dfOnm0hO0vGZYmN7Jc7G8zV+RvvFNz7CXiAsYWI+BWknP5EXbxk4iMM2FSbfDYS7vSWUaU6l+NQWadO25P+LvY1ptT3eG9ci6ZcqR9a0KvsKkp89sgy+BoI6LOgMVjfFs1Mt5D66/f7e/chSWVR96eZeBhRchmtm980f3bxd6EeRvxIa3KRDS2iZE3jsiHsqFr0twjTKr3KzOPA+psXsgY94xmHA+vbPytV7nw9tYgz+EwLUXwx3ynZCZ6UtHanILUBNPQ8p1tF6j38BnhoVvcNCX333BSFu87u+YfB2R5VklsvqZy7FASkYwK5QUrfnqyO/8W2RlxiobTfOKwGGGsbOFfbzgmMqdF7oDWfh/jEZh6QILTaO33dBKqFGLWpXRoEy9XZWEKSuZz1JrZaK8p0MOI2GbiS/5meC2IeGhWGCaN1BnVPNfN6J1ZgZfCORoiZKCD3L1jlckDNv8S/ELSsR8aSu0pjhkGhMF79weUvRNJaDiHsEpfSZd1lxnAYmdgf1gCzFG9xn1lGGihN2ehOr9DJQkd/lGspLYHkRUCnLqWvAR5uf6l9idrMPVnIELIU4S1cTOM7tDjeF4IopymF4FWOmp1JL4k9dou2R9jpDkMoqJG0wUChvxM6fpOKoyOgOfmvfl9NZLMv7lWToWSfw+QAj+y/QwffT//lowqAYFjRe445lwZD00SppGt88dGQBS7ki++fUxcPxbW6bQqiozyqE7ZTemOH5+v4dW2cMOsPBKUbzaktUlSLNPvN9gVAvL//jrvmekiXtpxhJNV36+0B+Ni8XOGRTsdPqotU3WJvh/9qGkl3jb+E7spbWaUTrsQXxsupq+Z6lQhoQFoO7sqiM27QU22gHUv2K8YpuwZ2wLWyZKT/bbAAtZdPi1Lx1ZDkHGFMJ6A/EB0BuoscFuqH9Ht7KZAsqyOe7nYeYgA7ifhi65IPWMq/E/DUI+YXxPkOFcqyNKBSFyLJ+G2EqSU6EMn1Pmf+p/ZVB55VT0UPKU4nzIQIhPUS1+k7rrIq3YWq4ilL2JVhoyToCNNWtSNW8dDFK19imHmbKNwM3hErKM/FtfV1dFg2n5+wSpKgz3qSxDYBRapHizjEsWsmsQYgGnRevHypWigDCGnHggiIFtNPaDWE4jy9x0NclgPDdr0I/pZ13OR/NRotCHXOoivCZAnQoIzvtYjDN604ERbReXbaPgz6kbnFexHBpjDc7qwzu1wCccAdyNhbDljWMeWuYR9ur9I/lYUFSoBpHNR3HDghBsye6WC4pjd7LE19p1+m3D5XHxCVNAk43gUiRH0CRTtmj3Ugx9HKABNtc92vPkj9irOsHtVBg1lFiFBRO3Tl+Os3oM31xkJM8j1VeRVNvMp4ZM74h+N/sn44GDLYl11zzy3GD/eaqfa/xbZL3f/2lZID5LRBgzYpZAzbz0w99S4+BacQVuBgMVa4nIgrKjaEkmVINsPs/18tFnydGmihSohxdnH+gTwystso1fQ9ucHpgPzd9TfufX7DeJIqAKg6D6mfZ2Io/cX+IaeWqhmInEkCeW6XdurA0wpRfsz8b3olrYhA+7yFy6izF7n2yc54ylXS/WzKEEg8dZbolCZeCWLbkRJzWbdVKLYiBQzFSwir2F7glkJ8dY576NJX4PeRp1B1XwyqGan/FL9UHw1wSdlgrEem2BtUP7K01qUg5IDqBURJJExwEwB/jvpI7SWDQDhuwtVKyrOd5jGUfGEOQH3xxiu3QPCv0B4MQhKSRlsBLYONO3E8Fqh1z2aqV3ErnsUNQ0ZIHtRuLvZCF/uG/S/MfmS0fssF+sveWbt44AIUPq/cA5hELc7+ODUAn6LzHIBJqq+bd0G+vB0Z0P95xPTlKPa1p4XP5kJob62nqH9W6sXwzoBIoJd0dWPrD+64BftH5Vq69OdfDQFO3xkl+rQsWF0DGuAXJLSI+ffvc+ikOjQ8ozKk09YbHoyFw+b7i3bc/WBjPGd+ljcuKHcXWO5iGJ1o4tUD0QdsNoowresHQ1kHpmifZ+qGR4b1X7jIPewPSQeXCbfC58+cpXg52/nsB60CVTGjHb1d5c8CnU54OvxCJqp25r0va12ZwS0ZRXjyA8wvU5DkhBfIlyM1jbDDdxVj+1+qdd3kzr/OzjLLhHBDeuVzAeE0rQIVNqylzgeTf/8GzwlWJbJoNWfnH8cYINoEhmvFDkh5rtAY5Ox0uNM2ZNOqJzPRcSqk3PQ6KSsOxsDY0XsTBlpRHsdC3jSidZS97JMj5lil1sZwLIRhL9DW6oZT6eo5cAem3iwXBRqWzH1DzIwWPdO+ADaIM+kLrkevXrrD9TK83ZdwVAluwNsG5MzS4AWnKt1ltsPlzQ/FSaiix06oLe3mR78nTWzPM+HKxQUil4TpUfpIiZthsOAzmCT/VWAJqFMNI20Gz4JlcQ1eejkLKYqvcxVjLdGT4R6Jmcu6q3vLl9nZHbZHa+8qLdNr85q+xHMWbd0Ao2PolVOnmkoSmo2y17LVj8IAHu++S0HQnn5ZrEE86qzBJV5z8fKMGJrEc+togq0qgH6f4rgT/iaI5dv1FDdY1BdrD5p0KSmi4P+PkkKODu0u+nNb50398meYbR8QRzqFYVWK1lrLDH80PsLpJelc5nKJ//z+jh4aovjDVJ33TyyE7l5+uXTjARtYBZF7lTsrZUpS0WJ6gGKHFTAzi2VeIoE9AdbzF+VSRoseU4DjCwIEDFKKPTVI4dY73UsoBteLltw8HhhUp9CAK6v/Z/eGedNndL77XoJQBUDWGjhThfxcVi6PFs2Khvoh3unAwG6IkL+QbXuqwf2nQ3ESqxESpVHk9nZ1MABLi8AAAAAAAAlgMrwQMAAADrNasiA///XfxyGLgI0Cc+Mkp34lwG9El/fUsB+dmcY5a2V2SX8GU5HtHdESL9L9eSnCXPpmVG7wiZnnhzNdSTtXUDEltf2iGQgOkPdkfb/3cd4FcRDcZaW9v29mIJYFty5M5Wrxc/9pCCEDfL/wb9d1b2iqt+cF8kTQoUoFlukjmF5gJgZnXzgqdNK7Pl4LRuZ4ZLJHoZeNWWZLDx7JHIHEjRR9yu9RIcDffNjKfFjYsVkVMO79LTj8LPuOwEhgezKWJ65LqoO5rZC8NtZ7/z/PYRVupwKc6ykuiJVtNUpNQm27kD5Fy3Lv6uxHl9AYYmJTrLl0eqjh1lAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADmeNYsgbKZb07z0fV3ON1uaSa0YLSgSpAOT8izVkw3PfVq1V9EtccRQ+V95nGLeJgadjPhGDw5ncGRWJn/EYkvoYPD2bRg/DEQ4eCrRuMF9/SwUPwvKe0UXMLxMRhYU1Hv6raKfDGk/C4Ys/5VXGz6IMXKoRivdwKVGsaby4b8vYvkTLiivEUimHC6GcRqh/ZRArTTkRidu5J5Ikd/dTwmzAYnT4S7SSMgL2ZcuxD0Rr3CxTiaQIC0W+lRKmfgqQALOr/DzWlcBvJvt4Vl1tISvqbtllE6AkVBfJPJ+IDjJyQVYcVaomgD5SE9KAzKXaIOyHIKFWZB4XIXR9Lrjhrdg3MZJ63g==" +_STT_AUDIO_MIME: str = "audio/ogg" + + +Modality = Literal["text", "tts", "stt"] + + +@dataclass(frozen=True) +class Probe: + provider: str + model: str + modality: Modality + + +_PROBES: list[Probe] = [ + # Text + Probe(provider="openai", model="gpt-4o-mini", modality="text"), + Probe(provider="google", model="gemini-2.5-flash", modality="text"), + Probe(provider="google-aistudio", model="gemini-2.5-flash", modality="text"), + Probe(provider="anthropic", model="claude-3-5-haiku-latest", modality="text"), + Probe(provider="sarvamai", model="sarvam-m", modality="text"), + # TTS + Probe(provider="openai", model="tts-1", modality="tts"), + Probe(provider="google", model="gemini-2.5-flash-preview-tts", modality="tts"), + Probe(provider="sarvamai", model="bulbul:v2", modality="tts"), + Probe(provider="elevenlabs", model="eleven_flash_v2_5", modality="tts"), + # STT + Probe(provider="openai", model="whisper-1", modality="stt"), + Probe(provider="google", model="gemini-2.5-pro", modality="stt"), + Probe(provider="sarvamai", model="saarika:v2.5", modality="stt"), + Probe(provider="elevenlabs", model="scribe_v1", modality="stt"), +] + + +def _build_provider( + session: Session, probe: Probe, *, org_id: int, project_id: int +) -> BaseProvider | None: + try: + return get_llm_provider( + session=session, + provider_type=probe.provider, + project_id=project_id, + organization_id=org_id, + ) + except (ValueError, RuntimeError) as e: + logger.warning( + f"[_build_provider] Client init failed | provider: {probe.provider}, " + f"modality: {probe.modality}, error: {e}" + ) + return None + + +def _build_config_and_input( + probe: Probe, +) -> tuple[KaapiCompletionConfig, str | AudioRef] | None: + if probe.modality == "text": + cfg = KaapiCompletionConfig.model_validate( + { + "provider": probe.provider, + "type": "text", + "params": { + "model": probe.model, + "temperature": 0.0, + "max_output_tokens": _PROBE_MAX_TOKENS, + }, + } + ) + return cfg, _PROBE_INPUT + + if probe.modality == "tts": + cfg = KaapiCompletionConfig.model_validate( + { + "provider": probe.provider, + "type": "tts", + "params": {"model": probe.model}, + } + ) + return cfg, _PROBE_INPUT + + # stt + if not _STT_AUDIO_B64: + return None + cfg = KaapiCompletionConfig.model_validate( + { + "provider": probe.provider, + "type": "stt", + "params": {"model": probe.model}, + } + ) + audio = AudioRef( + bytes_=base64.b64decode(_STT_AUDIO_B64), + mime_type=_STT_AUDIO_MIME, + ) + return cfg, audio + + +def _run_probe(provider: BaseProvider | None, probe: Probe) -> dict[str, Any]: + result: dict[str, Any] = { + "endpoint": "llm/call", + "provider": probe.provider, + "modality": probe.modality, + "model": probe.model, + "ok": False, + "latency_ms": None, + "error": None, + } + if provider is None: + result["error"] = "client_init_failed" + return result + + built = _build_config_and_input(probe) + if built is None: + result["error"] = "stt_audio_not_configured" + return result + config, resolved_input = built + query = QueryParams(input=_PROBE_INPUT if isinstance(resolved_input, str) else "") + + started = time.perf_counter() + try: + response, error = provider.execute( + completion_config=config, # type: ignore[arg-type] + query=query, + resolved_input=resolved_input, + ) + except Exception as e: + result["latency_ms"] = int((time.perf_counter() - started) * 1000) + result["error"] = f"{type(e).__name__}: {e}" + logger.warning( + f"[_run_probe] Raised | provider: {probe.provider}, " + f"modality: {probe.modality}, error: {result['error']}" + ) + return result + + result["latency_ms"] = int((time.perf_counter() - started) * 1000) + if response is None: + result["error"] = error or "no_response" + logger.warning( + f"[_run_probe] Failed | provider: {probe.provider}, " + f"modality: {probe.modality}, error: {result['error']}" + ) + return result + + result["ok"] = True + return result + + +def run_probes(*, session: Session) -> dict[str, Any]: + org_id = settings.HEALTH_PROBE_ORG_ID + project_id = settings.HEALTH_PROBE_PROJECT_ID + if org_id is None or project_id is None: + logger.warning( + "[run_probes] Health probe org/project not configured — skipping" + ) + return {"skipped": True, "reason": "health_probe_org_or_project_not_set"} + + logger.info( + f"[run_probes] Starting | probes: {len(_PROBES)}, " + f"org_id: {org_id}, project_id: {project_id}" + ) + + provider_cache: dict[str, BaseProvider | None] = {} + for p in _PROBES: + if p.provider not in provider_cache: + provider_cache[p.provider] = _build_provider( + session, p, org_id=org_id, project_id=project_id + ) + dispatch = [(p, provider_cache[p.provider]) for p in _PROBES] + + started = time.perf_counter() + with ThreadPoolExecutor(max_workers=_PROBE_WORKERS) as pool: + results = list(pool.map(lambda pp: _run_probe(pp[1], pp[0]), dispatch)) + elapsed_ms = int((time.perf_counter() - started) * 1000) + + ok_count = sum(1 for r in results if r["ok"]) + logger.info( + f"[run_probes] Completed | total: {len(results)}, ok: {ok_count}, " + f"failed: {len(results) - ok_count}, elapsed_ms: {elapsed_ms}" + ) + return { + "elapsed_ms": elapsed_ms, + "total": len(results), + "ok": ok_count, + "failed": len(results) - ok_count, + "results": results, + } + + +if __name__ == "__main__": + assert _PROBES, "probe list must not be empty" + modalities = {p.modality for p in _PROBES} + assert modalities == {"text", "tts", "stt"}, modalities + for probe in _PROBES: + if probe.modality == "stt" and not _STT_AUDIO_B64: + continue + built = _build_config_and_input(probe) + assert built is not None + print("ok") From 53ba5ed3fda9b0be3b6ca106558e5cfc359436ba Mon Sep 17 00:00:00 2001 From: Prajna1999 Date: Tue, 7 Jul 2026 12:11:17 +0530 Subject: [PATCH 03/16] chore: cleanup --- backend/app/services/health_probes.py | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/backend/app/services/health_probes.py b/backend/app/services/health_probes.py index d5f3ec70f..1b2f475dd 100644 --- a/backend/app/services/health_probes.py +++ b/backend/app/services/health_probes.py @@ -40,14 +40,11 @@ class Probe: Probe(provider="google", model="gemini-2.5-flash", modality="text"), Probe(provider="google-aistudio", model="gemini-2.5-flash", modality="text"), Probe(provider="anthropic", model="claude-3-5-haiku-latest", modality="text"), - Probe(provider="sarvamai", model="sarvam-m", modality="text"), # TTS - Probe(provider="openai", model="tts-1", modality="tts"), Probe(provider="google", model="gemini-2.5-flash-preview-tts", modality="tts"), - Probe(provider="sarvamai", model="bulbul:v2", modality="tts"), + Probe(provider="sarvamai", model="bulbul:v3", modality="tts"), Probe(provider="elevenlabs", model="eleven_flash_v2_5", modality="tts"), # STT - Probe(provider="openai", model="whisper-1", modality="stt"), Probe(provider="google", model="gemini-2.5-pro", modality="stt"), Probe(provider="sarvamai", model="saarika:v2.5", modality="stt"), Probe(provider="elevenlabs", model="scribe_v1", modality="stt"), @@ -205,15 +202,3 @@ def run_probes(*, session: Session) -> dict[str, Any]: "failed": len(results) - ok_count, "results": results, } - - -if __name__ == "__main__": - assert _PROBES, "probe list must not be empty" - modalities = {p.modality for p in _PROBES} - assert modalities == {"text", "tts", "stt"}, modalities - for probe in _PROBES: - if probe.modality == "stt" and not _STT_AUDIO_B64: - continue - built = _build_config_and_input(probe) - assert built is not None - print("ok") From 034b5015f49893a70ee121c3f70904851e9e3645 Mon Sep 17 00:00:00 2001 From: Prajna1999 Date: Fri, 10 Jul 2026 15:28:12 +0530 Subject: [PATCH 04/16] yolo --- backend/app/api/routes/cron.py | 56 +------ backend/app/core/config.py | 5 +- backend/app/crud/stats.py | 19 ++- backend/app/services/health_probes.py | 204 ----------------------- backend/app/services/stats.py | 134 +++++++++++++-- backend/app/tests/services/test_stats.py | 190 +++++++++++++++++++++ 6 files changed, 340 insertions(+), 268 deletions(-) delete mode 100644 backend/app/services/health_probes.py create mode 100644 backend/app/tests/services/test_stats.py diff --git a/backend/app/api/routes/cron.py b/backend/app/api/routes/cron.py index f6f68937e..fc888e908 100644 --- a/backend/app/api/routes/cron.py +++ b/backend/app/api/routes/cron.py @@ -9,8 +9,11 @@ from app.core.config import settings from app.crud.evaluations import process_all_pending_evaluations from app.services.job_monitoring import monitor_pending_jobs -from app.services.health_probes import run_probes -from app.services.stats import collect_daily_stats +from app.services.stats import ( + collect_daily_stats, + format_daily_stats_message, + post_daily_stats_to_discord, +) logger = logging.getLogger(__name__) @@ -35,20 +38,6 @@ "recovery_threshold": 1, } -HEALTH_PROBES_CRON_MONITOR_CONFIG: MonitorConfig = { - "schedule": { - "type": "interval", - "value": settings.HEALTH_PROBE_INTERVAL_MINUTES, - "unit": "minute", - }, - "timezone": "UTC", - "checkin_margin": 2, - "max_runtime": 2 * settings.HEALTH_PROBE_INTERVAL_MINUTES, - "failure_issue_threshold": 1, - "recovery_threshold": 1, -} - - DAILY_STATS_CRON_MONITOR_CONFIG: MonitorConfig = { "schedule": {"type": "crontab", "value": "0 0 * * *"}, "timezone": "UTC", @@ -158,10 +147,11 @@ async def evaluation_cron_job( monitor_slug="daily-stats-cron-job", monitor_config=DAILY_STATS_CRON_MONITOR_CONFIG, ) -def daily_stats_cron_job(session: SessionDep) -> dict: - logger.info("[daily_stats_cron_job] Cron job invoked") +def daily_stats_cron_job(session: SessionDep, hours: int | None = None) -> dict: + logger.info(f"[daily_stats_cron_job] Cron job invoked | hours={hours}") try: - result = collect_daily_stats(session=session) + result = collect_daily_stats(session=session, window_hours=hours) + post_daily_stats_to_discord(format_daily_stats_message(result)) logger.info( "[daily_stats_cron_job] Completed | window: %s", result["window"], @@ -176,34 +166,6 @@ def daily_stats_cron_job(session: SessionDep) -> dict: raise -@router.get( - "/cron/health-probes", - include_in_schema=False, - dependencies=[Depends(require_permission(Permission.SUPERUSER))], -) -@sentry_sdk.monitor( - monitor_slug="health-probes-cron-job", - monitor_config=HEALTH_PROBES_CRON_MONITOR_CONFIG, -) -def health_probes_cron_job(session: SessionDep) -> dict: - logger.info("[health_probes_cron_job] Cron job invoked") - try: - result = run_probes(session=session) - logger.info( - "[health_probes_cron_job] Completed | ok: %s, failed: %s", - result.get("ok"), - result.get("failed"), - ) - return result - except Exception as e: - logger.error( - f"[health_probes_cron_job] Error executing cron job: {e}", - exc_info=True, - ) - sentry_sdk.capture_exception(e) - raise - - @router.get( "/cron/pending-jobs", include_in_schema=False, diff --git a/backend/app/core/config.py b/backend/app/core/config.py index 8d8d79f24..693c71a6f 100644 --- a/backend/app/core/config.py +++ b/backend/app/core/config.py @@ -46,6 +46,7 @@ class Settings(BaseSettings): PROJECT_NAME: str API_VERSION: str = "0.5.0" SENTRY_DSN: HttpUrl | None = None + DISCORD_STATS_WEBHOOK_URL: HttpUrl | None = None POSTGRES_SERVER: str POSTGRES_PORT: int = 5432 POSTGRES_USER: str @@ -190,10 +191,6 @@ def AWS_S3_BUCKET(self) -> str: EVAL_FAST_STALL_THRESHOLD_MINUTES: int = 15 PENDING_JOB_QUERY_TIMEOUT_MS: int = 1000 - HEALTH_PROBE_ORG_ID: int | None = None - HEALTH_PROBE_PROJECT_ID: int | None = None - HEALTH_PROBE_INTERVAL_MINUTES: int = 5 - # AI-assisted prompt improvement settings. # See docs/srd-ai-prompt-improvement.md for the full design rationale. # Platform-owned Anthropic key shared by every org/project for this feature, diff --git a/backend/app/crud/stats.py b/backend/app/crud/stats.py index b54b2f95a..1f35bbca0 100644 --- a/backend/app/crud/stats.py +++ b/backend/app/crud/stats.py @@ -13,9 +13,7 @@ SELECT o.name AS organization_name, l.model AS model, - COALESCE(SUM((l.usage->>'total_tokens')::INTEGER), 0) AS sum_total_tokens, - COALESCE(SUM((l.usage->>'input_tokens')::INTEGER), 0) AS sum_input_tokens, - COALESCE(SUM((l.usage->>'output_tokens')::INTEGER), 0) AS sum_output_tokens + COALESCE(SUM((l.usage->>'total_tokens')::INTEGER), 0) AS sum_total_tokens FROM llm_call l INNER JOIN organization o ON l.organization_id = o.id WHERE l.inserted_at BETWEEN :start_at AND :end_at @@ -25,6 +23,20 @@ """ ) +_LLM_CALL_COUNTS_SQL = text( + """ + SELECT + o.name AS organization_name, + COUNT(*) AS call_count + FROM llm_call l + INNER JOIN organization o ON l.organization_id = o.id + WHERE l.inserted_at BETWEEN :start_at AND :end_at + AND l.deleted_at IS NULL + GROUP BY o.name + ORDER BY call_count DESC + """ +) + _LLM_MODALITY_COUNTS_SQL = text( """ SELECT @@ -95,6 +107,7 @@ def get_daily_stats( f"end_at: {end_at.isoformat()}" ) return { + "llm_call_counts": _rows(session, _LLM_CALL_COUNTS_SQL, params), "llm_call_token_summary": _rows(session, _LLM_TOKEN_SUMMARY_SQL, params), "llm_call_modality_counts": _rows(session, _LLM_MODALITY_COUNTS_SQL, params), "job_type_counts": _rows(session, _JOB_COUNTS_SQL, params), diff --git a/backend/app/services/health_probes.py b/backend/app/services/health_probes.py deleted file mode 100644 index 1b2f475dd..000000000 --- a/backend/app/services/health_probes.py +++ /dev/null @@ -1,204 +0,0 @@ -import base64 -import logging -import time -from concurrent.futures import ThreadPoolExecutor -from dataclasses import dataclass -from typing import Any, Literal - -from sqlmodel import Session - -from app.core.audio_utils import AudioRef -from app.core.config import settings -from app.models.llm import KaapiCompletionConfig, QueryParams -from app.services.llm.providers.base import BaseProvider -from app.services.llm.providers.registry import get_llm_provider - -logger = logging.getLogger(__name__) - -_PROBE_INPUT = "ping" -_PROBE_MAX_TOKENS = 1 -_PROBE_WORKERS = 4 - -# "Hello" ~1sec -_STT_AUDIO_B64: str = "T2dnUwACAAAAAAAAAACWAyvBAAAAAK8Ly3sBE09wdXNIZWFkAQI4AYC7AAAAAABPZ2dTAAAAAAAAAAAAAJYDK8EBAAAADsN7gwE2T3B1c1RhZ3MNAAAATGF2ZjYwLjE2LjEwMQEAAAAVAAAAZW5jb2Rlcj1MYXZmNjAuMTYuMTAxT2dnUwAAgLsAAAAAAACWAyvBAgAAAKnTMEhb//9A/wX/dv8o/v8F/wP/G/8e+/8S/zz/LP8J/wf3+/Hu/wT/Gv8l/w//DP8F7/8E/wr/Of9L/1L/QP8+/w7/SP8x/1n/IP8A9/8Y9v8C/wb/Kf8L+v8p/yD/4/x/cqF3yWnKJA+FRE/nSs5qstg30xlSAKfIW9e0H4hx095LUdI3HKNmbgjnKiDv3IlFUdpy7t2wPk1WZuuVX0o3MGAITvlC8ZVdG7R890ps2WXgJatlobIk8kSFl57Ispzj2WlqDg3BIwKyuQDIXFY49jbiiaJJ/2ln3N9+uTUe6pEr4GgTJeQOP5PM3wW8HJ8+2Fcccx6J7nQ9Fwwe+ye8YBWsHE6KHkKslGKqDuslGnQWiK9TdNnQuWro8YSrvnQ6K7VDSp/6fSj89BzyuLJ+VuyhAA5Sp3dYtytIBnR8OZKDR9rHG9Xk1gdZc6FX8p7Iwx23M3Kgg+dALdI1SVhHzoU1yNhlhz8wjqWiTUDwAAAAAAAB4tu7QcxPl1EsJBQi/OEnNcntPEWmTJUMnz1p+YbN2pKP7QughtAJibW3IjF8xMiQo+aDSo3/60XaFjProF4w44EheWxDL2v4SxPVmU0h/qBG4syaVwDXrUO0pbvw+lAVrp34fSoN0T01XerHqRRpwmlbbitWcauaQFtvYVzqUL0xkqQia7v/HGLA+5dmvAroSk4hWT8QOqq66DtdTyAJiWP2basqDIUwxBVVOnkMbHcrFScvAR1A9zayzA66G73Cyrbd2I/LL+BlRjuJ2pi8c9uBosB8HBytcvJ7DlSChPxIR7Iqtiq5uN5cKakTc03BNHlEOaeDm1EfkOfOkRmn3r28ggDJl0otTIWVz7wyznTfmvH6HTiyWe4AKlIJ3TNH8gdEJEvmQOv8SzXtqGlUeA77FwnWhtCsi1FMLlJNxA0tKIjyXNXwtJLvNd2sTfu0Qz1EnyjMOd16LqbYdHcvuLxLiTurviijKYa37+KQlpa0JyJClur5EfVHsFWpQC/VqGxFY0YNXVpCL8KV2slPthFj/5NwzOEaTPZPGTnJXe7fO6PtDPsUY/iz4q0gM798WKrPR803lqxcCgqZ+mCjGsbF+DCCYj3fwISbj6d/rbE5b0lWVzDNM/HkTCS7+exWY4iEdkh6gMRF+u+5rMBZ8BDBafDNPwHlEnllSdJKkRUSoXSvT3SZ6FqQkcq7cJT9afqEaHECNvb2DL8pg4xR0LQEruNgCaWg8LWvXPx8r9vzrAjfHOuuxMXH4DOEcCNj+Cciy/FIrFatJiyS21xuavTiZnH1SvavxOgolhAqKBDX6pnaaoAGkbeMi+kxGO7xS6zjG9vNzN8cI5qHOfr2LNGcvAMkvtC5qt6oyvW/OmUFyquyDN5Zb+m6CHGvnbeuvS2O72KxPhlWSQTYhp61XQljeUhhYKcMN2zOXwXY4QYWskt5YXgeexpM/NxMwzNSgawoojMtwrDi91z6VcuJVmIcCg0Z75ls0gQ+CfP+jbkTJWxiHKfs8dX2FHMyw58uPyu6jAG3tlNBeuWkh1/JfyAT5oZ84uXUA02zLDJTyUpHPhjEVBKlF2CFxiJEXVOlclAbL7idHaQkqTy6RlkwnjrGzygajqZ3nkftGHp0sapHwEDBZqURvTZV51V4kIycKmj5AtUBQESyCUJj5w06zFqxius8flGa3bIkFjHXitdl0ttsIrs0ww/FMZ+0HzRVzrlIHZuS1qwErW14/iJFJUv8wpnbXGNI07NclvoqRa64UVzpcVxEvvzaKdxRUbBTj2nrkRMM2mIrY5vXHTr6fRkF7M+Re8SvwmiFY/s6X6JaR6LqmFheGJkBwpbz8SpC/9d2BRl3VtC/GfWMnnZ8pvzSSU1swebkQb6tMY0wgzxeRn7gGxuCJcUoe8C75ThqIlYUvCbvoRKW13owiQlIcuTQOgk56VeqrHq04O5kyS6FGjenHbM6gKNq0iLzb5kD5r3WRNyZ5FTAy9VWVGzuezfHc2SnHPJWj+KaxNd8hWFdYoUclS3G7fjNqggRv8wQ5kQy/do8Vcl4Gwtq4A7XtitY1ZKYVv0iOyUtq00b1XLTbzA+ZZNLsplGSqI4DSTbzPOpeScWilFgNoUe0DUC0KD5pQqh0HGO/MICFpXbOtSv8pvNyQqsnY+gTCTRv77BseSsQbyT5AescVq+xBtVhs8t/JxmwpZ+JLNptgYhPGNHooiuAtgvESv5wkCqP7SjzpoMF9NblUFYkkCY9dG0qyJfhgW6o98eSFgyUqL+3YusDQFxogKz+GkSNXVxzHRuBT6QPShnRkPgDiBkL3usUezDEAEIPw2Wa3bBZRqryM8BCFKU9jRfHhGPLA3DU8YIwx4rdJcNiSil3LCqcmmY8QGrog4gBV+cVLoMqjevksEQNqYadQniIssNGkV97wW+RQrgdNTCyOdGx25b8O+FQsB42Ha7tmEQSFxyttlqgAoKnK97YQ78wClklb3lYxavDs1jF8lIB0w8lrV1i9MRvLEEJKpMdAhkB8ZSJ5ZlMqhuewjMxlrutNCMREjXLQAmQn5bLxFchjbNhphMBPXpfZUc+j83KPEr3m1IMNV5wgi2AFlW0uQ0hGQt7PJnvvbrVQKVi9xMQH1YkccfS/qVoZaGefvr9Ss+4HLGavpheS/X0QKavu8+g9eWkL5JoY75Der87JCguq7rnyBYBCd7Mpi7mRHr8ibi0FdXyzUAWc/pB2LVuI84GZ6oqfQUHmYCzMmMhHTrM9BY8BY8BIi7G/YhUDrQ9fYp0M7Ejpv5b21llJ2dN59dwGUQRv6S22xJJVQkIrM2/7y2EPzAK8XkliTK7UZxVSrrMCmnCijk90WYa+YUFoHTi7It5PNQppOCWDURdDbQIxs62BxmEBBhAUhvX6QrYJmunX1nMAHpFLK4K51HpqjFBLZVRhX1kUUnwEmJxp4v7mHM7n9rAVaJTcVZVZhKK3qfrJ5FtE5f/zFQdzNntux3vBzgpVdLkXgaEqvgc2jud1oWTH0NA6CBFPW2JUanJe9oKHTibX9mrVEBxYoPUHsUOh/tusIZo88NF5kHbJqIFvjmecSDi6XPixay+ZuWcrVtuHkxRjCF1mY4Az4Rc9aYrHOC/gGyxY0s4NgqKAtJpkKTHmOFcACNu2RsBQDYCVUxrrVOk/zANIxJy0Jai68H2KXedNd8eE8OKtFS8v5PnuknbCx+AMKmuU/9IHH0eoV06ka4c1orihLo1y/Vu+EOWGvkkbrAVlxODNDEvNdXkG+3Y+y4Y8o+aLhBXNcqiWbzDUtuXRsjIn2UtLofb6MaqMAKCkmfhpQ3ipCd5gXjsVcj65TE/kDrwCmvmLRayjgazg6IpSKnd0PUtnUjedc2bG5qMsLQqCPElJV9LMiQ1/2GihTjlS1U36FWIDgLiMuITOHccjLhF/phI5gU2K1zc9oJuSFZSKdOrplmTkK3oqj7ERBa84PUZXw5LvQq7SXmZVws4gQvHmq+eoxJBv0OeTATMzNmYadCWs4J65Jy7buSk2tj8P2/3UVElzTFFPzAKq1LPhtqUhArK5hA+Oj+I46RU6aaf3YPIiaFaamq1fKKoBWwMfA8sfKxFCYfEJncvXOIfc/PNUNNtceBtsm1pl2DhgNPQrYSreid/niE/xQwUiUXEfJ3CVR02srh8HaDfSLDmcj2e4bnmKsaG8fFBPIDY2nAt6skIRZacfMvQvK1IRXlRDkAenS3fge3EkIEYQKUdpQEr2oK8Q+F5SvtH5r6Iinesha2Jf9JC6TvpwgcSoHEsWM6LWyJV+NntV2laJVsg2yHAmju290pH3yobZZrkvY/yXoXjidIKcURTgYD/b+G1cJ6P1aTqdKuPn+KpegTgfgfAsaRov4eW9HuN7m6moq+QTx3WS48b3//Jb2StWtSc/AMva22mPzCAhSn87TC7XMR1HfFO5RJVCsXj4Vcw9ceoegZNWhn2sh4HXZCA61AKyUJRBq5zNOSVCzIYafumbPwf4nieScQ06T34ykr7Ne6qg5AFf57OPN8OMXKwiTX/2zAXQg9VpsaZ8YOk9cz/EDV4RAzNJCRhELI16AGRONtWjyNAoOsdAWbK0P4dxfj+2I38TiKwHSwzUbwozOijf3iLEBSPRuSZ5dZP8jz+HluMQZRJNtKB+yiO2o2LJLuy7yHA6K10Es/KJIP1czSBudKm+QEa9tNOk+jWzXxLY/AQWVGAcRIQ2PcVtRs7Hw9CZCEwBEDX+OOM/CvoLkVarKb/MApcmlzlkoe7rLEh8udVvRFkJnolTjpKqYGWiTo4hDCxqFL56Zw/tETKmDTunDK8uc2v1tqMpCgogGg4BGvZYHYaT1+wgzi9MsnyldgpZxWIVq7DxrM4LmqNdLaYfSkiDAAgmru7Zzve0FSlTlhcAcLa/dYUzMBzXbSYpsPobyj9078iSqE5UEJZaBv0QuZKpHVYEicsgK5CK63EBIKVO4TyLfyT0oEPFUc5W9YeZkiL5inPvhQ9URUM0SWRL3YO5KlrgRRPt3oJzOyxjNvObXm7qt4ALe3BWeZGl6qFliKDuHRdG/aS6S7Dbv2MLXMpLPQ+0+fqCXHbEQPfoLNnzyMud79TjVJr/BZ2k4xoBqh/MICUyL40CwGH1oVRRcCG0NrZn7vcx9GDFAgBc0YVW3ZyODEwmawCzkzfPJmmlwG7pU9+I7uu7RWb+jT6GF6Hvfo/WXY4/XzpGc2/lE+ACkSquJ/fIvBFu1pRUwj4RPPCrqPIvlhRhuxPaaml/IfJbTMsDbUMPDrWtYPO+RGmjake3vi9ilM3O4cVD24KbDntdTa6Y/wS0R8+Qng5543xadRteBBy4i0YHuOHQQbed0vhLcqZR2h0FGjK70R6yGT0THsDQqIVpioUBrBGF09XcO2IdjhnpTcqZRcdk/5/9inwem8+SD5bpd4NtrTf1HZoY6IoyIZS9WOAG5sFCTbcja889p6krbof6V0Tzi+jGGXsQs14wMCfEEdYPz/6914e3Y7F/PjCGXraQkAJA1aAB2CQkiT1kbRM26i/MGdRjc9j/u50ZEirMdylXWu7KohwjfNk9ZlKfTr93IvU4XE5Hli5m+ukf+a8+LGs3jKCSqxglwymSRMoebFx7rixa7HrinW09L7385dSXs2W1U7xuNYa6kdOeGyfpQ6zPGcXrrUyPTlbg30lMxYgbb/0Pq/HEWVNs1x60hqeF5BFVPkMGktxfIhUpiAoHPVjh6MTmToRK/Y//fa6TXBgKaKc6qjT4A1QgQ+/cRpMKpybRsSt+77ZB5Jp7X8IK24uodcbwZLbYBn9UVu7y8iijuQY4lUQCKNA5bDm/VGgycLGMfU5NCagjeX4lw9X+mdSEjXn5bKb4kKPxbttRO0Lxprlz/TPWCXzCHtDfSPNH5bv0DTdx3rEDgs8EkixFJJsnagACWvSd/UjqH8wCtwXqSDfcYfVFx2tJGvdnBccxJF8WUXtV+e9TD0wK6U3+OAwPqOMFCXxbQf/5QH5AmGyp0bN/g5nrFAGfCMQfRhCrpPLQDMRPCtKRk2OzX07bxgcwKI56f4DOG20nhlEkw+273tA3w2Jin16RMDz8l/Omg13C95V+zuSavcElGimo5QYoQejdcGqHUTzVhjTT8OqDh15eJni8fGWUlNJWWBPKK6ZZano2yPBjrG7sYEl6raf+gidczMorJlFFA+AbYEUB7IXjb+SmFiap6jOZrEr3wTYGZMfKqlrGL5ldc46cmirpTfV5MFOkbA7x0YxwKimW4Htqa/AEjj+1JsAFBZoF8Uppz8wClyQsy2b1ZOuztjy0XnGUe+yzTkuLbaP+CRYKouZwlCAsvVyqd5mqLbQCQj5Yv3VyouHC4SbpDHtSfFq+fjRCGfNUrj9NxFrWC5EqF7YaX0JtiwA2yGajQMdPN67hRjhIGAZAhjydmKNotq4lRJJoKMDZfIwcFKo3ez46sgO4tjouCO+KMcR7vA0GXo2kn9Nsr6dTCSXfTRavAiqaGvIsXawo1VKy5uILNNp6UK9AzSH6P0UnRc5t57hw0I8NSO3ckz193P8+ydKShJ4MxQIv9twjm5oFoSKTnxDPhRWKfN3uzkX/QTWK1ZwIzYd2f1K0qjfMsdhDNCLdkrf186Qz8maoIW/MICFpOVfSrKI8sCefkY49jcsDXBreZ3L5i8FL4JlgdmkVJDMxkf8nTycFz/AwzMRa8Smly9LsOD5uWCoTxcs3iG+tfvHe9Ugr5JUzMaDDAVkxM0WM8XCRqbMZk9rmuQh5VXYW9mALTST6SkvYkTl4KqJ61BEwbfSoUzUhJsouSrxAXu7ftBgCRLdzm5L1L75dz8bCP4+odOD8pJPT3yyYgTtqWDq/Oqd9ibq1zk0xkaPWcJGRXfL9LKPcq62OJrWPSRuAuWgSQk2stWRa1pXFM914JsVQ6FYrN/GFrzSHCl5bRXGqEFFiKltsADa0lZ9cqwIq69jfzAKXJa2D4hhpzLI7Wq3xvFGEKmw1+53J3OVuD29BZ7f+8gCDI9tOkFubOJGiyzxvkgi3W1YVWaYM88FLHkZWs9QBbS2qFyMR0lp8pUELAlTNCCFkaPpzT7oZ6Is9qJ5oWqNt9HAOM4sjpyAiLAxI6fTL9K2dc9rFjlfxKC5ghfIMAk3Y2uPesCen4fWkHn8qGHhVYF7ItPK3PLAGRylcBd8PnQaK8JLGwvhIz19y9n0MrpEnq8mdtqIaNhmvLquuKpjHQZM9DFIdDInZSgJ5A1Dt2AQQdT0ClRp4/Sj9V/CpRQg7EfmVIWLPqAG3vTB2a/bUAD4JARWOmH/KxHyp7SG+u2ZNqGwRMWnBwc/4AS+tUG8ouoq5+TQ87jW/jv7RxGea2q2QWvhXGNsACF9lsR8VsM+k7IFrYcm4KdwtXq1njasNGO++3qO93OEuBjMYN1vpIoSJThuxOXVd52i+RhBQ8p5ZcfhzxNhawnZh0o3Tu6dL8eqM8sZyPjGfgF8mSJAG4lIgPwROjh8TVRB4XzMNnmF/caiZRT00rkLa1R7j4zCwu4T5eeFrN7UszqwEY4XCV3sJExOwYhcKY0hJ8lM2vnLa95z6Hfu++PS+hWd/7QPJTyCa4TfvmgNZmuwAHwPDa3tWq1AeRHPvyq1kavbDHxTapblL0HctLTO/lT2IpXo8vPoHJHudE08B3TncZGTd24v5gbBwerFHg7ADfmJ3Tpbk2c2V0+h8cCWw8i5k449OXX7kOa9eVKGuDyxwTRWt08a/hgZa1W8gLK8LakLECZrwVXU/XmWu8Bp4tJT6A2pMpkDVSIzgVg9xfmrr5n+b0hJ9v/LW0bzmzgmwRxdXAA7o61U0enlGiBJq3naL3xtln9wl91uUxVWwMcDBDedUrGLE8XzCHT5ChQX8tLaujIwoUDj8McDh+XmV7ABlIa3itb2xeX3dqm+rj5j2ASXvyKklVQITX8rJ17eTLtKwAhYwjLvAUCqhlRO9X2bebMMfQJAxcINtBewkl095Cz8ysqjy3dYPaHyIBG7Sl0bca5G9UMghtRgNB46SVp3CwKMsIkj+HwGTHA1LpQ5dDTndX+CG6TqmxHzLCcEewPV0FmJLYrATKsngFv69325fdOZAIFgmlZ2Qw7oALWSzifCyDCh7RwdZAmD3ju8tLB10x2vHB4WgZcLfG1jzTVx6IrsPYQQwTpXHzqjWFfJ2CkstjtQcokAkKG/lGhvJm5gvGR1CeuqFZ8k2fKTB2sOeYtBJGB12GjbburEr+ZM75oD6KKyFNsi9GFAkcwOMRflJ/gWdYsrdbqkp0u/Kyc38ehe6mw09CetgdISbO2506aApuD9ZrhSeAFQ2k5lsU51F+n+8vsgh8MEyKeB9TFgO08b+oKhFtaSzo/ruUNcO52WHhnIierL3rgCRz3dx+iAwuYtFyyu/C4FSlfXLV8hBN867N2rE60Yb+4uNm9qYjxLxah3qrXRa9+R3XaVYVf6dqtmCIMhW7YlgVLNDRbyPIKJ0HCRRF8OHy24V703cxi2zZa/nx13644SBZLFGnMpmamU/nQL7KeuYrOS1m9usiIQRoF+xMmyb6avNIJF2XDeFVhPQ3TVi7a2ZDpLngIsHD8Mu32FAd363Bs58IClwg2z0QsmIpfGKLOT9JKhHPmwPxSp5Q1VRMbI7VwcR/QP+Gs3638l1gL2UY1Vd/MYucNoKsj4a8+VnNIZoZ4ay8nfl1Ft4J3B9y0+CArixiUkeuY7kyUn9cFe7LqxKQlNbWk9fDD9S6tjoF/uWXsPMxxAjGUyBczKj5H3p/D+6l7Z1e+VJheS2xjRuOGDHTuYVKPmwA0bplQRXQwEC1xyftAxsmLAG2NUAvU3+pF152MUuZ06mXOb1c+LlB99Fi7AoaPfE5vgokwLJOkMHT9VZbwc4Ox2HtlbFKJr5Tu5SmMmIPtTUCS/vNn/VJuuB8uhZPNZZOYBhPGWQokDS0K46CU1EMSeMtmc1Ka9A7hSMB8wyv6Jp7+aLdsoI0R5t6SLoyT61X2aQDHTIBJWBIYNcfmjl7NYKgsFoCIySJSktRXwX2t2GKpzS83/JVcH0TytNPo1uUDEy7rUgiRFrdMZfSln/rtxpaZw/tg3EhTwxdhT0n2ZxcpbKVFFWzr0NsJCZAiPVMaqbtVwSFNb9WhJrWMsAQwlzglBUSASWgGtlvXR7uA1fFxoERp1NXdzw2Lxhq4X2wOlGzeNYSo4GEVdkiR/ueWBZ63c7ovNwDKO9IMc/kvQ9pF+ntrfIGsjdVRAk7C0ueegMEYkAw+AVmrXxLU7+g3KxiJuwI8iR2sGJS/2UGgKhoSOSVW6TJ6YEHzH4TKRtYsMqGgtd6oCSpW0kdxjKhTGK4PqNbJpPMThPchrsaQCFpZRcJPKc189yNDWPb0OBg7XV0E/qcCTfhpr/WvS+2ghkB4/JYf7Afhh+slkUxzQQu2x+nE7YK0BvPqoDxLWta+6QQZrxO7/iVMd1tMlJlFaiNhZutgC4RcwLdrc4rUlRgJSUe/hE5dd2SLrh6GMxsh1NXali0NnOd9Sp1v0tW8Cm/S8zCKTvnxpdY0XWXRxJVANsYv2ID7oaqCJMfC7PMKyZN5eV0pfpdaMcaLHAM+1uZB4VlBSzk5gtMnuqP20TwbYNZ+8qpK3OwPnVXrv4CmP745fQCWZgmIt50IdSTZ5e3gtHMYY03ek1z71cmMWtSiOfIBpngBbVs7vzB5XlzzX0X/sYrPUeRFQ4Ud8mC35T0/+1DkC4zpH1jAx2zQyTAakQ/t4zUP6g+QhhN5/JdbaI2oP+fQHTUJQ2LSYfM8eFuDSNIGS+0hvw2BaBYxThOFIwODKmkA3akBGDv1da4MyvWJaOLJPj9xV4ErJJUS2YY8mUixQekVw/j8WEtVIDjMTb0qGpL9iwILVg0yYQvaJMpzRKhyFVudcW/3Bx48qVjikUelnTXmGrRMmCw/B742LlE+a3TuciHD24vo1sP9HLXiIzeFYG6x0cw2tUof+efCEeuCtdRp7OHtsUNNTtlVyy/U1vae1s344pf8Yc83knx3mzXyvEKOrn3RZQB8ZZRzpmXf1coJ933xrSuLk1H10fSOA3hS2i2WUz2E6BbT87yp9t3bRSJ9fqrQu0GID1r8qq/2Ub6x1axewXj/GvbFqCirz36nV+MGHA9WxgSXmX4jn2czfjk392upqxoKlkjXFyP22JjoOSh/2uz77z6R0gp2CsH+G/v6utoBGlHffk1yCY0nSEGtCs9UTucTjIS93xjxqiv1FKF3FTmO03OHBCYr+YCzlCcwuV3wTPmrEpeeHOxCguilQb+EEel0aNDU+QX74n06StK3cT2+wkG0Yz8gJsK1yn37iX/NxltXiFG1vncpGQOiAMyGvKUyjJin6U/+GWBgoUDOWV9hn2vKQNHNKSHD33WrvtSu5HOdDOfw8mxIEpAH0HVSrUi/tfyv7CaBYSp83PQzx2WfOawYmVAZlc25c9qcgTZROX2ix29iILPi1Blva53VC59MIOvENqpJqhpwdpRDyVZWqzTOID0hS/GM6Unk0MH5CpU3nu8PEkccnTKoNhH5suHdDr9rI6LmZlbTU57HqcyfSwgEwkJlZQkXclwHxgiRfzApllT80HmGCxdbkoFl4hertlwRYcZKP9y/TsPiYp7P5Rc2VBK8MWto9zITPE/S3BbKUgS5r9f9LQrBwJUf+d+gOEyBUs37HO6jnHe4VZ3TYOdJUY3wjXz3mJhpchz8TSjFQF3wYow2roxVaa5sV6LLYrZ+EcZjuaASEoyJLNAhweEnx3f8tlXFgxgjeBgiuxuwy+x5aZKnfn74VEhoYWnKYxLC7JlI1J0SvSyZmeDTJQY4R0Vl9dpB5IqHZpoYWb9NdDmbbkWjegN8dZ1oF6c070adEa2VvQQRgEl3b2bO6d7rava2qYN26MyoLjQNVwUxj9ypirw4boceWUXeKxDYGK6uqlwq2bOJvH6YvgIKoFayL2276A1H49noKLMpd4tKh2pDse98LNyk5WBsicNegdtXun47vU+dCsoFbUcFS0FYZROxkQYP/Lx8w8a7X2x0Z5FTRhH77CSBtE2FO8r3JeahF+pqjTUqCuEjI75eEPr5/AfgTJmq9cfVo5byj7K3b9EP+q/av/QGaTt5/LZVoNw2PhnqSjUu/tickN9hH3wXPp0xdiubL40NTZSt6Xnjr6/U00+N5GUH76oP5vpGf9j2WOtGnqV6BCLV9P9nV92tEuVPS2YPAK5D8nvlvj6Qx83RkAqlpmqyOKcEUoa0aOnjARNPP2DI/2M6nGSLye33EfnBGDunnNsLG59cRLfk+BbZoHVaZAbT83G6R30vhBGTx8B+HBNj1ccn3wsbaXZ5P+fCG61mVDSS+vNx0W8Q6qwANz4zmfKL7ML4QVqEBm+Y8dOXIoqfRe5Q344wWNnI1R+81XpYZyfauxVCOp3O3bGe3ptmO31FZQX5DW5Dx+VNnBzHgpm9PAhJIlj6UhKU3cBPc3z6LNdn5V1hUP473zX+TWligW7ZGWR2R/b6in0vM8iiPslJ+Rb6ZLaW3gFbTP95/LZVxuh7uPD0oPjhy6FfQkmvGTC/J39OK7K6CGvZXnZHemXfU7gwx39EvO2HAY7W9p+t4WdCY5YOhnvCWLA5G7NpT3cw657ULyorMvnQChXPmDQOQCwaWcFOaLwuhS8w4QKEH/Cr1mMfgOncXMZ/sx4M6nOBM50Af1rYPX37PN++oqX6ogm3iaDLdFnjq+6V4oMQmg0zQn3VgivvSLHBSmw2aW1lkXzLz672y//pUo3jAgGVdeZGNP40//f6JOQZUPWYL7stsNThKLXAkB+WTuKFzGXwTq1Xa6pINOLHA0mC9itmoytE4+5AoCJcHY5s+AsTThzSf3Lvf9XUcpaFn+XkMl99QOZsSKcFQlsBrgsaKV3WgRzYJIW9blXVdgM09Smx8lwaoplQ3ws98TVuo+mZ79LOz9Mw8dpEWWaDkJR+yPUYAeAi1rN5/LgW9EL0PBQElq3eQRKk2O6M6Q1IVt+9ZGECZieYd6z94vUh7wWAV0Y1ukVPHWLMXWgBXuuHeFsaMF4ZXYH1pnuOBiYKXSgcdpqgp4+tps9arx8SCNdmGt7NRm76sGwj53rA7Ec1xTs0p/YbWQ9pdU1xTkWkQwPAOGzYaHLPZc0cSbhMydd6p2FIcU5GfcGERmc3uuK1KEzpy2LjveWqwGl60ONUj7SBFLypXCDGDsOeIPf/5CvHXzrjY5IbRX/GZL23w6RETRXL13P+0CELll4QwKxNP1uzEZmshCXC92A6F4i5lVagq4pL4K3RG9hiqHhvue5LfGOzw0beJFAskxdqvXxQkAv8ID23Jl8RQIY2blYfHi9Wj8X6u9pfUcYLtF7RyClWIBSmMs8QHKsm3KM370HW+N/twewdhDVAPsffb+klZIlIABNv/LwCyA+Rd/y6v5V7L3ltuFSHn1sAwAbtrJNpPeM50fVYgAXKoTDkx/w10PadOZJBypkC6DC1/gptj7fWfzgg3ihqQUTNv9C1Y9o25CTkiyo7ijlggDVpOCnnSwULCPyiEG75plqGwCo/f6CqTHD4RtkkZpPHpmHSAAZQZ6wHgyXMIA0szqSDbmKWvXrS5z5AMOYJKt/py3ucZsTmvRPuf4i1WZ6bq4+rchsKy/S1QKvsf4AR00oc9iKK6853PUqTmbbtWFcMzZFFibJBO8fwar7XKNw7w7Mj0twh1ksivIOGBlRH5EkmVUDGKMqM6Eerf0tl7TRhCRpDCsYFoybUOonuoV61jglTFKAiVuJiVjzxuaeqJ//PiRuWxoFoTbPrLcd1tpOqFYT+99lGgWmCCAVKZ+V5M7xIGpOyJCkauQ6gUpXNy3b8usQULLB/1YCR1c5SNY+n2nYR7egj90fiBqIIIcgPlbfIji9iUpXmRmhjDTS+Up66L8aVdKmt3D4dwbWIJ3aeqgBYy0pv942aLOZyY/kd9UT4s3RSuGVMDOSX1OoC1zd2YUMTaDX+awpYUvq4N08ICA9R78f7xLjbDk61k4SMONxkUZ+9JCYScUn6oXJq7sqrHKdUjGuFK+xlTNBpPapQpUk6MK0a2nHeKXiHzavdhTDeYHFNlZo0e6VutQjgZTzzIF9LTJuZQu50TcS2v/rJ6d4cdRgfZpXhkYn4hE2jileIqFp1KOPU2BjlRLHuzln3Ba0JCkzIxzsEIlsxs9VHywu26YuXJn5hsxHDtjiQWvkEQnQiGGbCYAeQWp48v0fbKYNn46e27TWa+azn2H6kTfpMqQLcQTPq2xl3d/y7qxWbtm2XZt0Wex5JZk78vOJLSm56+iTW1sDZgSPg4I8PvP3u5+IcOVp8M8oEVq4xT0XaDpONODHimMc3y6vvmVrviE+LuGTfq6rfITzQxLNDFyAUb9U4neXnGMrZnxBJTvZYzHHAoV1+x/gXyX9BO8xbTpnOKCoGOfCTDXy2K8aTDZLAXr+OR+jv/DShcONwrYV0f8u9Ss0EOdukW8ZM6EluUM4+LN14VyL/Og784lLGb7vAvGmt7jHxReIFWCWWySIdzPSJDgDI/9KqKMrldOlvhG9rvvfdd8geBXwED1XvJFWA0OTB13r+kLVpFHJv2KeWOdYCxTBzhC8ttv8ZLJ/6QH+qhc/+ADl5/OgTB0YiLoOPvRnfGhCbbHkfKTRDnWTwlsvrZgQgKL77RIScKjlq0YL1fodOoP1WqzbdslR0rDCob1RflwWLc0ytciGVS/HsMwDenDmnZD2jinJTBSYO/FzO5i1wA1bu6UBuUCQGcTDx1napT/H/9goIVP7UxPUrC9xbJ+tRnvaRa/dqMQ1IIQ8YwMWj2UX8ILC8/2diNcUCAjSTNzh0c9etZhG5gLrVr6l/JrlyN17b4DQvwMENDy6HlBY3J3Jo8BI8v40j+2jbcPQveg+3dwa0tD6UGvG+TFoGvlA8QDi9iyVfQr4zo/KX/jDErfd0vanukCG0uSsAlxOxr2PB/f/vkHo5cDWcg3KdokFw9WhruNPyzx6yRAYxN0/4IaEZU07VwiDjyLrxVqZx/9ofW49okN6yGMyZspfBBO6UlvSbf9eFWVkR/GB7XWsmNllQs+Kw1XoRjr4P3oeIKLfcy1OIfOvVdw0h/loTEFXtFZJ4r8UDcyyQylelYVQCAQxSLJZLpprOBaiFeW7D+Zdtdd2KbDWyD7DvQsuxcNnHs3pdVLNgRPYGiVtnTjSFdOCnlLMh5zbAsVThPBzkQ9cXrNMN6AY2W0glhtprfaHplUOTbJ/SFmaEVYSf3kC1Z6VmX5g4D52gY+6xgY+qQQ9Uj4hDoNelpwElqDmTwmqXaoKAdwwmq3NplA++LE4hwjqRQVTN6TF/j2ppsITs5j7MLQdxftZjiC63rLONVX1UJHbffGzpgDpvq6bKZ2ExV+ykueC/CL0u9knTCoUE9903zQ615CVBp+MVVPJHIQ7OxUgqc2+McOdR99P0gbaObktAQCrWkVEkr/zSuOAFVwdAH+LgwDP3F4a0dQmAh6glnAoHiexe/9hmsgjKgBVixeAUImpXH2wdTyyGy/qnF5hcOEJTt/mF+PLDiGm+gv8UVaaldk9XvHXaf3MsQGbE2ZoXCNUEaiQsqvGyx6HZh25psy8NhnwaU2EzN/mwkLnE7GjrJoB0o+Svsr5ByBsaD5wzz3ogkFxOnhqL6CJ5pcETeQW+vbnG9CmLE8xbrX1SupAcsI5XQyn1Y0v+6yX95gZnChiR/gBhAqmGdB48TINXIWeKeydWb22bLu2Tp2SDvyFor2bj061eVJW5+TDgcV7PyJV74glr0xtHScSHkMJ/5XW1HmKo2IWI9BuGQGdijsRRQJtnqFevANbs3E7rzf4ZqyQGuWoKmfgSltH9KY0GKSK/BDJ3jIW8IzcfqVi0iJ7a5zTqQYIajyXNKib+eVa1BHUoKVEADVvX/+iWwCmB/LlH4AUbKM9FP1j4yyw5fyx3/zax2v5TJ7WmFtHJJ8wcdqjdU+CvxTdPbFBM0Jw37oVJNVwht9HgelkoBbQWwKp/ZZoAhOvIr+ORYvgFQItodx1X2RCkW0P5eCy+b3nVb6D4NILYXeHZYZqt1jJlF0fXkTX69HIZqfKOsPH8yRL8UPtRdXdKQiAz9cscbRz9XN5kav5SorORKm4ysYfN7fuM4CZ8Fs9H1QDeKwPxJWo3HC4st0hARI7Ex76DmseDhepub1o9ultq2kebGz/TBfjsL5csXsTQdCL7VyG+T5+BUcEe8UzltHm/eK7BRLKnmur677sG7OWconLR5/QHItNRS0wXd6VPFdxQQ5NGIKQ3446ZfMwfg6A3flpj/Pz8Q/VKAmsxUg6Jg4C8ozY+W0mUbbMvGCBT78SJK8CFIfu/EjUpJ3N15b1JsADwCyjjPTZ6mm06CrE8bEewtbZquqhUxh4vVYBMQRKeUZqF4pfgc7yIJoTHfhapMd6EDxIGdU83JFIhMEdEQPsDA1sFGYnQQc/jhMHmmZaJ3Ca8LNRPMIBYb4FOwKKkrCFKhTKuN7XGk8G2cmXoz0TMYOzwv0j0rxz0Jixar3xd+6xSw5BKkzICFy3SA70l9oQpjya8ioQbd8T6hQXCayYPABuohwT1JtzzF5qPmsf29nzruA34xSyMqZSZ0rOw9UStU3LYjKeXJZJG2wLfAZZ86l38U8bUlAVw6q7JK8WfXgqtzd6cdz7GlZ78kHhJV87yQS/LKozwfBkBVvyMoORCrwCIqBJyK7FWfHeV7k/YmJdqP+hE2IIDbzI5J+mU9Twc8yTUFm+E9txHOXbyH1IIXqwt90pprIU2TRnSPGnyngvvj1jE6S8HyaJZJAQL+bBoOB03whLUaI5HJUmM49h8tsFC1XkThOJHrVurKDNPT2TAl1YjDoUAjFOoijjf16QE5cEXT569814DlZnxUUmI/xMimhMFOU7oOO7wG3geWEQqELIdPgMd0PEBHi+hOldqK62XdAth+mTi1oGreJHJm/9/qvtXxaiD/FOpi942aL1zJXeJxXHMfNjA9Sxxt9xYrvIx3R5mu03JYpgc3sY2YzZ3UVpuQMdK+fg6gzGkTOVUpEs/BbicYMc5rNL/Kv+PH2JgpvlihwVpYTcgoP9oXXUhewFGBf3txEDJsrz2iRMtvWiOQkvzsXGbTiBfPZMTimm+kGIOp5a/bzM51394M+g2IK6bYYYFxxvE+oL4wsy+IJFve4R/rwJ69mJ2WxMXMqC22wxQw2AhV/s3VxCydTCDXMl5xLvhyjVcA64kJrgZ+UDhiTWM7A9CkCNXQd6XA63SmM9z+HdsA2a/QsLs8x9ovtdm0m9jkO4KnLwy6AZWnGEafvx6ROlhNlUDRCGXf69K1Iua2UG/KHemQ9Ba/FO8J7HV0MA5RGj+M2OoMwyYOR9gJkc87Iweue8pk/74lRFs2g7QhlvBnYtQLZzjFxQDexoPG0cCOxHAhg6piexP1XTR593x/pWphRaaN3DCJy1M1MxprTM0mT9AR1WxkDGPkBEQEycyA2oUykTLyqB24+JUzHQqhNrhtwYf0sJwrh1KH8cGEsvQlZgqknJT6QL8jOLSsjdcPqS7NwPil/cmNv9zimdKhGQBOewF6JAOJmfhdqiOl+Gs9yZsYiXJgMXkZlGZBKHOYpFW1YkFkY3ZDVnoRKhIaCF+cvUcE4vvYPscf6UrV6SnZnLbrpFAl7EX+do1/FtjLldAMqrGYPClYm3+QcZ1lOCP87CjQ3uGPSa1FfdWOxJfmzAW+OThvGFQAscuL2l5lTCNwSZJVoS5Xdb6SpDbqZ+9x/C5mA5q+N7QTA+bYGXX0wqEWZMPlCoJz4uwCpNbWvcUBZyeTJIrxHW9QEE24ZEvXcukslM9hS2PfmfRS/Vi0Jn3yMmuJ/+nASD2IT/JZmKcUjk43Q8psOF1z8ukV63S1a33JKJYBt6dBRwkh+RdgwCblow22WOdal7IhNrQR1FA1pJ2YTqOEKW+9xIuT49wHl8XBVBnyQrMkkjd5MhHuk1aoiJD5XZkD3BLGOYIrFWRqWMD7rqytussvnT8WzkF5Ft0+jo+8Jq9JLPats/dfOnm0hO0vGZYmN7Jc7G8zV+RvvFNz7CXiAsYWI+BWknP5EXbxk4iMM2FSbfDYS7vSWUaU6l+NQWadO25P+LvY1ptT3eG9ci6ZcqR9a0KvsKkp89sgy+BoI6LOgMVjfFs1Mt5D66/f7e/chSWVR96eZeBhRchmtm980f3bxd6EeRvxIa3KRDS2iZE3jsiHsqFr0twjTKr3KzOPA+psXsgY94xmHA+vbPytV7nw9tYgz+EwLUXwx3ynZCZ6UtHanILUBNPQ8p1tF6j38BnhoVvcNCX333BSFu87u+YfB2R5VklsvqZy7FASkYwK5QUrfnqyO/8W2RlxiobTfOKwGGGsbOFfbzgmMqdF7oDWfh/jEZh6QILTaO33dBKqFGLWpXRoEy9XZWEKSuZz1JrZaK8p0MOI2GbiS/5meC2IeGhWGCaN1BnVPNfN6J1ZgZfCORoiZKCD3L1jlckDNv8S/ELSsR8aSu0pjhkGhMF79weUvRNJaDiHsEpfSZd1lxnAYmdgf1gCzFG9xn1lGGihN2ehOr9DJQkd/lGspLYHkRUCnLqWvAR5uf6l9idrMPVnIELIU4S1cTOM7tDjeF4IopymF4FWOmp1JL4k9dou2R9jpDkMoqJG0wUChvxM6fpOKoyOgOfmvfl9NZLMv7lWToWSfw+QAj+y/QwffT//lowqAYFjRe445lwZD00SppGt88dGQBS7ki++fUxcPxbW6bQqiozyqE7ZTemOH5+v4dW2cMOsPBKUbzaktUlSLNPvN9gVAvL//jrvmekiXtpxhJNV36+0B+Ni8XOGRTsdPqotU3WJvh/9qGkl3jb+E7spbWaUTrsQXxsupq+Z6lQhoQFoO7sqiM27QU22gHUv2K8YpuwZ2wLWyZKT/bbAAtZdPi1Lx1ZDkHGFMJ6A/EB0BuoscFuqH9Ht7KZAsqyOe7nYeYgA7ifhi65IPWMq/E/DUI+YXxPkOFcqyNKBSFyLJ+G2EqSU6EMn1Pmf+p/ZVB55VT0UPKU4nzIQIhPUS1+k7rrIq3YWq4ilL2JVhoyToCNNWtSNW8dDFK19imHmbKNwM3hErKM/FtfV1dFg2n5+wSpKgz3qSxDYBRapHizjEsWsmsQYgGnRevHypWigDCGnHggiIFtNPaDWE4jy9x0NclgPDdr0I/pZ13OR/NRotCHXOoivCZAnQoIzvtYjDN604ERbReXbaPgz6kbnFexHBpjDc7qwzu1wCccAdyNhbDljWMeWuYR9ur9I/lYUFSoBpHNR3HDghBsye6WC4pjd7LE19p1+m3D5XHxCVNAk43gUiRH0CRTtmj3Ugx9HKABNtc92vPkj9irOsHtVBg1lFiFBRO3Tl+Os3oM31xkJM8j1VeRVNvMp4ZM74h+N/sn44GDLYl11zzy3GD/eaqfa/xbZL3f/2lZID5LRBgzYpZAzbz0w99S4+BacQVuBgMVa4nIgrKjaEkmVINsPs/18tFnydGmihSohxdnH+gTwystso1fQ9ucHpgPzd9TfufX7DeJIqAKg6D6mfZ2Io/cX+IaeWqhmInEkCeW6XdurA0wpRfsz8b3olrYhA+7yFy6izF7n2yc54ylXS/WzKEEg8dZbolCZeCWLbkRJzWbdVKLYiBQzFSwir2F7glkJ8dY576NJX4PeRp1B1XwyqGan/FL9UHw1wSdlgrEem2BtUP7K01qUg5IDqBURJJExwEwB/jvpI7SWDQDhuwtVKyrOd5jGUfGEOQH3xxiu3QPCv0B4MQhKSRlsBLYONO3E8Fqh1z2aqV3ErnsUNQ0ZIHtRuLvZCF/uG/S/MfmS0fssF+sveWbt44AIUPq/cA5hELc7+ODUAn6LzHIBJqq+bd0G+vB0Z0P95xPTlKPa1p4XP5kJob62nqH9W6sXwzoBIoJd0dWPrD+64BftH5Vq69OdfDQFO3xkl+rQsWF0DGuAXJLSI+ffvc+ikOjQ8ozKk09YbHoyFw+b7i3bc/WBjPGd+ljcuKHcXWO5iGJ1o4tUD0QdsNoowresHQ1kHpmifZ+qGR4b1X7jIPewPSQeXCbfC58+cpXg52/nsB60CVTGjHb1d5c8CnU54OvxCJqp25r0va12ZwS0ZRXjyA8wvU5DkhBfIlyM1jbDDdxVj+1+qdd3kzr/OzjLLhHBDeuVzAeE0rQIVNqylzgeTf/8GzwlWJbJoNWfnH8cYINoEhmvFDkh5rtAY5Ox0uNM2ZNOqJzPRcSqk3PQ6KSsOxsDY0XsTBlpRHsdC3jSidZS97JMj5lil1sZwLIRhL9DW6oZT6eo5cAem3iwXBRqWzH1DzIwWPdO+ADaIM+kLrkevXrrD9TK83ZdwVAluwNsG5MzS4AWnKt1ltsPlzQ/FSaiix06oLe3mR78nTWzPM+HKxQUil4TpUfpIiZthsOAzmCT/VWAJqFMNI20Gz4JlcQ1eejkLKYqvcxVjLdGT4R6Jmcu6q3vLl9nZHbZHa+8qLdNr85q+xHMWbd0Ao2PolVOnmkoSmo2y17LVj8IAHu++S0HQnn5ZrEE86qzBJV5z8fKMGJrEc+togq0qgH6f4rgT/iaI5dv1FDdY1BdrD5p0KSmi4P+PkkKODu0u+nNb50398meYbR8QRzqFYVWK1lrLDH80PsLpJelc5nKJ//z+jh4aovjDVJ33TyyE7l5+uXTjARtYBZF7lTsrZUpS0WJ6gGKHFTAzi2VeIoE9AdbzF+VSRoseU4DjCwIEDFKKPTVI4dY73UsoBteLltw8HhhUp9CAK6v/Z/eGedNndL77XoJQBUDWGjhThfxcVi6PFs2Khvoh3unAwG6IkL+QbXuqwf2nQ3ESqxESpVHk9nZ1MABLi8AAAAAAAAlgMrwQMAAADrNasiA///XfxyGLgI0Cc+Mkp34lwG9El/fUsB+dmcY5a2V2SX8GU5HtHdESL9L9eSnCXPpmVG7wiZnnhzNdSTtXUDEltf2iGQgOkPdkfb/3cd4FcRDcZaW9v29mIJYFty5M5Wrxc/9pCCEDfL/wb9d1b2iqt+cF8kTQoUoFlukjmF5gJgZnXzgqdNK7Pl4LRuZ4ZLJHoZeNWWZLDx7JHIHEjRR9yu9RIcDffNjKfFjYsVkVMO79LTj8LPuOwEhgezKWJ65LqoO5rZC8NtZ7/z/PYRVupwKc6ykuiJVtNUpNQm27kD5Fy3Lv6uxHl9AYYmJTrLl0eqjh1lAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADmeNYsgbKZb07z0fV3ON1uaSa0YLSgSpAOT8izVkw3PfVq1V9EtccRQ+V95nGLeJgadjPhGDw5ncGRWJn/EYkvoYPD2bRg/DEQ4eCrRuMF9/SwUPwvKe0UXMLxMRhYU1Hv6raKfDGk/C4Ys/5VXGz6IMXKoRivdwKVGsaby4b8vYvkTLiivEUimHC6GcRqh/ZRArTTkRidu5J5Ikd/dTwmzAYnT4S7SSMgL2ZcuxD0Rr3CxTiaQIC0W+lRKmfgqQALOr/DzWlcBvJvt4Vl1tISvqbtllE6AkVBfJPJ+IDjJyQVYcVaomgD5SE9KAzKXaIOyHIKFWZB4XIXR9Lrjhrdg3MZJ63g==" -_STT_AUDIO_MIME: str = "audio/ogg" - - -Modality = Literal["text", "tts", "stt"] - - -@dataclass(frozen=True) -class Probe: - provider: str - model: str - modality: Modality - - -_PROBES: list[Probe] = [ - # Text - Probe(provider="openai", model="gpt-4o-mini", modality="text"), - Probe(provider="google", model="gemini-2.5-flash", modality="text"), - Probe(provider="google-aistudio", model="gemini-2.5-flash", modality="text"), - Probe(provider="anthropic", model="claude-3-5-haiku-latest", modality="text"), - # TTS - Probe(provider="google", model="gemini-2.5-flash-preview-tts", modality="tts"), - Probe(provider="sarvamai", model="bulbul:v3", modality="tts"), - Probe(provider="elevenlabs", model="eleven_flash_v2_5", modality="tts"), - # STT - Probe(provider="google", model="gemini-2.5-pro", modality="stt"), - Probe(provider="sarvamai", model="saarika:v2.5", modality="stt"), - Probe(provider="elevenlabs", model="scribe_v1", modality="stt"), -] - - -def _build_provider( - session: Session, probe: Probe, *, org_id: int, project_id: int -) -> BaseProvider | None: - try: - return get_llm_provider( - session=session, - provider_type=probe.provider, - project_id=project_id, - organization_id=org_id, - ) - except (ValueError, RuntimeError) as e: - logger.warning( - f"[_build_provider] Client init failed | provider: {probe.provider}, " - f"modality: {probe.modality}, error: {e}" - ) - return None - - -def _build_config_and_input( - probe: Probe, -) -> tuple[KaapiCompletionConfig, str | AudioRef] | None: - if probe.modality == "text": - cfg = KaapiCompletionConfig.model_validate( - { - "provider": probe.provider, - "type": "text", - "params": { - "model": probe.model, - "temperature": 0.0, - "max_output_tokens": _PROBE_MAX_TOKENS, - }, - } - ) - return cfg, _PROBE_INPUT - - if probe.modality == "tts": - cfg = KaapiCompletionConfig.model_validate( - { - "provider": probe.provider, - "type": "tts", - "params": {"model": probe.model}, - } - ) - return cfg, _PROBE_INPUT - - # stt - if not _STT_AUDIO_B64: - return None - cfg = KaapiCompletionConfig.model_validate( - { - "provider": probe.provider, - "type": "stt", - "params": {"model": probe.model}, - } - ) - audio = AudioRef( - bytes_=base64.b64decode(_STT_AUDIO_B64), - mime_type=_STT_AUDIO_MIME, - ) - return cfg, audio - - -def _run_probe(provider: BaseProvider | None, probe: Probe) -> dict[str, Any]: - result: dict[str, Any] = { - "endpoint": "llm/call", - "provider": probe.provider, - "modality": probe.modality, - "model": probe.model, - "ok": False, - "latency_ms": None, - "error": None, - } - if provider is None: - result["error"] = "client_init_failed" - return result - - built = _build_config_and_input(probe) - if built is None: - result["error"] = "stt_audio_not_configured" - return result - config, resolved_input = built - query = QueryParams(input=_PROBE_INPUT if isinstance(resolved_input, str) else "") - - started = time.perf_counter() - try: - response, error = provider.execute( - completion_config=config, # type: ignore[arg-type] - query=query, - resolved_input=resolved_input, - ) - except Exception as e: - result["latency_ms"] = int((time.perf_counter() - started) * 1000) - result["error"] = f"{type(e).__name__}: {e}" - logger.warning( - f"[_run_probe] Raised | provider: {probe.provider}, " - f"modality: {probe.modality}, error: {result['error']}" - ) - return result - - result["latency_ms"] = int((time.perf_counter() - started) * 1000) - if response is None: - result["error"] = error or "no_response" - logger.warning( - f"[_run_probe] Failed | provider: {probe.provider}, " - f"modality: {probe.modality}, error: {result['error']}" - ) - return result - - result["ok"] = True - return result - - -def run_probes(*, session: Session) -> dict[str, Any]: - org_id = settings.HEALTH_PROBE_ORG_ID - project_id = settings.HEALTH_PROBE_PROJECT_ID - if org_id is None or project_id is None: - logger.warning( - "[run_probes] Health probe org/project not configured — skipping" - ) - return {"skipped": True, "reason": "health_probe_org_or_project_not_set"} - - logger.info( - f"[run_probes] Starting | probes: {len(_PROBES)}, " - f"org_id: {org_id}, project_id: {project_id}" - ) - - provider_cache: dict[str, BaseProvider | None] = {} - for p in _PROBES: - if p.provider not in provider_cache: - provider_cache[p.provider] = _build_provider( - session, p, org_id=org_id, project_id=project_id - ) - dispatch = [(p, provider_cache[p.provider]) for p in _PROBES] - - started = time.perf_counter() - with ThreadPoolExecutor(max_workers=_PROBE_WORKERS) as pool: - results = list(pool.map(lambda pp: _run_probe(pp[1], pp[0]), dispatch)) - elapsed_ms = int((time.perf_counter() - started) * 1000) - - ok_count = sum(1 for r in results if r["ok"]) - logger.info( - f"[run_probes] Completed | total: {len(results)}, ok: {ok_count}, " - f"failed: {len(results) - ok_count}, elapsed_ms: {elapsed_ms}" - ) - return { - "elapsed_ms": elapsed_ms, - "total": len(results), - "ok": ok_count, - "failed": len(results) - ok_count, - "results": results, - } diff --git a/backend/app/services/stats.py b/backend/app/services/stats.py index 2f0e65355..fcf5297be 100644 --- a/backend/app/services/stats.py +++ b/backend/app/services/stats.py @@ -2,28 +2,28 @@ from datetime import timedelta from typing import Any +import requests from sqlmodel import Session +from app.core.config import settings from app.core.util import now from app.crud.stats import get_daily_stats logger = logging.getLogger(__name__) -DAILY_WINDOW = timedelta(hours=24) +DAILY_WINDOW = timedelta(hours=168) +_MAX_ROWS_PER_SECTION = 20 +_DISCORD_CHUNK_LIMIT = 1900 # Discord content cap is 2000; leave headroom -def collect_daily_stats(*, session: Session) -> dict[str, Any]: +def collect_daily_stats( + *, session: Session, window_hours: int | None = None +) -> dict[str, Any]: end_at = now() - start_at = end_at - DAILY_WINDOW - logger.info( - f"[collect_daily_stats] Starting | start_at: {start_at.isoformat()}, " - f"end_at: {end_at.isoformat()}" + start_at = end_at - ( + timedelta(hours=window_hours) if window_hours else DAILY_WINDOW ) stats = get_daily_stats(session=session, start_at=start_at, end_at=end_at) - logger.info( - f"[collect_daily_stats] Completed | sections: {len(stats)}, " - f"llm_calls: {len(stats['llm_call_token_summary'])}" - ) return { "window": { "start_at": start_at.isoformat(), @@ -31,3 +31,117 @@ def collect_daily_stats(*, session: Session) -> dict[str, Any]: }, "stats": stats, } + + +def section_counts(result: dict[str, Any]) -> dict[str, int]: + return { + section: len(rows) + for section, rows in result["stats"].items() + if isinstance(rows, list) + } + + +_COL_ALIASES = { + "organization_name": "org", + "sum_total_tokens": "tokens", + "call_count": "calls", + "job_count": "jobs", + "row_count": "count", +} + +_SECTION_TITLES = { + "llm_call_counts": "LLM Calls", + "llm_call_token_summary": "LLM Tokens", + "llm_call_modality_counts": "LLM Modality", + "job_type_counts": "Jobs by Type", + "evaluation_run_counts": "Evaluation Runs", + "stt_result_counts": "STT Results", + "tts_result_counts": "TTS Results", + "assessment_counts": "Assessments", +} + + +def _short_ts(iso: str) -> str: + return iso.split(".", 1)[0].replace("T", " ") + + +def _fmt_num(v: Any) -> str: + return f"{v:,}" if isinstance(v, int) else str(v) + + +def _render_section(section: str, rows: list[dict[str, Any]]) -> str | None: + title = _SECTION_TITLES.get(section, section) + if not rows: + return None + cols = list(rows[0].keys()) + sample = rows[:_MAX_ROWS_PER_SECTION] + labels = {c: _COL_ALIASES.get(c, c) for c in cols} + is_num = {c: all(isinstance(r.get(c), (int, float)) for r in sample) for c in cols} + widths = { + c: max(len(labels[c]), *(len(_fmt_num(r.get(c, ""))) for r in sample)) + for c in cols + } + + def fmt(val: Any, c: str) -> str: + s = _fmt_num(val) + return f"{s:>{widths[c]}}" if is_num[c] else f"{s:<{widths[c]}}" + + header = " ".join(fmt(labels[c], c) for c in cols) + body = "\n".join(" ".join(fmt(r.get(c, ""), c) for c in cols) for r in sample) + truncated = ( + f"\n… +{len(rows) - _MAX_ROWS_PER_SECTION} more" + if len(rows) > _MAX_ROWS_PER_SECTION + else "" + ) + return f"\n{title}\n```\n{header}\n{body}{truncated}\n```" + + +def format_daily_stats_message(result: dict[str, Any]) -> str: + window = result["window"] + start = _short_ts(window["start_at"]) + end = _short_ts(window["end_at"]) + blocks = [f"Daily Stats · {start} → {end} UTC"] + + quiet: list[str] = [] + for section, rows in result["stats"].items(): + if not isinstance(rows, list): + continue + rendered = _render_section(section, rows) + if rendered is None: + quiet.append(_SECTION_TITLES.get(section, section)) + else: + blocks.append(rendered) + + if quiet: + blocks.append("\nNo data: " + ", ".join(quiet)) + return "\n".join(blocks) + + +def _chunk_message(message: str, limit: int) -> list[str]: + chunks: list[str] = [] + buf = "" + for block in message.split("\n\n"): + if len(buf) + len(block) + 2 > limit and buf: + chunks.append(buf) + buf = block + else: + buf = f"{buf}\n\n{block}" if buf else block + if buf: + chunks.append(buf) + return chunks + + +def post_daily_stats_to_discord(message: str) -> None: + """Fire-and-forget Discord post. No-op if webhook not configured. + Logs and stops on the first failed chunk — never raises.""" + url = settings.DISCORD_STATS_WEBHOOK_URL + if not url: + return + for chunk in _chunk_message(message, _DISCORD_CHUNK_LIMIT): + try: + requests.post( + str(url), json={"content": chunk}, timeout=5 + ).raise_for_status() + except requests.RequestException as e: + logger.warning(f"[post_daily_stats_to_discord] Webhook post failed: {e}") + return diff --git a/backend/app/tests/services/test_stats.py b/backend/app/tests/services/test_stats.py new file mode 100644 index 000000000..c1f235aa1 --- /dev/null +++ b/backend/app/tests/services/test_stats.py @@ -0,0 +1,190 @@ +from unittest.mock import MagicMock, patch + +import requests + +from app.services import stats as stats_mod +from app.services.stats import ( + _chunk_message, + collect_daily_stats, + format_daily_stats_message, + post_daily_stats_to_discord, + section_counts, +) + + +def _sample_result() -> dict: + return { + "window": { + "start_at": "2026-07-03T09:33:52.141126", + "end_at": "2026-07-10T09:33:52.141126", + }, + "stats": { + "llm_call_counts": [ + {"organization_name": "Acme", "call_count": 15}, + ], + "llm_call_token_summary": [ + { + "organization_name": "Acme", + "model": "gpt-4", + "sum_total_tokens": 3800, + }, + { + "organization_name": "Beta", + "model": "claude-4", + "sum_total_tokens": 241, + }, + ], + "stt_result_counts": [], + "tts_result_counts": [], + }, + } + + +def test_section_counts_returns_row_len_per_list_section(): + counts = section_counts(_sample_result()) + assert counts == { + "llm_call_counts": 1, + "llm_call_token_summary": 2, + "stt_result_counts": 0, + "tts_result_counts": 0, + } + + +def test_section_counts_skips_non_list_values(): + result = {"stats": {"a": [{"x": 1}], "b": "not-a-list", "c": None}} + assert section_counts(result) == {"a": 1} + + +def test_format_message_header_shows_trimmed_window(): + msg = format_daily_stats_message(_sample_result()) + first_line = msg.splitlines()[0] + assert first_line.startswith("Daily Stats") + assert "2026-07-03 09:33:52" in first_line + assert "2026-07-10 09:33:52" in first_line + assert ".141126" not in first_line # microseconds trimmed + + +def test_format_message_uses_friendly_section_titles_and_col_aliases(): + msg = format_daily_stats_message(_sample_result()) + assert "LLM Calls" in msg + assert "LLM Tokens" in msg + # column aliases applied + assert "org" in msg + assert "calls" in msg + assert "tokens" in msg + # raw column names should not appear + assert "organization_name" not in msg + assert "sum_total_tokens" not in msg + + +def test_format_message_collapses_empty_sections_into_one_line(): + msg = format_daily_stats_message(_sample_result()) + quiet_lines = [ln for ln in msg.splitlines() if ln.startswith("No data:")] + assert len(quiet_lines) == 1 + assert "STT Results" in quiet_lines[0] + assert "TTS Results" in quiet_lines[0] + + +def test_format_message_formats_numbers_with_commas(): + msg = format_daily_stats_message(_sample_result()) + assert "3,800" in msg + assert "3800" not in msg + + +def test_format_message_truncates_long_sections(): + big = { + "window": {"start_at": "2026-01-01T00:00:00", "end_at": "2026-01-08T00:00:00"}, + "stats": { + "evaluation_run_counts": [ + {"organization_name": f"org-{i}", "row_count": i} for i in range(50) + ] + }, + } + msg = format_daily_stats_message(big) + assert "… +30 more" in msg # 50 rows - 20 MAX = 30 + + +def test_chunk_message_splits_on_section_boundaries_when_over_limit(): + # Each block ~40 chars — a few fit per 100-char chunk, forcing multiple chunks. + blocks = [f"block-{i}-{'x' * 30}" for i in range(6)] + message = "\n\n".join(blocks) + chunks = _chunk_message(message, limit=100) + assert len(chunks) > 1 + # Every original block appears somewhere in the chunks. + joined = "\n\n".join(chunks) + for b in blocks: + assert b in joined + + +def test_chunk_message_single_chunk_when_under_limit(): + assert _chunk_message("short message", limit=1000) == ["short message"] + + +def test_post_to_discord_noop_when_webhook_unset(): + with patch.object(stats_mod.settings, "DISCORD_STATS_WEBHOOK_URL", None), patch( + "app.services.stats.requests.post" + ) as mock_post: + post_daily_stats_to_discord("anything") + mock_post.assert_not_called() + + +def test_post_to_discord_posts_each_chunk(): + posted: list[str] = [] + + def fake_post(url, json, timeout): + posted.append(json["content"]) + resp = MagicMock() + resp.raise_for_status = lambda: None + return resp + + long_msg = "hdr\n\n" + "\n\n".join(f"block-{i}" * 100 for i in range(4)) + with patch.object( + stats_mod.settings, "DISCORD_STATS_WEBHOOK_URL", "https://x/hook" + ), patch("app.services.stats.requests.post", side_effect=fake_post): + post_daily_stats_to_discord(long_msg) + assert len(posted) >= 2 # message chunked + + +def test_post_to_discord_swallows_request_exception(): + with patch.object( + stats_mod.settings, "DISCORD_STATS_WEBHOOK_URL", "https://x/hook" + ), patch( + "app.services.stats.requests.post", side_effect=requests.ConnectionError("boom") + ): + # must not raise + post_daily_stats_to_discord("hello") + + +def test_post_to_discord_stops_after_first_failed_chunk(): + calls = {"n": 0} + + def flaky_post(url, json, timeout): + calls["n"] += 1 + raise requests.ConnectionError("nope") + + long_msg = "\n\n".join(f"block-{i}" * 200 for i in range(4)) + with patch.object( + stats_mod.settings, "DISCORD_STATS_WEBHOOK_URL", "https://x/hook" + ), patch("app.services.stats.requests.post", side_effect=flaky_post): + post_daily_stats_to_discord(long_msg) + assert calls["n"] == 1 # bailed after first failure, didn't retry rest + + +def test_collect_daily_stats_uses_default_window_when_hours_none(): + with patch( + "app.services.stats.get_daily_stats", return_value={"x": []} + ) as mock_get: + result = collect_daily_stats(session=MagicMock()) + call_kwargs = mock_get.call_args.kwargs + delta = call_kwargs["end_at"] - call_kwargs["start_at"] + assert delta == stats_mod.DAILY_WINDOW + assert result["stats"] == {"x": []} + assert "start_at" in result["window"] and "end_at" in result["window"] + + +def test_collect_daily_stats_honours_window_hours_override(): + with patch("app.services.stats.get_daily_stats", return_value={}) as mock_get: + collect_daily_stats(session=MagicMock(), window_hours=3) + call_kwargs = mock_get.call_args.kwargs + delta = call_kwargs["end_at"] - call_kwargs["start_at"] + assert delta.total_seconds() == 3 * 3600 From bcc57ea08aa87f015b178a333e4a8ff088f9022c Mon Sep 17 00:00:00 2001 From: Prajna1999 Date: Fri, 10 Jul 2026 15:28:43 +0530 Subject: [PATCH 05/16] add cron for daily-stats --- scripts/python/invoke-cron.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/python/invoke-cron.py b/scripts/python/invoke-cron.py index 64df37b25..e589c65aa 100644 --- a/scripts/python/invoke-cron.py +++ b/scripts/python/invoke-cron.py @@ -18,6 +18,7 @@ ENDPOINTS = [ "/api/v1/cron/evaluations", "/api/v1/cron/pending-jobs", + "/api/v1/cron/daily-stats", ] REQUEST_TIMEOUT = 30 # Timeout for requests in seconds From ca355348abc329a08603fe703871f57bcb645760 Mon Sep 17 00:00:00 2001 From: Prajna1999 Date: Wed, 29 Jul 2026 11:22:52 +0530 Subject: [PATCH 06/16] fix comments --- backend/app/api/routes/cron.py | 5 ++++- backend/app/crud/stats.py | 8 +++++--- backend/app/services/stats.py | 12 ++---------- backend/app/tests/services/test_stats.py | 18 +----------------- 4 files changed, 12 insertions(+), 31 deletions(-) diff --git a/backend/app/api/routes/cron.py b/backend/app/api/routes/cron.py index fc888e908..9ea3483c3 100644 --- a/backend/app/api/routes/cron.py +++ b/backend/app/api/routes/cron.py @@ -1,4 +1,5 @@ import logging +from typing import Any import sentry_sdk from fastapi import APIRouter, Depends @@ -147,7 +148,9 @@ async def evaluation_cron_job( monitor_slug="daily-stats-cron-job", monitor_config=DAILY_STATS_CRON_MONITOR_CONFIG, ) -def daily_stats_cron_job(session: SessionDep, hours: int | None = None) -> dict: +def daily_stats_cron_job( + session: SessionDep, hours: int | None = None +) -> dict[str, Any]: logger.info(f"[daily_stats_cron_job] Cron job invoked | hours={hours}") try: result = collect_daily_stats(session=session, window_hours=hours) diff --git a/backend/app/crud/stats.py b/backend/app/crud/stats.py index 1f35bbca0..3ad21cf61 100644 --- a/backend/app/crud/stats.py +++ b/backend/app/crud/stats.py @@ -2,7 +2,7 @@ from datetime import datetime from typing import Any -from sqlalchemy import text +from sqlalchemy import TextClause, text from sqlmodel import Session logger = logging.getLogger(__name__) @@ -73,7 +73,7 @@ ) -def _org_count_sql(table: str) -> Any: +def _org_count_sql(table: str) -> TextClause: return text( f""" SELECT @@ -94,7 +94,9 @@ def _org_count_sql(table: str) -> Any: _ASSESSMENT_COUNTS_SQL = _org_count_sql("assessment") -def _rows(session: Session, stmt: Any, params: dict[str, Any]) -> list[dict[str, Any]]: +def _rows( + session: Session, stmt: TextClause, params: dict[str, Any] +) -> list[dict[str, Any]]: return [dict(row) for row in session.execute(stmt, params).mappings().all()] diff --git a/backend/app/services/stats.py b/backend/app/services/stats.py index fcf5297be..070b91dea 100644 --- a/backend/app/services/stats.py +++ b/backend/app/services/stats.py @@ -11,7 +11,7 @@ logger = logging.getLogger(__name__) -DAILY_WINDOW = timedelta(hours=168) +DEFAULT_STATS_WINDOW = timedelta(hours=168) # 7-day rolling window by default _MAX_ROWS_PER_SECTION = 20 _DISCORD_CHUNK_LIMIT = 1900 # Discord content cap is 2000; leave headroom @@ -21,7 +21,7 @@ def collect_daily_stats( ) -> dict[str, Any]: end_at = now() start_at = end_at - ( - timedelta(hours=window_hours) if window_hours else DAILY_WINDOW + timedelta(hours=window_hours) if window_hours else DEFAULT_STATS_WINDOW ) stats = get_daily_stats(session=session, start_at=start_at, end_at=end_at) return { @@ -33,14 +33,6 @@ def collect_daily_stats( } -def section_counts(result: dict[str, Any]) -> dict[str, int]: - return { - section: len(rows) - for section, rows in result["stats"].items() - if isinstance(rows, list) - } - - _COL_ALIASES = { "organization_name": "org", "sum_total_tokens": "tokens", diff --git a/backend/app/tests/services/test_stats.py b/backend/app/tests/services/test_stats.py index c1f235aa1..e74f64a22 100644 --- a/backend/app/tests/services/test_stats.py +++ b/backend/app/tests/services/test_stats.py @@ -8,7 +8,6 @@ collect_daily_stats, format_daily_stats_message, post_daily_stats_to_discord, - section_counts, ) @@ -40,21 +39,6 @@ def _sample_result() -> dict: } -def test_section_counts_returns_row_len_per_list_section(): - counts = section_counts(_sample_result()) - assert counts == { - "llm_call_counts": 1, - "llm_call_token_summary": 2, - "stt_result_counts": 0, - "tts_result_counts": 0, - } - - -def test_section_counts_skips_non_list_values(): - result = {"stats": {"a": [{"x": 1}], "b": "not-a-list", "c": None}} - assert section_counts(result) == {"a": 1} - - def test_format_message_header_shows_trimmed_window(): msg = format_daily_stats_message(_sample_result()) first_line = msg.splitlines()[0] @@ -177,7 +161,7 @@ def test_collect_daily_stats_uses_default_window_when_hours_none(): result = collect_daily_stats(session=MagicMock()) call_kwargs = mock_get.call_args.kwargs delta = call_kwargs["end_at"] - call_kwargs["start_at"] - assert delta == stats_mod.DAILY_WINDOW + assert delta == stats_mod.DEFAULT_STATS_WINDOW assert result["stats"] == {"x": []} assert "start_at" in result["window"] and "end_at" in result["window"] From ac11c4648a0cbf84a008bd5905b3e42d955c0394 Mon Sep 17 00:00:00 2001 From: Prajna1999 Date: Sun, 2 Aug 2026 20:50:27 +0530 Subject: [PATCH 07/16] simplify SQL queries and add last 24 hours stats --- backend/app/api/routes/cron.py | 24 +-- backend/app/crud/stats.py | 192 ++++++++++++---------- backend/app/services/stats.py | 194 ++++++++--------------- backend/app/tests/services/test_stats.py | 169 ++++---------------- 4 files changed, 217 insertions(+), 362 deletions(-) diff --git a/backend/app/api/routes/cron.py b/backend/app/api/routes/cron.py index 9ea3483c3..c285d0fe2 100644 --- a/backend/app/api/routes/cron.py +++ b/backend/app/api/routes/cron.py @@ -10,11 +10,8 @@ from app.core.config import settings from app.crud.evaluations import process_all_pending_evaluations from app.services.job_monitoring import monitor_pending_jobs -from app.services.stats import ( - collect_daily_stats, - format_daily_stats_message, - post_daily_stats_to_discord, -) +from app.crud.stats import get_daily_stats +from app.services.stats import format_sections, post_to_discord logger = logging.getLogger(__name__) @@ -40,7 +37,7 @@ } DAILY_STATS_CRON_MONITOR_CONFIG: MonitorConfig = { - "schedule": {"type": "crontab", "value": "0 0 * * *"}, + "schedule": {"type": "crontab", "value": "0 9 * * *"}, "timezone": "UTC", "checkin_margin": 5, "max_runtime": 10, @@ -148,18 +145,11 @@ async def evaluation_cron_job( monitor_slug="daily-stats-cron-job", monitor_config=DAILY_STATS_CRON_MONITOR_CONFIG, ) -def daily_stats_cron_job( - session: SessionDep, hours: int | None = None -) -> dict[str, Any]: - logger.info(f"[daily_stats_cron_job] Cron job invoked | hours={hours}") +def daily_stats_cron_job(session: SessionDep) -> dict[str, Any]: try: - result = collect_daily_stats(session=session, window_hours=hours) - post_daily_stats_to_discord(format_daily_stats_message(result)) - logger.info( - "[daily_stats_cron_job] Completed | window: %s", - result["window"], - ) - return result + stats = get_daily_stats(session=session) + post_to_discord(format_sections(stats)) + return stats except Exception as e: logger.error( f"[daily_stats_cron_job] Error executing cron job: {e}", diff --git a/backend/app/crud/stats.py b/backend/app/crud/stats.py index 3ad21cf61..dbae337a6 100644 --- a/backend/app/crud/stats.py +++ b/backend/app/crud/stats.py @@ -1,120 +1,150 @@ -import logging -from datetime import datetime from typing import Any -from sqlalchemy import TextClause, text +from sqlalchemy import text from sqlmodel import Session -logger = logging.getLogger(__name__) +# Every query reports two rolling windows in one pass: the last 24 hours and the +# last 7 days (168 hours), broken down per organization and per project. The +# 24h FILTER count is a subset of the rows the 7d WHERE already selected. - -_LLM_TOKEN_SUMMARY_SQL = text( - """ +LLM_CALLS = """ SELECT - o.name AS organization_name, - l.model AS model, - COALESCE(SUM((l.usage->>'total_tokens')::INTEGER), 0) AS sum_total_tokens + o.name AS organization, + p.name AS project, + COUNT(*) FILTER (WHERE l.inserted_at >= now() - INTERVAL '24 hours') AS calls_24h, + COUNT(*) AS calls_7d FROM llm_call l INNER JOIN organization o ON l.organization_id = o.id - WHERE l.inserted_at BETWEEN :start_at AND :end_at + INNER JOIN project p ON l.project_id = p.id + WHERE l.inserted_at >= now() - INTERVAL '168 hours' AND l.deleted_at IS NULL - GROUP BY o.name, l.model - ORDER BY sum_total_tokens DESC - """ -) + GROUP BY o.name, p.name + ORDER BY calls_7d DESC +""" -_LLM_CALL_COUNTS_SQL = text( - """ +LLM_TOKENS = """ SELECT - o.name AS organization_name, - COUNT(*) AS call_count + o.name AS organization, + p.name AS project, + l.model AS model, + COALESCE(SUM((l.usage->>'total_tokens')::INTEGER) + FILTER (WHERE l.inserted_at >= now() - INTERVAL '24 hours'), 0) AS tokens_24h, + COALESCE(SUM((l.usage->>'total_tokens')::INTEGER), 0) AS tokens_7d FROM llm_call l INNER JOIN organization o ON l.organization_id = o.id - WHERE l.inserted_at BETWEEN :start_at AND :end_at + INNER JOIN project p ON l.project_id = p.id + WHERE l.inserted_at >= now() - INTERVAL '168 hours' AND l.deleted_at IS NULL - GROUP BY o.name - ORDER BY call_count DESC - """ -) + GROUP BY o.name, p.name, l.model + ORDER BY tokens_7d DESC +""" -_LLM_MODALITY_COUNTS_SQL = text( - """ +LLM_MODALITY = """ SELECT - o.name AS organization_name, + o.name AS organization, + p.name AS project, CASE WHEN l.input_type = 'text' AND l.output_type = 'text' THEN 'TEXT' WHEN l.input_type = 'audio' AND l.output_type = 'text' THEN 'STT' WHEN l.input_type = 'text' AND l.output_type = 'audio' THEN 'TTS' ELSE 'OTHER' END AS modality, - COUNT(*) AS call_count + COUNT(*) FILTER (WHERE l.inserted_at >= now() - INTERVAL '24 hours') AS calls_24h, + COUNT(*) AS calls_7d FROM llm_call l INNER JOIN organization o ON l.organization_id = o.id - WHERE l.inserted_at BETWEEN :start_at AND :end_at + INNER JOIN project p ON l.project_id = p.id + WHERE l.inserted_at >= now() - INTERVAL '168 hours' AND l.deleted_at IS NULL - GROUP BY o.name, modality - ORDER BY o.name, modality - """ -) + GROUP BY o.name, p.name, modality + ORDER BY o.name, p.name, modality +""" -_JOB_COUNTS_SQL = text( - """ +JOBS = """ SELECT - o.name AS organization_name, + o.name AS organization, + p.name AS project, j.job_type AS job_type, - COUNT(*) AS job_count + COUNT(*) FILTER (WHERE j.inserted_at >= now() - INTERVAL '24 hours') AS jobs_24h, + COUNT(*) AS jobs_7d FROM job j INNER JOIN project p ON j.project_id = p.id INNER JOIN organization o ON p.organization_id = o.id - WHERE j.inserted_at BETWEEN :start_at AND :end_at - GROUP BY o.name, j.job_type - ORDER BY o.name, j.job_type - """ -) + WHERE j.inserted_at >= now() - INTERVAL '168 hours' + GROUP BY o.name, p.name, j.job_type + ORDER BY o.name, p.name, j.job_type +""" +EVALUATION_RUNS = """ + SELECT + o.name AS organization, + p.name AS project, + COUNT(*) FILTER (WHERE t.inserted_at >= now() - INTERVAL '24 hours') AS count_24h, + COUNT(*) AS count_7d + FROM evaluation_run t + INNER JOIN organization o ON t.organization_id = o.id + INNER JOIN project p ON t.project_id = p.id + WHERE t.inserted_at >= now() - INTERVAL '168 hours' + GROUP BY o.name, p.name + ORDER BY count_7d DESC +""" -def _org_count_sql(table: str) -> TextClause: - return text( - f""" - SELECT - o.name AS organization_name, - COUNT(*) AS row_count - FROM {table} t - INNER JOIN organization o ON t.organization_id = o.id - WHERE t.inserted_at BETWEEN :start_at AND :end_at - GROUP BY o.name - ORDER BY row_count DESC - """ - ) +STT_RESULTS = """ + SELECT + o.name AS organization, + p.name AS project, + COUNT(*) FILTER (WHERE t.inserted_at >= now() - INTERVAL '24 hours') AS count_24h, + COUNT(*) AS count_7d + FROM stt_result t + INNER JOIN organization o ON t.organization_id = o.id + INNER JOIN project p ON t.project_id = p.id + WHERE t.inserted_at >= now() - INTERVAL '168 hours' + GROUP BY o.name, p.name + ORDER BY count_7d DESC +""" +TTS_RESULTS = """ + SELECT + o.name AS organization, + p.name AS project, + COUNT(*) FILTER (WHERE t.inserted_at >= now() - INTERVAL '24 hours') AS count_24h, + COUNT(*) AS count_7d + FROM tts_result t + INNER JOIN organization o ON t.organization_id = o.id + INNER JOIN project p ON t.project_id = p.id + WHERE t.inserted_at >= now() - INTERVAL '168 hours' + GROUP BY o.name, p.name + ORDER BY count_7d DESC +""" -_EVAL_RUN_COUNTS_SQL = _org_count_sql("evaluation_run") -_STT_RESULT_COUNTS_SQL = _org_count_sql("stt_result") -_TTS_RESULT_COUNTS_SQL = _org_count_sql("tts_result") -_ASSESSMENT_COUNTS_SQL = _org_count_sql("assessment") +ASSESSMENTS = """ + SELECT + o.name AS organization, + p.name AS project, + COUNT(*) FILTER (WHERE t.inserted_at >= now() - INTERVAL '24 hours') AS count_24h, + COUNT(*) AS count_7d + FROM assessment t + INNER JOIN organization o ON t.organization_id = o.id + INNER JOIN project p ON t.project_id = p.id + WHERE t.inserted_at >= now() - INTERVAL '168 hours' + GROUP BY o.name, p.name + ORDER BY count_7d DESC +""" -def _rows( - session: Session, stmt: TextClause, params: dict[str, Any] -) -> list[dict[str, Any]]: - return [dict(row) for row in session.execute(stmt, params).mappings().all()] +def _rows(session: Session, sql: str) -> list[dict[str, Any]]: + result = session.connection().execute(text(sql)) + return [dict(row) for row in result.mappings().all()] -def get_daily_stats( - *, session: Session, start_at: datetime, end_at: datetime -) -> dict[str, list[dict[str, Any]]]: - params = {"start_at": start_at, "end_at": end_at} - logger.info( - f"[get_daily_stats] Collecting stats | start_at: {start_at.isoformat()}, " - f"end_at: {end_at.isoformat()}" - ) - return { - "llm_call_counts": _rows(session, _LLM_CALL_COUNTS_SQL, params), - "llm_call_token_summary": _rows(session, _LLM_TOKEN_SUMMARY_SQL, params), - "llm_call_modality_counts": _rows(session, _LLM_MODALITY_COUNTS_SQL, params), - "job_type_counts": _rows(session, _JOB_COUNTS_SQL, params), - "evaluation_run_counts": _rows(session, _EVAL_RUN_COUNTS_SQL, params), - "stt_result_counts": _rows(session, _STT_RESULT_COUNTS_SQL, params), - "tts_result_counts": _rows(session, _TTS_RESULT_COUNTS_SQL, params), - "assessment_counts": _rows(session, _ASSESSMENT_COUNTS_SQL, params), - } +def get_daily_stats(*, session: Session) -> dict[str, list[dict[str, Any]]]: + stats: dict[str, list[dict[str, Any]]] = {} + stats["LLM Calls"] = _rows(session, LLM_CALLS) + stats["LLM Tokens"] = _rows(session, LLM_TOKENS) + stats["LLM Modality"] = _rows(session, LLM_MODALITY) + stats["Jobs by Type"] = _rows(session, JOBS) + stats["Evaluation Runs"] = _rows(session, EVALUATION_RUNS) + stats["STT Results"] = _rows(session, STT_RESULTS) + stats["TTS Results"] = _rows(session, TTS_RESULTS) + stats["Assessments"] = _rows(session, ASSESSMENTS) + return stats diff --git a/backend/app/services/stats.py b/backend/app/services/stats.py index 070b91dea..c9c1bf1b8 100644 --- a/backend/app/services/stats.py +++ b/backend/app/services/stats.py @@ -1,139 +1,81 @@ import logging -from datetime import timedelta from typing import Any import requests -from sqlmodel import Session from app.core.config import settings -from app.core.util import now -from app.crud.stats import get_daily_stats logger = logging.getLogger(__name__) -DEFAULT_STATS_WINDOW = timedelta(hours=168) # 7-day rolling window by default -_MAX_ROWS_PER_SECTION = 20 -_DISCORD_CHUNK_LIMIT = 1900 # Discord content cap is 2000; leave headroom - - -def collect_daily_stats( - *, session: Session, window_hours: int | None = None -) -> dict[str, Any]: - end_at = now() - start_at = end_at - ( - timedelta(hours=window_hours) if window_hours else DEFAULT_STATS_WINDOW - ) - stats = get_daily_stats(session=session, start_at=start_at, end_at=end_at) - return { - "window": { - "start_at": start_at.isoformat(), - "end_at": end_at.isoformat(), - }, - "stats": stats, - } - - -_COL_ALIASES = { - "organization_name": "org", - "sum_total_tokens": "tokens", - "call_count": "calls", - "job_count": "jobs", - "row_count": "count", -} - -_SECTION_TITLES = { - "llm_call_counts": "LLM Calls", - "llm_call_token_summary": "LLM Tokens", - "llm_call_modality_counts": "LLM Modality", - "job_type_counts": "Jobs by Type", - "evaluation_run_counts": "Evaluation Runs", - "stt_result_counts": "STT Results", - "tts_result_counts": "TTS Results", - "assessment_counts": "Assessments", -} - - -def _short_ts(iso: str) -> str: - return iso.split(".", 1)[0].replace("T", " ") - - -def _fmt_num(v: Any) -> str: - return f"{v:,}" if isinstance(v, int) else str(v) - - -def _render_section(section: str, rows: list[dict[str, Any]]) -> str | None: - title = _SECTION_TITLES.get(section, section) - if not rows: - return None - cols = list(rows[0].keys()) - sample = rows[:_MAX_ROWS_PER_SECTION] - labels = {c: _COL_ALIASES.get(c, c) for c in cols} - is_num = {c: all(isinstance(r.get(c), (int, float)) for r in sample) for c in cols} - widths = { - c: max(len(labels[c]), *(len(_fmt_num(r.get(c, ""))) for r in sample)) - for c in cols - } - - def fmt(val: Any, c: str) -> str: - s = _fmt_num(val) - return f"{s:>{widths[c]}}" if is_num[c] else f"{s:<{widths[c]}}" - - header = " ".join(fmt(labels[c], c) for c in cols) - body = "\n".join(" ".join(fmt(r.get(c, ""), c) for c in cols) for r in sample) - truncated = ( - f"\n… +{len(rows) - _MAX_ROWS_PER_SECTION} more" - if len(rows) > _MAX_ROWS_PER_SECTION - else "" - ) - return f"\n{title}\n```\n{header}\n{body}{truncated}\n```" - - -def format_daily_stats_message(result: dict[str, Any]) -> str: - window = result["window"] - start = _short_ts(window["start_at"]) - end = _short_ts(window["end_at"]) - blocks = [f"Daily Stats · {start} → {end} UTC"] - - quiet: list[str] = [] - for section, rows in result["stats"].items(): - if not isinstance(rows, list): +DISCORD_LIMIT = 1900 # Discord caps message content at 2000; leave headroom. +MAX_COL_WIDTH = 18 # Cap each column so wide tables don't wrap in Discord. + + +def _clip(text: str) -> str: + if len(text) <= MAX_COL_WIDTH: + return text + return text[: MAX_COL_WIDTH - 1] + "…" + + +def format_sections(stats: dict[str, list[dict[str, Any]]]) -> list[str]: + sections: list[str] = [] + for title, rows in stats.items(): + if not rows: + sections.append(f"**{title}**\n_no data_") continue - rendered = _render_section(section, rows) - if rendered is None: - quiet.append(_SECTION_TITLES.get(section, section)) - else: - blocks.append(rendered) - - if quiet: - blocks.append("\nNo data: " + ", ".join(quiet)) - return "\n".join(blocks) - - -def _chunk_message(message: str, limit: int) -> list[str]: - chunks: list[str] = [] - buf = "" - for block in message.split("\n\n"): - if len(buf) + len(block) + 2 > limit and buf: - chunks.append(buf) - buf = block - else: - buf = f"{buf}\n\n{block}" if buf else block - if buf: - chunks.append(buf) - return chunks - - -def post_daily_stats_to_discord(message: str) -> None: - """Fire-and-forget Discord post. No-op if webhook not configured. - Logs and stops on the first failed chunk — never raises.""" + + columns = list(rows[0].keys()) + + # Numeric columns get thousands separators and are right-aligned so they + # read as a clean column; text columns stay left-aligned. + numeric = { + c: all(isinstance(row[c], (int, float)) for row in rows) for c in columns + } + + def cell(column: str, row: dict[str, Any]) -> str: + value = row[column] + text = f"{value:,}" if numeric[column] else str(value) + return _clip(text) + + widths = {} + for column in columns: + cell_lengths = [len(cell(column, row)) for row in rows] + widths[column] = max(len(_clip(column)), max(cell_lengths)) + + def align(text: str, column: str) -> str: + width = widths[column] + return text.rjust(width) if numeric[column] else text.ljust(width) + + header = " ".join(align(_clip(column), column) for column in columns) + lines = [header.rstrip()] + for row in rows: + line = " ".join(align(cell(column, row), column) for column in columns) + lines.append(line.rstrip()) + + table = "\n".join(lines) + sections.append(f"**{title}**\n```\n{table}\n```") + return sections + + +def post_to_discord(sections: list[str]) -> None: url = settings.DISCORD_STATS_WEBHOOK_URL if not url: return - for chunk in _chunk_message(message, _DISCORD_CHUNK_LIMIT): - try: - requests.post( - str(url), json={"content": chunk}, timeout=5 - ).raise_for_status() - except requests.RequestException as e: - logger.warning(f"[post_daily_stats_to_discord] Webhook post failed: {e}") - return + + # Pack whole sections into messages under Discord's size cap so no code + # block is split across two posts. + chunk = "Daily Stats · last 24h and 7d (UTC)" + for section in sections: + if len(chunk) + len(section) + 2 > DISCORD_LIMIT: + _post(str(url), chunk) + chunk = "" + chunk = f"{chunk}\n\n{section}" if chunk else section + if chunk: + _post(str(url), chunk) + + +def _post(url: str, content: str) -> None: + try: + requests.post(url, json={"content": content}, timeout=5) + except requests.RequestException as e: + logger.warning(f"[post_to_discord] Webhook post failed: {e}") diff --git a/backend/app/tests/services/test_stats.py b/backend/app/tests/services/test_stats.py index e74f64a22..45a9078b4 100644 --- a/backend/app/tests/services/test_stats.py +++ b/backend/app/tests/services/test_stats.py @@ -1,174 +1,67 @@ -from unittest.mock import MagicMock, patch +from unittest.mock import patch import requests from app.services import stats as stats_mod -from app.services.stats import ( - _chunk_message, - collect_daily_stats, - format_daily_stats_message, - post_daily_stats_to_discord, -) +from app.services.stats import format_sections, post_to_discord -def _sample_result() -> dict: +def _sample_stats() -> dict: return { - "window": { - "start_at": "2026-07-03T09:33:52.141126", - "end_at": "2026-07-10T09:33:52.141126", - }, - "stats": { - "llm_call_counts": [ - {"organization_name": "Acme", "call_count": 15}, - ], - "llm_call_token_summary": [ - { - "organization_name": "Acme", - "model": "gpt-4", - "sum_total_tokens": 3800, - }, - { - "organization_name": "Beta", - "model": "claude-4", - "sum_total_tokens": 241, - }, - ], - "stt_result_counts": [], - "tts_result_counts": [], - }, + "LLM Calls": [ + { + "organization": "Acme", + "project": "Alpha", + "calls_24h": 3, + "calls_7d": 15, + }, + ], + "STT Results": [], } -def test_format_message_header_shows_trimmed_window(): - msg = format_daily_stats_message(_sample_result()) - first_line = msg.splitlines()[0] - assert first_line.startswith("Daily Stats") - assert "2026-07-03 09:33:52" in first_line - assert "2026-07-10 09:33:52" in first_line - assert ".141126" not in first_line # microseconds trimmed - - -def test_format_message_uses_friendly_section_titles_and_col_aliases(): - msg = format_daily_stats_message(_sample_result()) - assert "LLM Calls" in msg - assert "LLM Tokens" in msg - # column aliases applied - assert "org" in msg - assert "calls" in msg - assert "tokens" in msg - # raw column names should not appear - assert "organization_name" not in msg - assert "sum_total_tokens" not in msg - - -def test_format_message_collapses_empty_sections_into_one_line(): - msg = format_daily_stats_message(_sample_result()) - quiet_lines = [ln for ln in msg.splitlines() if ln.startswith("No data:")] - assert len(quiet_lines) == 1 - assert "STT Results" in quiet_lines[0] - assert "TTS Results" in quiet_lines[0] - - -def test_format_message_formats_numbers_with_commas(): - msg = format_daily_stats_message(_sample_result()) - assert "3,800" in msg - assert "3800" not in msg - - -def test_format_message_truncates_long_sections(): - big = { - "window": {"start_at": "2026-01-01T00:00:00", "end_at": "2026-01-08T00:00:00"}, - "stats": { - "evaluation_run_counts": [ - {"organization_name": f"org-{i}", "row_count": i} for i in range(50) - ] - }, - } - msg = format_daily_stats_message(big) - assert "… +30 more" in msg # 50 rows - 20 MAX = 30 - - -def test_chunk_message_splits_on_section_boundaries_when_over_limit(): - # Each block ~40 chars — a few fit per 100-char chunk, forcing multiple chunks. - blocks = [f"block-{i}-{'x' * 30}" for i in range(6)] - message = "\n\n".join(blocks) - chunks = _chunk_message(message, limit=100) - assert len(chunks) > 1 - # Every original block appears somewhere in the chunks. - joined = "\n\n".join(chunks) - for b in blocks: - assert b in joined +def test_format_sections_renders_bold_title_and_aligned_table(): + sections = format_sections(_sample_stats()) + llm_section = next(s for s in sections if s.startswith("**LLM Calls**")) + assert "organization project calls_24h calls_7d" in llm_section + assert "Acme Alpha 3 15" in llm_section + assert llm_section.count("```") == 2 # wrapped in one code block -def test_chunk_message_single_chunk_when_under_limit(): - assert _chunk_message("short message", limit=1000) == ["short message"] +def test_format_sections_marks_empty_sections(): + sections = format_sections(_sample_stats()) + stt_section = next(s for s in sections if s.startswith("**STT Results**")) + assert stt_section == "**STT Results**\n_no data_" def test_post_to_discord_noop_when_webhook_unset(): with patch.object(stats_mod.settings, "DISCORD_STATS_WEBHOOK_URL", None), patch( "app.services.stats.requests.post" ) as mock_post: - post_daily_stats_to_discord("anything") + post_to_discord(["anything"]) mock_post.assert_not_called() -def test_post_to_discord_posts_each_chunk(): +def test_post_to_discord_packs_sections_under_size_limit(): posted: list[str] = [] def fake_post(url, json, timeout): posted.append(json["content"]) - resp = MagicMock() - resp.raise_for_status = lambda: None - return resp - long_msg = "hdr\n\n" + "\n\n".join(f"block-{i}" * 100 for i in range(4)) + big_sections = ["x" * 1000 for _ in range(4)] with patch.object( stats_mod.settings, "DISCORD_STATS_WEBHOOK_URL", "https://x/hook" ), patch("app.services.stats.requests.post", side_effect=fake_post): - post_daily_stats_to_discord(long_msg) - assert len(posted) >= 2 # message chunked + post_to_discord(big_sections) + assert len(posted) >= 2 # split into multiple messages + assert all(len(content) <= 2000 for content in posted) def test_post_to_discord_swallows_request_exception(): with patch.object( stats_mod.settings, "DISCORD_STATS_WEBHOOK_URL", "https://x/hook" ), patch( - "app.services.stats.requests.post", side_effect=requests.ConnectionError("boom") + "app.services.stats.requests.post", + side_effect=requests.ConnectionError("boom"), ): - # must not raise - post_daily_stats_to_discord("hello") - - -def test_post_to_discord_stops_after_first_failed_chunk(): - calls = {"n": 0} - - def flaky_post(url, json, timeout): - calls["n"] += 1 - raise requests.ConnectionError("nope") - - long_msg = "\n\n".join(f"block-{i}" * 200 for i in range(4)) - with patch.object( - stats_mod.settings, "DISCORD_STATS_WEBHOOK_URL", "https://x/hook" - ), patch("app.services.stats.requests.post", side_effect=flaky_post): - post_daily_stats_to_discord(long_msg) - assert calls["n"] == 1 # bailed after first failure, didn't retry rest - - -def test_collect_daily_stats_uses_default_window_when_hours_none(): - with patch( - "app.services.stats.get_daily_stats", return_value={"x": []} - ) as mock_get: - result = collect_daily_stats(session=MagicMock()) - call_kwargs = mock_get.call_args.kwargs - delta = call_kwargs["end_at"] - call_kwargs["start_at"] - assert delta == stats_mod.DEFAULT_STATS_WINDOW - assert result["stats"] == {"x": []} - assert "start_at" in result["window"] and "end_at" in result["window"] - - -def test_collect_daily_stats_honours_window_hours_override(): - with patch("app.services.stats.get_daily_stats", return_value={}) as mock_get: - collect_daily_stats(session=MagicMock(), window_hours=3) - call_kwargs = mock_get.call_args.kwargs - delta = call_kwargs["end_at"] - call_kwargs["start_at"] - assert delta.total_seconds() == 3 * 3600 + post_to_discord(["hello"]) # must not raise From bf5baff42e4974c2e246241f9e9e0c58a38d137b Mon Sep 17 00:00:00 2001 From: Prajna1999 Date: Sun, 2 Aug 2026 21:16:39 +0530 Subject: [PATCH 08/16] one tiny senior engineer instruction and ost request raises --- .claude/agents/senior-engineer.md | 3 +++ backend/app/services/stats.py | 6 +++-- backend/app/tests/services/test_stats.py | 29 +++++++++++++++++++++++- 3 files changed, 35 insertions(+), 3 deletions(-) diff --git a/.claude/agents/senior-engineer.md b/.claude/agents/senior-engineer.md index 777a23f6f..856d11099 100644 --- a/.claude/agents/senior-engineer.md +++ b/.claude/agents/senior-engineer.md @@ -46,6 +46,9 @@ to what the task needs. **Cross-cutting:** when a service or crud function wraps an external SDK or raw HTTP call, also Read `.claude/conventions/error-handling.md` and apply its source-tagged, fault-based pattern. + **Coding Style:** unroll loops for better readability and maintainability. Prefer clarity over clever + language tricks. Do not write one-liner loops that involves 1) nested loops 2) complex pydantic interfaces. + 4. **Before writing a helper, check it doesn't already exist.** Anything generic — wrapping an external SDK or its error handling, building/parsing a domain payload, hitting cloud storage, loading config — usually has a canonical version already, and it rarely lives in a neighbor file. diff --git a/backend/app/services/stats.py b/backend/app/services/stats.py index c9c1bf1b8..bc8787315 100644 --- a/backend/app/services/stats.py +++ b/backend/app/services/stats.py @@ -76,6 +76,8 @@ def post_to_discord(sections: list[str]) -> None: def _post(url: str, content: str) -> None: try: - requests.post(url, json={"content": content}, timeout=5) + response = requests.post(url, json={"content": content}, timeout=5) + response.raise_for_status() except requests.RequestException as e: - logger.warning(f"[post_to_discord] Webhook post failed: {e}") + # Log only the exception type — the message can contain the webhook URL. + logger.warning(f"[_post] Webhook post failed: {type(e).__name__}") diff --git a/backend/app/tests/services/test_stats.py b/backend/app/tests/services/test_stats.py index 45a9078b4..16fb2089c 100644 --- a/backend/app/tests/services/test_stats.py +++ b/backend/app/tests/services/test_stats.py @@ -1,4 +1,4 @@ -from unittest.mock import patch +from unittest.mock import MagicMock, patch import requests @@ -34,6 +34,22 @@ def test_format_sections_marks_empty_sections(): assert stt_section == "**STT Results**\n_no data_" +def test_format_sections_clips_long_values_and_formats_numbers(): + stats = { + "LLM Tokens": [ + { + "organization": "Org", + "model": "gemini-3.1-flash-tts-preview", # 28 chars, over the cap + "tokens_7d": 89271, + }, + ], + } + section = format_sections(stats)[0] + assert "gemini-3.1-flash-…" in section # clipped to 17 chars + ellipsis + assert "gemini-3.1-flash-tts-preview" not in section + assert "89,271" in section # thousands separator applied + + def test_post_to_discord_noop_when_webhook_unset(): with patch.object(stats_mod.settings, "DISCORD_STATS_WEBHOOK_URL", None), patch( "app.services.stats.requests.post" @@ -47,6 +63,7 @@ def test_post_to_discord_packs_sections_under_size_limit(): def fake_post(url, json, timeout): posted.append(json["content"]) + return MagicMock() # provides raise_for_status() big_sections = ["x" * 1000 for _ in range(4)] with patch.object( @@ -65,3 +82,13 @@ def test_post_to_discord_swallows_request_exception(): side_effect=requests.ConnectionError("boom"), ): post_to_discord(["hello"]) # must not raise + + +def test_post_to_discord_swallows_non_success_status(): + response = MagicMock() + response.raise_for_status.side_effect = requests.HTTPError("429 Too Many Requests") + with patch.object( + stats_mod.settings, "DISCORD_STATS_WEBHOOK_URL", "https://x/hook" + ), patch("app.services.stats.requests.post", return_value=response): + post_to_discord(["hello"]) # must not raise + response.raise_for_status.assert_called_once() From 24407b902986d678ca5ebc10076fdb6a499e9246 Mon Sep 17 00:00:00 2001 From: Prajna1999 Date: Mon, 3 Aug 2026 10:13:32 +0530 Subject: [PATCH 09/16] fix codecoverage --- backend/app/tests/api/routes/test_cron.py | 57 ++++++++++++++++++++++- backend/app/tests/crud/test_stats.py | 27 +++++++++++ 2 files changed, 83 insertions(+), 1 deletion(-) create mode 100644 backend/app/tests/crud/test_stats.py diff --git a/backend/app/tests/api/routes/test_cron.py b/backend/app/tests/api/routes/test_cron.py index 7b52ee4f9..399ed6eee 100644 --- a/backend/app/tests/api/routes/test_cron.py +++ b/backend/app/tests/api/routes/test_cron.py @@ -1,7 +1,9 @@ -from unittest.mock import AsyncMock, patch +from unittest.mock import AsyncMock, MagicMock, patch +import pytest from fastapi.testclient import TestClient +from app.api.routes import cron from app.core.config import settings from app.tests.utils.auth import TestAuthContext @@ -272,6 +274,58 @@ def test_pending_jobs_cron_job_requires_superuser( assert "superuser" in response_data["error"].lower() +def test_daily_stats_cron_job_success( + client: TestClient, + superuser_api_key: TestAuthContext, +) -> None: + """Returns the collected stats and posts them to Discord.""" + stats = { + "LLM Calls": [ + {"organization": "Acme", "project": "Alpha", "calls_24h": 1, "calls_7d": 2} + ], + } + with ( + patch("app.api.routes.cron.get_daily_stats", return_value=stats), + patch("app.api.routes.cron.post_to_discord") as post, + ): + response = client.get( + f"{settings.API_V1_STR}/cron/daily-stats", + headers={"X-API-KEY": superuser_api_key.key}, + ) + + assert response.status_code == 200 + assert response.json() == stats + post.assert_called_once() + + +def test_daily_stats_cron_job_requires_superuser( + client: TestClient, + user_api_key: TestAuthContext, +) -> None: + """Non-superuser cannot access the daily stats cron endpoint.""" + response = client.get( + f"{settings.API_V1_STR}/cron/daily-stats", + headers={"X-API-KEY": user_api_key.key}, + ) + + assert response.status_code == 403 + + +def test_daily_stats_cron_job_captures_and_reraises_on_error() -> None: + """On failure the job reports to Sentry and re-raises.""" + with ( + patch( + "app.api.routes.cron.get_daily_stats", + side_effect=RuntimeError("boom"), + ), + patch("app.api.routes.cron.sentry_sdk") as sentry, + ): + with pytest.raises(RuntimeError): + cron.daily_stats_cron_job(session=MagicMock()) + + sentry.capture_exception.assert_called_once() + + def test_evaluation_cron_job_not_in_schema( client: TestClient, ) -> None: @@ -285,6 +339,7 @@ def test_evaluation_cron_job_not_in_schema( # Endpoint should not be in the schema due to include_in_schema=False assert f"{settings.API_V1_STR}/cron/evaluations" not in paths assert f"{settings.API_V1_STR}/cron/pending-jobs" not in paths + assert f"{settings.API_V1_STR}/cron/daily-stats" not in paths def test_cron_intervals_match_to_prevent_sentry_monitor_drift() -> None: diff --git a/backend/app/tests/crud/test_stats.py b/backend/app/tests/crud/test_stats.py new file mode 100644 index 000000000..b7453ed24 --- /dev/null +++ b/backend/app/tests/crud/test_stats.py @@ -0,0 +1,27 @@ +from unittest.mock import MagicMock + +from app.crud.stats import get_daily_stats + +EXPECTED_SECTIONS = { + "LLM Calls", + "LLM Tokens", + "LLM Modality", + "Jobs by Type", + "Evaluation Runs", + "STT Results", + "TTS Results", + "Assessments", +} + + +def test_get_daily_stats_runs_every_section_and_maps_rows(): + row = {"organization": "Acme", "project": "Alpha", "calls_7d": 5} + session = MagicMock() + execute = session.connection.return_value.execute + execute.return_value.mappings.return_value.all.return_value = [row] + + stats = get_daily_stats(session=session) + + assert set(stats) == EXPECTED_SECTIONS + assert execute.call_count == len(EXPECTED_SECTIONS) # one query per section + assert stats["LLM Calls"] == [row] # _rows unpacks each mapping into a dict From 4234ba6c2eabe04d17c04313c852bd7165c3a3f1 Mon Sep 17 00:00:00 2001 From: Prajna1999 Date: Fri, 14 Aug 2026 10:09:59 +0530 Subject: [PATCH 10/16] better borders --- backend/app/services/stats.py | 74 +++++++++++++++------- backend/app/tests/services/test_stats.py | 79 +++++++++++++++++------- 2 files changed, 108 insertions(+), 45 deletions(-) diff --git a/backend/app/services/stats.py b/backend/app/services/stats.py index bc8787315..df7d5eeba 100644 --- a/backend/app/services/stats.py +++ b/backend/app/services/stats.py @@ -1,4 +1,5 @@ import logging +from datetime import date from typing import Any import requests @@ -7,8 +8,11 @@ logger = logging.getLogger(__name__) -DISCORD_LIMIT = 1900 # Discord caps message content at 2000; leave headroom. +EMBED_TOTAL_LIMIT = 5900 # Discord caps all embed text in a message at 6000 chars. +EMBED_FIELD_LIMIT = 1000 # Discord caps a single field value at 1024 chars. +MAX_FIELDS_PER_EMBED = 25 # Discord embed field cap. MAX_COL_WIDTH = 18 # Cap each column so wide tables don't wrap in Discord. +BORDER_COLOR = 0x3B82F6 # Blue left-border accent on the Discord embed. def _clip(text: str) -> str: @@ -17,11 +21,12 @@ def _clip(text: str) -> str: return text[: MAX_COL_WIDTH - 1] + "…" -def format_sections(stats: dict[str, list[dict[str, Any]]]) -> list[str]: - sections: list[str] = [] +def format_sections(stats: dict[str, list[dict[str, Any]]]) -> list[dict[str, str]]: + """Build one Discord embed field per stat section (name = title, value = table).""" + fields: list[dict[str, str]] = [] for title, rows in stats.items(): if not rows: - sections.append(f"**{title}**\n_no data_") + fields.append({"name": title, "value": "_no data_"}) continue columns = list(rows[0].keys()) @@ -53,30 +58,57 @@ def align(text: str, column: str) -> str: lines.append(line.rstrip()) table = "\n".join(lines) - sections.append(f"**{title}**\n```\n{table}\n```") - return sections + value = f"```\n{table}\n```" + if len(value) > EMBED_FIELD_LIMIT: + cutoff = EMBED_FIELD_LIMIT - len("\n…\n```") + value = f"{value[:cutoff]}\n…\n```" + fields.append({"name": title, "value": value}) + return fields -def post_to_discord(sections: list[str]) -> None: +def post_to_discord(fields: list[dict[str, str]], *, today: date | None = None) -> None: url = settings.DISCORD_STATS_WEBHOOK_URL if not url: return - # Pack whole sections into messages under Discord's size cap so no code - # block is split across two posts. - chunk = "Daily Stats · last 24h and 7d (UTC)" - for section in sections: - if len(chunk) + len(section) + 2 > DISCORD_LIMIT: - _post(str(url), chunk) - chunk = "" - chunk = f"{chunk}\n\n{section}" if chunk else section - if chunk: - _post(str(url), chunk) - - -def _post(url: str, content: str) -> None: + stat_date = today or date.today() + title = f"Date: {stat_date.month}/{stat_date.day}/{stat_date.year}" + description = "Daily platform feature stats" + + embed = _new_embed(title, description) + for field in fields: + field_cost = len(field["name"]) + len(field["value"]) + if embed["fields"] and ( + len(embed["fields"]) >= MAX_FIELDS_PER_EMBED + or _embed_size(embed) + field_cost > EMBED_TOTAL_LIMIT + ): + _post(str(url), embed) + embed = _new_embed(title, description) + embed["fields"].append(field) + if embed["fields"]: + _post(str(url), embed) + + +def _new_embed(title: str, description: str) -> dict[str, Any]: + return { + "title": title, + "description": description, + "color": BORDER_COLOR, + "fields": [], + } + + +def _embed_size(embed: dict[str, Any]) -> int: + return ( + len(embed["title"]) + + len(embed["description"]) + + sum(len(f["name"]) + len(f["value"]) for f in embed["fields"]) + ) + + +def _post(url: str, embed: dict[str, Any]) -> None: try: - response = requests.post(url, json={"content": content}, timeout=5) + response = requests.post(url, json={"embeds": [embed]}, timeout=5) response.raise_for_status() except requests.RequestException as e: # Log only the exception type — the message can contain the webhook URL. diff --git a/backend/app/tests/services/test_stats.py b/backend/app/tests/services/test_stats.py index 16fb2089c..9b19f8448 100644 --- a/backend/app/tests/services/test_stats.py +++ b/backend/app/tests/services/test_stats.py @@ -1,3 +1,4 @@ +from datetime import date from unittest.mock import MagicMock, patch import requests @@ -20,18 +21,18 @@ def _sample_stats() -> dict: } -def test_format_sections_renders_bold_title_and_aligned_table(): - sections = format_sections(_sample_stats()) - llm_section = next(s for s in sections if s.startswith("**LLM Calls**")) - assert "organization project calls_24h calls_7d" in llm_section - assert "Acme Alpha 3 15" in llm_section - assert llm_section.count("```") == 2 # wrapped in one code block +def test_format_sections_renders_field_per_section_with_aligned_table(): + fields = format_sections(_sample_stats()) + llm_field = next(f for f in fields if f["name"] == "LLM Calls") + assert "organization project calls_24h calls_7d" in llm_field["value"] + assert "Acme Alpha 3 15" in llm_field["value"] + assert llm_field["value"].count("```") == 2 # wrapped in one code block def test_format_sections_marks_empty_sections(): - sections = format_sections(_sample_stats()) - stt_section = next(s for s in sections if s.startswith("**STT Results**")) - assert stt_section == "**STT Results**\n_no data_" + fields = format_sections(_sample_stats()) + stt_field = next(f for f in fields if f["name"] == "STT Results") + assert stt_field["value"] == "_no data_" def test_format_sections_clips_long_values_and_formats_numbers(): @@ -44,34 +45,64 @@ def test_format_sections_clips_long_values_and_formats_numbers(): }, ], } - section = format_sections(stats)[0] - assert "gemini-3.1-flash-…" in section # clipped to 17 chars + ellipsis - assert "gemini-3.1-flash-tts-preview" not in section - assert "89,271" in section # thousands separator applied + value = format_sections(stats)[0]["value"] + assert "gemini-3.1-flash-…" in value # clipped to 17 chars + ellipsis + assert "gemini-3.1-flash-tts-preview" not in value + assert "89,271" in value # thousands separator applied def test_post_to_discord_noop_when_webhook_unset(): with patch.object(stats_mod.settings, "DISCORD_STATS_WEBHOOK_URL", None), patch( "app.services.stats.requests.post" ) as mock_post: - post_to_discord(["anything"]) + post_to_discord([{"name": "X", "value": "y"}]) mock_post.assert_not_called() -def test_post_to_discord_packs_sections_under_size_limit(): - posted: list[str] = [] +def test_post_to_discord_sets_title_description_and_border_color(): + posted: list[dict] = [] def fake_post(url, json, timeout): - posted.append(json["content"]) - return MagicMock() # provides raise_for_status() + posted.append(json["embeds"][0]) + return MagicMock() - big_sections = ["x" * 1000 for _ in range(4)] with patch.object( stats_mod.settings, "DISCORD_STATS_WEBHOOK_URL", "https://x/hook" ), patch("app.services.stats.requests.post", side_effect=fake_post): - post_to_discord(big_sections) - assert len(posted) >= 2 # split into multiple messages - assert all(len(content) <= 2000 for content in posted) + post_to_discord( + [{"name": "LLM Calls", "value": "```\nx\n```"}], + today=date(2026, 8, 10), + ) + + assert len(posted) == 1 + embed = posted[0] + assert embed["title"] == "Date: 8/10/2026" + assert embed["description"] == "Daily platform feature stats" + assert embed["color"] == stats_mod.BORDER_COLOR + assert embed["fields"] == [{"name": "LLM Calls", "value": "```\nx\n```"}] + + +def test_post_to_discord_splits_fields_across_embeds_under_size_limit(): + posted: list[dict] = [] + + def fake_post(url, json, timeout): + posted.append(json["embeds"][0]) + return MagicMock() + + big_fields = [{"name": f"Section {i}", "value": "x" * 2000} for i in range(4)] + with patch.object( + stats_mod.settings, "DISCORD_STATS_WEBHOOK_URL", "https://x/hook" + ), patch("app.services.stats.requests.post", side_effect=fake_post): + post_to_discord(big_fields) + + assert len(posted) >= 2 # split into multiple embeds/messages + for embed in posted: + size = ( + len(embed["title"]) + + len(embed["description"]) + + sum(len(f["name"]) + len(f["value"]) for f in embed["fields"]) + ) + assert size <= stats_mod.EMBED_TOTAL_LIMIT def test_post_to_discord_swallows_request_exception(): @@ -81,7 +112,7 @@ def test_post_to_discord_swallows_request_exception(): "app.services.stats.requests.post", side_effect=requests.ConnectionError("boom"), ): - post_to_discord(["hello"]) # must not raise + post_to_discord([{"name": "X", "value": "y"}]) # must not raise def test_post_to_discord_swallows_non_success_status(): @@ -90,5 +121,5 @@ def test_post_to_discord_swallows_non_success_status(): with patch.object( stats_mod.settings, "DISCORD_STATS_WEBHOOK_URL", "https://x/hook" ), patch("app.services.stats.requests.post", return_value=response): - post_to_discord(["hello"]) # must not raise + post_to_discord([{"name": "X", "value": "y"}]) # must not raise response.raise_for_status.assert_called_once() From b0b7833f2f19ad65e1a3ba814a754d4b94649f78 Mon Sep 17 00:00:00 2001 From: Prajna1999 Date: Fri, 14 Aug 2026 11:54:04 +0530 Subject: [PATCH 11/16] fix orientation of message --- backend/app/services/stats.py | 90 ++++++++++++++++-------- backend/app/tests/services/test_stats.py | 19 ++--- 2 files changed, 69 insertions(+), 40 deletions(-) diff --git a/backend/app/services/stats.py b/backend/app/services/stats.py index df7d5eeba..85944614c 100644 --- a/backend/app/services/stats.py +++ b/backend/app/services/stats.py @@ -11,14 +11,13 @@ EMBED_TOTAL_LIMIT = 5900 # Discord caps all embed text in a message at 6000 chars. EMBED_FIELD_LIMIT = 1000 # Discord caps a single field value at 1024 chars. MAX_FIELDS_PER_EMBED = 25 # Discord embed field cap. -MAX_COL_WIDTH = 18 # Cap each column so wide tables don't wrap in Discord. BORDER_COLOR = 0x3B82F6 # Blue left-border accent on the Discord embed. +COLUMN_LABELS = {"24h": "Last 24hrs", "7d": "Last 7 days"} -def _clip(text: str) -> str: - if len(text) <= MAX_COL_WIDTH: - return text - return text[: MAX_COL_WIDTH - 1] + "…" +def _column_label(column: str) -> str: + suffix = column.rsplit("_", 1)[-1] + return COLUMN_LABELS.get(suffix, column) def format_sections(stats: dict[str, list[dict[str, Any]]]) -> list[dict[str, str]]: @@ -30,38 +29,67 @@ def format_sections(stats: dict[str, list[dict[str, Any]]]) -> list[dict[str, st continue columns = list(rows[0].keys()) - - # Numeric columns get thousands separators and are right-aligned so they - # read as a clean column; text columns stay left-aligned. - numeric = { - c: all(isinstance(row[c], (int, float)) for row in rows) for c in columns - } + numeric_cols = [ + c for c in columns if all(isinstance(row[c], (int, float)) for row in rows) + ] + text_cols = [c for c in columns if c not in numeric_cols] + + # Repeating org/project on every row is what pushed rows past Discord's + # embed width and wrapped the trailing columns. Instead, group rows by + # every text column but the last one and print that group once as a + # bold heading (bold only works outside the code block), leaving just + # the varying column + metrics — short enough to fit — in the table. + group_cols = text_cols[:-1] + row_col = text_cols[-1] if text_cols else None + table_cols = ([row_col] if row_col else []) + numeric_cols + headers = {c: _column_label(c) for c in table_cols} def cell(column: str, row: dict[str, Any]) -> str: value = row[column] - text = f"{value:,}" if numeric[column] else str(value) - return _clip(text) - - widths = {} - for column in columns: - cell_lengths = [len(cell(column, row)) for row in rows] - widths[column] = max(len(_clip(column)), max(cell_lengths)) + return f"{value:,}" if column in numeric_cols else str(value) - def align(text: str, column: str) -> str: - width = widths[column] - return text.rjust(width) if numeric[column] else text.ljust(width) - - header = " ".join(align(_clip(column), column) for column in columns) - lines = [header.rstrip()] + groups: dict[tuple[Any, ...], list[dict[str, Any]]] = {} for row in rows: - line = " ".join(align(cell(column, row), column) for column in columns) - lines.append(line.rstrip()) - - table = "\n".join(lines) - value = f"```\n{table}\n```" + groups.setdefault(tuple(row[c] for c in group_cols), []).append(row) + + blocks = [] + for key, group_rows in groups.items(): + # Column width is driven by the widest value, uncapped, so long + # names (e.g. model ids) are never truncated. + widths = { + c: max(len(headers[c]), max(len(cell(c, r)) for r in group_rows)) + for c in table_cols + } + + def align(text: str, column: str) -> str: + width = widths[column] + return ( + text.rjust(width) if column in numeric_cols else text.ljust(width) + ) + + lines = [ + " ".join(align(headers[c], c) for c in table_cols).rstrip(), + ] + for row in group_rows: + lines.append( + " ".join(align(cell(c, row), c) for c in table_cols).rstrip() + ) + + table = "\n".join(lines) + block = f"```\n{table}\n```" + if group_cols: + block = f"**{' / '.join(str(k) for k in key)}**\n{block}" + blocks.append(block) + + value = "\n".join(blocks) if len(value) > EMBED_FIELD_LIMIT: cutoff = EMBED_FIELD_LIMIT - len("\n…\n```") - value = f"{value[:cutoff]}\n…\n```" + truncated = value[:cutoff] + value = ( + f"{truncated}\n…\n```" + if truncated.count("```") % 2 == 1 + else f"{truncated}\n…" + ) fields.append({"name": title, "value": value}) return fields @@ -72,7 +100,7 @@ def post_to_discord(fields: list[dict[str, str]], *, today: date | None = None) return stat_date = today or date.today() - title = f"Date: {stat_date.month}/{stat_date.day}/{stat_date.year}" + title = f"Date: {stat_date.day}/{stat_date.month}/{stat_date.year}" description = "Daily platform feature stats" embed = _new_embed(title, description) diff --git a/backend/app/tests/services/test_stats.py b/backend/app/tests/services/test_stats.py index 9b19f8448..860a7b9d6 100644 --- a/backend/app/tests/services/test_stats.py +++ b/backend/app/tests/services/test_stats.py @@ -21,12 +21,14 @@ def _sample_stats() -> dict: } -def test_format_sections_renders_field_per_section_with_aligned_table(): +def test_format_sections_groups_by_leading_columns_with_bold_heading(): fields = format_sections(_sample_stats()) llm_field = next(f for f in fields if f["name"] == "LLM Calls") - assert "organization project calls_24h calls_7d" in llm_field["value"] - assert "Acme Alpha 3 15" in llm_field["value"] - assert llm_field["value"].count("```") == 2 # wrapped in one code block + value = llm_field["value"] + assert value.startswith("**Acme**\n```\n") # bold heading outside the code block + assert "project Last 24hrs Last 7 days" in value + assert "Alpha 3 15" in value + assert value.count("```") == 2 # rows wrapped in one code block def test_format_sections_marks_empty_sections(): @@ -35,19 +37,18 @@ def test_format_sections_marks_empty_sections(): assert stt_field["value"] == "_no data_" -def test_format_sections_clips_long_values_and_formats_numbers(): +def test_format_sections_does_not_truncate_long_model_names(): stats = { "LLM Tokens": [ { "organization": "Org", - "model": "gemini-3.1-flash-tts-preview", # 28 chars, over the cap + "model": "gemini-3.1-flash-tts-preview", # long, must stay whole "tokens_7d": 89271, }, ], } value = format_sections(stats)[0]["value"] - assert "gemini-3.1-flash-…" in value # clipped to 17 chars + ellipsis - assert "gemini-3.1-flash-tts-preview" not in value + assert "gemini-3.1-flash-tts-preview" in value # not clipped assert "89,271" in value # thousands separator applied @@ -76,7 +77,7 @@ def fake_post(url, json, timeout): assert len(posted) == 1 embed = posted[0] - assert embed["title"] == "Date: 8/10/2026" + assert embed["title"] == "Date: 10/8/2026" assert embed["description"] == "Daily platform feature stats" assert embed["color"] == stats_mod.BORDER_COLOR assert embed["fields"] == [{"name": "LLM Calls", "value": "```\nx\n```"}] From bf04c13b9964549788801ba78e01e78383e1ea03 Mon Sep 17 00:00:00 2001 From: Prajna Prayas Date: Fri, 14 Aug 2026 15:30:42 +0530 Subject: [PATCH 12/16] Update backend/app/services/stats.py Co-authored-by: Ayush <80516839+Ayush8923@users.noreply.github.com> --- backend/app/services/stats.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/backend/app/services/stats.py b/backend/app/services/stats.py index 85944614c..b914891f8 100644 --- a/backend/app/services/stats.py +++ b/backend/app/services/stats.py @@ -8,10 +8,10 @@ logger = logging.getLogger(__name__) -EMBED_TOTAL_LIMIT = 5900 # Discord caps all embed text in a message at 6000 chars. -EMBED_FIELD_LIMIT = 1000 # Discord caps a single field value at 1024 chars. -MAX_FIELDS_PER_EMBED = 25 # Discord embed field cap. -BORDER_COLOR = 0x3B82F6 # Blue left-border accent on the Discord embed. +DISCORD_EMBED_TOTAL_TEXT_LIMIT = 5900 +DISCORD_EMBED_FIELD_VALUE_LIMIT = 1000 +DISCORD_EMBED_FIELD_COUNT_LIMIT = 25 +DISCORD_EMBED_BORDER_COLOR = 0x3B82F6 COLUMN_LABELS = {"24h": "Last 24hrs", "7d": "Last 7 days"} From 029fba87a740cee05fee18338f5e613c4092e09d Mon Sep 17 00:00:00 2001 From: Prajna Prayas Date: Fri, 14 Aug 2026 15:30:59 +0530 Subject: [PATCH 13/16] Update backend/app/services/stats.py Co-authored-by: Ayush <80516839+Ayush8923@users.noreply.github.com> --- backend/app/services/stats.py | 1 - 1 file changed, 1 deletion(-) diff --git a/backend/app/services/stats.py b/backend/app/services/stats.py index b914891f8..a830dfc5f 100644 --- a/backend/app/services/stats.py +++ b/backend/app/services/stats.py @@ -139,5 +139,4 @@ def _post(url: str, embed: dict[str, Any]) -> None: response = requests.post(url, json={"embeds": [embed]}, timeout=5) response.raise_for_status() except requests.RequestException as e: - # Log only the exception type — the message can contain the webhook URL. logger.warning(f"[_post] Webhook post failed: {type(e).__name__}") From defb7ad3924f00e20812c64310b3a919cef12d56 Mon Sep 17 00:00:00 2001 From: Prajna1999 Date: Fri, 14 Aug 2026 15:48:42 +0530 Subject: [PATCH 14/16] Restyle Discord daily-stats message: aligned tables, typed embeds, group empty sections Rebuilds the stats formatter to match the requested Discord embed design (bold group headings, monospace tables with proper column spacing, full model names), replaces dict[str, Any] with typed StatRow/EmbedField/DiscordEmbed shapes, fixes stale constant references, and collapses inactive sections into a single "No activity this week" field instead of one per empty section. Co-Authored-By: Claude Sonnet 5 --- backend/app/services/stats.py | 63 +++++++++++++++++------- backend/app/tests/services/test_stats.py | 24 ++++++--- 2 files changed, 62 insertions(+), 25 deletions(-) diff --git a/backend/app/services/stats.py b/backend/app/services/stats.py index a830dfc5f..1741c8169 100644 --- a/backend/app/services/stats.py +++ b/backend/app/services/stats.py @@ -1,6 +1,6 @@ import logging from datetime import date -from typing import Any +from typing import TypedDict import requests @@ -8,24 +8,44 @@ logger = logging.getLogger(__name__) -DISCORD_EMBED_TOTAL_TEXT_LIMIT = 5900 -DISCORD_EMBED_FIELD_VALUE_LIMIT = 1000 -DISCORD_EMBED_FIELD_COUNT_LIMIT = 25 -DISCORD_EMBED_BORDER_COLOR = 0x3B82F6 +DISCORD_EMBED_TOTAL_TEXT_LIMIT = ( + 5900 # Discord caps all embed text in a message at 6000 chars. +) +DISCORD_EMBED_FIELD_VALUE_LIMIT = ( + 1000 # Discord caps a single field value at 1024 chars. +) +DISCORD_EMBED_FIELD_COUNT_LIMIT = 25 # Discord embed field cap. +DISCORD_EMBED_BORDER_COLOR = 0x3B82F6 # Blue left-border accent on the Discord embed. COLUMN_LABELS = {"24h": "Last 24hrs", "7d": "Last 7 days"} +StatValue = str | int | float +StatRow = dict[str, StatValue] + + +class EmbedField(TypedDict): + name: str + value: str + + +class DiscordEmbed(TypedDict): + title: str + description: str + color: int + fields: list[EmbedField] + def _column_label(column: str) -> str: suffix = column.rsplit("_", 1)[-1] return COLUMN_LABELS.get(suffix, column) -def format_sections(stats: dict[str, list[dict[str, Any]]]) -> list[dict[str, str]]: +def format_sections(stats: dict[str, list[StatRow]]) -> list[EmbedField]: """Build one Discord embed field per stat section (name = title, value = table).""" - fields: list[dict[str, str]] = [] + fields: list[EmbedField] = [] + inactive_titles: list[str] = [] for title, rows in stats.items(): if not rows: - fields.append({"name": title, "value": "_no data_"}) + inactive_titles.append(title) continue columns = list(rows[0].keys()) @@ -44,11 +64,11 @@ def format_sections(stats: dict[str, list[dict[str, Any]]]) -> list[dict[str, st table_cols = ([row_col] if row_col else []) + numeric_cols headers = {c: _column_label(c) for c in table_cols} - def cell(column: str, row: dict[str, Any]) -> str: + def cell(column: str, row: StatRow) -> str: value = row[column] return f"{value:,}" if column in numeric_cols else str(value) - groups: dict[tuple[Any, ...], list[dict[str, Any]]] = {} + groups: dict[tuple[StatValue, ...], list[StatRow]] = {} for row in rows: groups.setdefault(tuple(row[c] for c in group_cols), []).append(row) @@ -82,8 +102,8 @@ def align(text: str, column: str) -> str: blocks.append(block) value = "\n".join(blocks) - if len(value) > EMBED_FIELD_LIMIT: - cutoff = EMBED_FIELD_LIMIT - len("\n…\n```") + if len(value) > DISCORD_EMBED_FIELD_VALUE_LIMIT: + cutoff = DISCORD_EMBED_FIELD_VALUE_LIMIT - len("\n…\n```") truncated = value[:cutoff] value = ( f"{truncated}\n…\n```" @@ -91,10 +111,14 @@ def align(text: str, column: str) -> str: else f"{truncated}\n…" ) fields.append({"name": title, "value": value}) + + if inactive_titles: + bullets = "\n".join(f"• {t}" for t in inactive_titles) + fields.append({"name": "No activity this week", "value": bullets}) return fields -def post_to_discord(fields: list[dict[str, str]], *, today: date | None = None) -> None: +def post_to_discord(fields: list[EmbedField], *, today: date | None = None) -> None: url = settings.DISCORD_STATS_WEBHOOK_URL if not url: return @@ -107,8 +131,8 @@ def post_to_discord(fields: list[dict[str, str]], *, today: date | None = None) for field in fields: field_cost = len(field["name"]) + len(field["value"]) if embed["fields"] and ( - len(embed["fields"]) >= MAX_FIELDS_PER_EMBED - or _embed_size(embed) + field_cost > EMBED_TOTAL_LIMIT + len(embed["fields"]) >= DISCORD_EMBED_FIELD_COUNT_LIMIT + or _embed_size(embed) + field_cost > DISCORD_EMBED_TOTAL_TEXT_LIMIT ): _post(str(url), embed) embed = _new_embed(title, description) @@ -117,16 +141,16 @@ def post_to_discord(fields: list[dict[str, str]], *, today: date | None = None) _post(str(url), embed) -def _new_embed(title: str, description: str) -> dict[str, Any]: +def _new_embed(title: str, description: str) -> DiscordEmbed: return { "title": title, "description": description, - "color": BORDER_COLOR, + "color": DISCORD_EMBED_BORDER_COLOR, "fields": [], } -def _embed_size(embed: dict[str, Any]) -> int: +def _embed_size(embed: DiscordEmbed) -> int: return ( len(embed["title"]) + len(embed["description"]) @@ -134,9 +158,10 @@ def _embed_size(embed: dict[str, Any]) -> int: ) -def _post(url: str, embed: dict[str, Any]) -> None: +def _post(url: str, embed: DiscordEmbed) -> None: try: response = requests.post(url, json={"embeds": [embed]}, timeout=5) response.raise_for_status() except requests.RequestException as e: + # Log only the exception type — the message can contain the webhook URL. logger.warning(f"[_post] Webhook post failed: {type(e).__name__}") diff --git a/backend/app/tests/services/test_stats.py b/backend/app/tests/services/test_stats.py index 860a7b9d6..a6704677e 100644 --- a/backend/app/tests/services/test_stats.py +++ b/backend/app/tests/services/test_stats.py @@ -31,10 +31,22 @@ def test_format_sections_groups_by_leading_columns_with_bold_heading(): assert value.count("```") == 2 # rows wrapped in one code block -def test_format_sections_marks_empty_sections(): - fields = format_sections(_sample_stats()) - stt_field = next(f for f in fields if f["name"] == "STT Results") - assert stt_field["value"] == "_no data_" +def test_format_sections_groups_empty_sections_into_one_field(): + stats = { + "LLM Calls": [ + {"organization": "Acme", "project": "Alpha", "calls_24h": 3, "calls_7d": 15} + ], + "Evaluation Runs": [], + "STT Results": [], + "TTS Results": [], + "Assessments": [], + } + fields = format_sections(stats) + assert [f["name"] for f in fields] == ["LLM Calls", "No activity this week"] + inactive_field = fields[-1] + assert inactive_field["value"] == ( + "• Evaluation Runs\n• STT Results\n• TTS Results\n• Assessments" + ) def test_format_sections_does_not_truncate_long_model_names(): @@ -79,7 +91,7 @@ def fake_post(url, json, timeout): embed = posted[0] assert embed["title"] == "Date: 10/8/2026" assert embed["description"] == "Daily platform feature stats" - assert embed["color"] == stats_mod.BORDER_COLOR + assert embed["color"] == stats_mod.DISCORD_EMBED_BORDER_COLOR assert embed["fields"] == [{"name": "LLM Calls", "value": "```\nx\n```"}] @@ -103,7 +115,7 @@ def fake_post(url, json, timeout): + len(embed["description"]) + sum(len(f["name"]) + len(f["value"]) for f in embed["fields"]) ) - assert size <= stats_mod.EMBED_TOTAL_LIMIT + assert size <= stats_mod.DISCORD_EMBED_TOTAL_TEXT_LIMIT def test_post_to_discord_swallows_request_exception(): From 34f2c66436c02297ba23d9e9920dea5aa4c004c5 Mon Sep 17 00:00:00 2001 From: Prajna1999 Date: Tue, 18 Aug 2026 11:08:50 +0530 Subject: [PATCH 15/16] remove dict[str,any], coalsece SQL queries --- backend/app/crud/stats.py | 84 +++++++++++++-------------------------- 1 file changed, 28 insertions(+), 56 deletions(-) diff --git a/backend/app/crud/stats.py b/backend/app/crud/stats.py index dbae337a6..a13d6b38f 100644 --- a/backend/app/crud/stats.py +++ b/backend/app/crud/stats.py @@ -1,11 +1,11 @@ -from typing import Any +import re from sqlalchemy import text from sqlmodel import Session -# Every query reports two rolling windows in one pass: the last 24 hours and the -# last 7 days (168 hours), broken down per organization and per project. The -# 24h FILTER count is a subset of the rows the 7d WHERE already selected. +StatValue = str | int | float +StatRow = dict[str, StatValue] + LLM_CALLS = """ SELECT @@ -75,13 +75,13 @@ ORDER BY o.name, p.name, j.job_type """ -EVALUATION_RUNS = """ +_SIMPLE_COUNT = """ SELECT o.name AS organization, p.name AS project, COUNT(*) FILTER (WHERE t.inserted_at >= now() - INTERVAL '24 hours') AS count_24h, COUNT(*) AS count_7d - FROM evaluation_run t + FROM {table} t INNER JOIN organization o ON t.organization_id = o.id INNER JOIN project p ON t.project_id = p.id WHERE t.inserted_at >= now() - INTERVAL '168 hours' @@ -89,62 +89,34 @@ ORDER BY count_7d DESC """ -STT_RESULTS = """ - SELECT - o.name AS organization, - p.name AS project, - COUNT(*) FILTER (WHERE t.inserted_at >= now() - INTERVAL '24 hours') AS count_24h, - COUNT(*) AS count_7d - FROM stt_result t - INNER JOIN organization o ON t.organization_id = o.id - INNER JOIN project p ON t.project_id = p.id - WHERE t.inserted_at >= now() - INTERVAL '168 hours' - GROUP BY o.name, p.name - ORDER BY count_7d DESC -""" +SIMPLE_COUNT_TABLES = { + "Evaluation Runs": "evaluation_run", + "STT Results": "stt_result", + "TTS Results": "tts_result", + "Assessments": "assessment", +} -TTS_RESULTS = """ - SELECT - o.name AS organization, - p.name AS project, - COUNT(*) FILTER (WHERE t.inserted_at >= now() - INTERVAL '24 hours') AS count_24h, - COUNT(*) AS count_7d - FROM tts_result t - INNER JOIN organization o ON t.organization_id = o.id - INNER JOIN project p ON t.project_id = p.id - WHERE t.inserted_at >= now() - INTERVAL '168 hours' - GROUP BY o.name, p.name - ORDER BY count_7d DESC -""" +_IDENTIFIER = re.compile(r"^[a-z_][a-z0-9_]*$") -ASSESSMENTS = """ - SELECT - o.name AS organization, - p.name AS project, - COUNT(*) FILTER (WHERE t.inserted_at >= now() - INTERVAL '24 hours') AS count_24h, - COUNT(*) AS count_7d - FROM assessment t - INNER JOIN organization o ON t.organization_id = o.id - INNER JOIN project p ON t.project_id = p.id - WHERE t.inserted_at >= now() - INTERVAL '168 hours' - GROUP BY o.name, p.name - ORDER BY count_7d DESC -""" + +def _simple_count_sql(table: str) -> str: + if not _IDENTIFIER.match(table): + raise ValueError(f"unsafe table identifier: {table!r}") + return _SIMPLE_COUNT.format(table=table) -def _rows(session: Session, sql: str) -> list[dict[str, Any]]: +def _rows(session: Session, sql: str) -> list[StatRow]: result = session.connection().execute(text(sql)) return [dict(row) for row in result.mappings().all()] -def get_daily_stats(*, session: Session) -> dict[str, list[dict[str, Any]]]: - stats: dict[str, list[dict[str, Any]]] = {} - stats["LLM Calls"] = _rows(session, LLM_CALLS) - stats["LLM Tokens"] = _rows(session, LLM_TOKENS) - stats["LLM Modality"] = _rows(session, LLM_MODALITY) - stats["Jobs by Type"] = _rows(session, JOBS) - stats["Evaluation Runs"] = _rows(session, EVALUATION_RUNS) - stats["STT Results"] = _rows(session, STT_RESULTS) - stats["TTS Results"] = _rows(session, TTS_RESULTS) - stats["Assessments"] = _rows(session, ASSESSMENTS) +def get_daily_stats(*, session: Session) -> dict[str, list[StatRow]]: + stats: dict[str, list[StatRow]] = { + "LLM Calls": _rows(session, LLM_CALLS), + "LLM Tokens": _rows(session, LLM_TOKENS), + "LLM Modality": _rows(session, LLM_MODALITY), + "Jobs by Type": _rows(session, JOBS), + } + for label, table in SIMPLE_COUNT_TABLES.items(): + stats[label] = _rows(session, _simple_count_sql(table)) return stats From 0f119884a22fabfc6e6765ba0252bc1ccef7f9c3 Mon Sep 17 00:00:00 2001 From: Prajna1999 Date: Tue, 18 Aug 2026 11:20:01 +0530 Subject: [PATCH 16/16] single entrypoint for type safety --- backend/app/api/routes/cron.py | 5 ++--- backend/app/services/stats.py | 4 +--- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/backend/app/api/routes/cron.py b/backend/app/api/routes/cron.py index c285d0fe2..9e11d033a 100644 --- a/backend/app/api/routes/cron.py +++ b/backend/app/api/routes/cron.py @@ -1,5 +1,4 @@ import logging -from typing import Any import sentry_sdk from fastapi import APIRouter, Depends @@ -10,7 +9,7 @@ from app.core.config import settings from app.crud.evaluations import process_all_pending_evaluations from app.services.job_monitoring import monitor_pending_jobs -from app.crud.stats import get_daily_stats +from app.crud.stats import StatRow, get_daily_stats from app.services.stats import format_sections, post_to_discord logger = logging.getLogger(__name__) @@ -145,7 +144,7 @@ async def evaluation_cron_job( monitor_slug="daily-stats-cron-job", monitor_config=DAILY_STATS_CRON_MONITOR_CONFIG, ) -def daily_stats_cron_job(session: SessionDep) -> dict[str, Any]: +def daily_stats_cron_job(session: SessionDep) -> dict[str, list[StatRow]]: try: stats = get_daily_stats(session=session) post_to_discord(format_sections(stats)) diff --git a/backend/app/services/stats.py b/backend/app/services/stats.py index 1741c8169..da92b1b0b 100644 --- a/backend/app/services/stats.py +++ b/backend/app/services/stats.py @@ -5,6 +5,7 @@ import requests from app.core.config import settings +from app.crud.stats import StatRow, StatValue logger = logging.getLogger(__name__) @@ -18,9 +19,6 @@ DISCORD_EMBED_BORDER_COLOR = 0x3B82F6 # Blue left-border accent on the Discord embed. COLUMN_LABELS = {"24h": "Last 24hrs", "7d": "Last 7 days"} -StatValue = str | int | float -StatRow = dict[str, StatValue] - class EmbedField(TypedDict): name: str