Merge agent-web-integration: Ask-the-Crypt agent in the web app

In-app RAG agent (FastAPI + Next.js 'Ask the Crypt' panel, cited places pinned
on the map), model-agnostic (Claude or free local Ollama), agent-result UI
polish, a public/-assets Docker fix, and refreshed demo media.

Author: NeilP211

README.md

@@ -14,7 +14,7 @@ country, ranked by visual similarity, distance, and lore.
 
 <br />
 
-<img src="docs/images/demo.gif" alt="Crypt demo: drop a photo, get ranked haunted places on the map" width="840" />
+<img src="docs/images/demo.gif" alt="Crypt demo: ask a question and get cited haunted places pinned on the map" width="840" />
 
 </div>
 
@@ -60,15 +60,15 @@ Rust**. The pieces:
 
 ## Screenshots
 
-<img src="docs/images/app-search.png" alt="Visual search results" width="100%" />
+<img src="docs/images/app-search.png" alt="Ask the Crypt: cited places in the sidebar, pinned on the map" width="100%" />
 
-Drop a photo and Crypt returns ranked matches: numbered pins on the map and
-scored results in the sidebar.
+Ask the Crypt a question and the agent answers with cited places: listed in the
+sidebar and pinned on the map (here, haunted cemeteries across Illinois).
 
 | | |
 |:--|:--|
-| <img src="docs/images/app-detail.png" alt="Location detail panel" /> | <img src="docs/images/app-map.png" alt="Full-bleed haunted map" /> |
-| Every place opens a detail card with a category illustration, the recorded haunting, and links to dig deeper. | Collapse the panel for a map |
+| <img src="docs/images/app-detail.png" alt="Location detail panel" /> | <img src="docs/images/app-map.png" alt="The Ask box beside the full haunted map" /> |
+| Every place opens a detail card with a category illustration, the recorded haunting, and links to dig deeper. | The Ask box and visual search sit beside the full haunted-places map. |
 
 ## Highlights
 

agent/README.md

@@ -6,6 +6,8 @@ retrieval stack with a tool-using planner and a faithfulness eval harness, so
 every answer is grounded in retrieved evidence and the whole pipeline is
 measured, not vibes.
 
+<img src="../docs/images/demo.gif" alt="Ask the Crypt: a question returns cited places pinned on the map" width="840" />
+
 ## What it does
 
 ```

agent/crypt_agent/agent.py

@@ -28,6 +28,13 @@
 
 Keep answers tight: a few sentences or a short list, each item cited."""
 
+# Used for the local-model (Ollama) path, which retrieves first then answers.
+RAG_SYSTEM = (
+    "You are Crypt, answering questions about haunted and abandoned places "
+    "strictly from the evidence given to you, citing each place's id in square "
+    "brackets. Never invent places or details."
+)
+
 _CITE_RE = re.compile(r"\[(\d+)\]")
 
 
@@ -52,6 +59,10 @@ def __init__(self, retriever: Retriever | None = None, llm: LLM | None = None):
         self.llm = llm or LLM()
 
     def answer(self, question: str, max_steps: int = 5) -> AgentResult:
+        # The tool-use loop targets Anthropic; the local (Ollama) path uses a
+        # single-shot retrieve-then-answer, reliable on small models.
+        if self.llm.settings.provider != "anthropic":
+            return self._answer_rag(question)
         toolbox = Toolbox(self.retriever)
         messages: list[dict] = [{"role": "user", "content": question}]
         tool_calls_log: list[dict] = []
@@ -105,6 +116,28 @@ def answer(self, question: str, max_steps: int = 5) -> AgentResult:
         )
 
 
+    def _answer_rag(self, question: str, k: int = 6) -> AgentResult:
+        """Single-shot retrieve-then-answer, used for the local-model path."""
+        hits = self.retriever.search(question, k=k)
+        context = "\n".join(f"[{h.place.id}] {h.place.document()}" for h in hits)
+        prompt = (
+            f"PLACES:\n{context}\n\nQUESTION: {question}\n\n"
+            "Answer using ONLY the places above. After each factual claim, cite the "
+            "place id in square brackets like [12]. If the places do not answer the "
+            "question, say so plainly. Keep it to a few sentences."
+        )
+        resp = self.llm.message([{"role": "user", "content": prompt}], system=RAG_SYSTEM)
+        cited = sorted({int(m) for m in _CITE_RE.findall(resp.text)})
+        return AgentResult(
+            question=question,
+            answer=resp.text,
+            cited_ids=cited,
+            seen_ids=sorted({h.place.id for h in hits}),
+            steps=1,
+            tool_calls=[],
+        )
+
+
 def _stringify(output: dict) -> str:
     import json
 

agent/crypt_agent/api.py

@@ -0,0 +1,112 @@
+"""HTTP service that exposes the Crypt agent to the web frontend.
+
+Endpoints:
+  GET  /health          -> readiness, whether an LLM key is present, corpus size
+  POST /search {query}  -> hybrid retrieval results with coordinates (no key)
+  POST /ask {question}  -> grounded, cited answer + the cited places (needs key)
+
+The retriever (and its models) load once at startup and are shared across
+requests. Run with: ``uvicorn crypt_agent.api:app --port 8088``.
+"""
+
+from __future__ import annotations
+
+from functools import lru_cache
+
+from fastapi import FastAPI, HTTPException
+from fastapi.middleware.cors import CORSMiddleware
+from pydantic import BaseModel
+
+from .agent import CryptAgent
+from .config import load_settings
+from .corpus import Place
+from .llm import LLM, LLMUnavailable
+from .retrieval import Retriever
+
+app = FastAPI(title="Crypt Agent", version="0.1.0")
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],  # dev: the Next.js frontend is a separate origin
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+
+@lru_cache(maxsize=1)
+def _retriever() -> Retriever:
+    return Retriever()
+
+
+@lru_cache(maxsize=1)
+def _agent() -> CryptAgent:
+    return CryptAgent(retriever=_retriever(), llm=LLM())
+
+
+def _place_dto(place: Place) -> dict:
+    return {
+        "id": place.id,
+        "name": place.name,
+        "city": place.city,
+        "state": place.state,
+        "structure_type": place.structure_type,
+        "lat": place.lat,
+        "lng": place.lng,
+        "description": place.description,
+    }
+
+
+class SearchRequest(BaseModel):
+    query: str
+    state: str | None = None
+    structure_type: str | None = None
+    k: int = 6
+
+
+class AskRequest(BaseModel):
+    question: str
+    max_steps: int = 5
+
+
+@app.get("/health")
+def health() -> dict:
+    settings = load_settings()
+    active_model = settings.model if settings.provider == "anthropic" else settings.ollama_model
+    return {
+        "ok": True,
+        "provider": settings.provider,
+        "has_llm": settings.has_llm,
+        "model": active_model,
+        "corpus_size": len(_retriever().places),
+    }
+
+
+@app.post("/search")
+def search(req: SearchRequest) -> dict:
+    filters: dict = {}
+    if req.state:
+        filters["state"] = req.state
+    if req.structure_type:
+        filters["structure_type"] = req.structure_type
+    hits = _retriever().search(req.query, k=req.k, filters=filters or None)
+    return {
+        "results": [
+            {**_place_dto(h.place), "rerank_score": h.rerank_score} for h in hits
+        ]
+    }
+
+
+@app.post("/ask")
+def ask(req: AskRequest) -> dict:
+    try:
+        result = _agent().answer(req.question, max_steps=req.max_steps)
+    except LLMUnavailable as exc:
+        raise HTTPException(status_code=503, detail=str(exc))
+    places = _retriever().places
+    cited = [_place_dto(places[i]) for i in result.cited_ids if 0 <= i < len(places)]
+    return {
+        "question": result.question,
+        "answer": result.answer,
+        "grounded": result.grounded,
+        "steps": result.steps,
+        "cited": cited,
+    }

agent/crypt_agent/config.py

@@ -29,24 +29,40 @@
 
 @dataclass(frozen=True)
 class Settings:
-    """Resolved settings for a single run."""
+    """Resolved settings for a single run.
 
+    ``provider`` is "anthropic" when an API key is present, otherwise "ollama"
+    (a free local model). Both back the same agent + judge code.
+    """
+
+    provider: str  # "anthropic" | "ollama"
     anthropic_api_key: str | None
-    model: str
+    model: str          # Anthropic model id
+    ollama_host: str
+    ollama_model: str
     embed_model: str
     rerank_model: str
     max_tokens: int
 
     @property
     def has_llm(self) -> bool:
-        return bool(self.anthropic_api_key)
+        if self.provider == "anthropic":
+            return bool(self.anthropic_api_key)
+        return True  # ollama path; availability is checked at call time
 
 
 def load_settings() -> Settings:
+    key = os.environ.get("ANTHROPIC_API_KEY")
+    provider = os.environ.get("CRYPT_LLM_PROVIDER", "auto")
+    if provider == "auto":
+        provider = "anthropic" if key else "ollama"
     return Settings(
-        anthropic_api_key=os.environ.get("ANTHROPIC_API_KEY"),
-        # Sonnet is the price/quality sweet spot for an agent loop + judge.
+        provider=provider,
+        anthropic_api_key=key,
+        # Sonnet is the price/quality sweet spot for the hosted path.
         model=os.environ.get("CRYPT_AGENT_MODEL", "claude-sonnet-4-6"),
+        ollama_host=os.environ.get("OLLAMA_HOST", "http://localhost:11434"),
+        ollama_model=os.environ.get("CRYPT_OLLAMA_MODEL", "llama3.2:3b"),
         embed_model=os.environ.get("CRYPT_EMBED_MODEL", "BAAI/bge-small-en-v1.5"),
         rerank_model=os.environ.get(
             "CRYPT_RERANK_MODEL", "cross-encoder/ms-marco-MiniLM-L-6-v2"

agent/crypt_agent/llm.py

@@ -50,6 +50,11 @@ def message(
         tools: list[dict] | None = None,
         max_tokens: int | None = None,
     ) -> LLMResponse:
+        if self.settings.provider == "ollama":
+            return self._ollama_message(messages, system, max_tokens)
+        return self._anthropic_message(messages, system, tools, max_tokens)
+
+    def _anthropic_message(self, messages, system, tools, max_tokens) -> LLMResponse:
         kwargs: dict = {
             "model": self.settings.model,
             "max_tokens": max_tokens or self.settings.max_tokens,
@@ -74,3 +79,29 @@ def message(
             stop_reason=resp.stop_reason,
             raw=resp,
         )
+
+    def _ollama_message(self, messages, system, max_tokens) -> LLMResponse:
+        try:
+            import ollama
+        except ImportError as exc:  # pragma: no cover
+            raise LLMUnavailable("the 'ollama' package is not installed") from exc
+        msgs = ([{"role": "system", "content": system}] if system else []) + list(messages)
+        try:
+            resp = ollama.Client(host=self.settings.ollama_host).chat(
+                model=self.settings.ollama_model,
+                messages=msgs,
+                options={
+                    "temperature": 0.2,
+                    "num_predict": max_tokens or self.settings.max_tokens,
+                },
+            )
+        except Exception as exc:
+            raise LLMUnavailable(
+                f"could not reach local Ollama model '{self.settings.ollama_model}'. "
+                f"Is 'ollama serve' running and the model pulled? ({exc})"
+            ) from exc
+        msg = resp["message"] if isinstance(resp, dict) else resp.message
+        content = msg["content"] if isinstance(msg, dict) else msg.content
+        return LLMResponse(
+            text=(content or "").strip(), tool_calls=[], stop_reason="stop", raw=resp
+        )

agent/pyproject.toml

@@ -11,6 +11,10 @@ dependencies = [
     "numpy>=1.24",
     "hnswlib>=0.8",
     "kagglehub>=0.3",
+    "fastapi>=0.110",
+    "uvicorn>=0.27",
+    "pydantic>=2.0",
+    "ollama>=0.3",
 ]
 
 [project.optional-dependencies]

agent/tests/test_agent.py

@@ -28,6 +28,8 @@ class FakeLLM:
     def __init__(self, responses):
         self.responses = list(responses)
         self.calls = 0
+        # The agent reads provider to pick the tool-loop vs the RAG path.
+        self.settings = SimpleNamespace(provider="anthropic")
 
     def message(self, messages, system=None, tools=None, max_tokens=None):
         resp = self.responses[self.calls]
@@ -66,6 +68,19 @@ def test_agent_runs_tools_then_answers_grounded():
     assert result.tool_calls[0]["name"] == "search_places"
 
 
+def test_agent_rag_path_for_local_provider():
+    # A non-anthropic provider uses single-shot retrieve-then-answer.
+    llm = FakeLLM([_final("The most haunted hospital is Waverly Hills [0].")])
+    llm.settings = SimpleNamespace(provider="ollama")
+    agent = CryptAgent(retriever=FakeRetriever(), llm=llm)
+    result = agent.answer("what is the most haunted hospital?")
+    assert result.cited_ids == [0]
+    assert result.seen_ids == [0]
+    assert result.grounded is True
+    assert result.steps == 1
+    assert llm.calls == 1  # one shot, no tool loop
+
+
 def test_agent_flags_ungrounded_citation():
     # The model cites [1] but never retrieved it -> grounded must be False.
     llm = FakeLLM([

deploy/Dockerfile.web

@@ -20,5 +20,8 @@ COPY --from=builder /app/.next ./.next
 COPY --from=builder /app/node_modules ./node_modules
 COPY --from=builder /app/package.json ./package.json
 COPY --from=builder /app/next.config.mjs ./next.config.mjs
+# Static assets in public/ (logo, etc.) must be present for `next start` to
+# serve them; without this, /brand.png and friends 404.
+COPY --from=builder /app/public ./public
 EXPOSE 3000
 CMD ["npm", "start"]