diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 0000000..c3ac5ef --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,43 @@ +name: Docs + +on: + push: + branches: [main] + workflow_dispatch: + +permissions: + contents: read + pages: write + id-token: write + +concurrency: + group: pages + cancel-in-progress: false + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + + - uses: actions/setup-python@v7 + with: + python-version: "3.12" + + - run: pip install -e ".[docs]" + + - run: pdoc --output-directory site --docformat google logquill + + - uses: actions/upload-pages-artifact@v3 + with: + path: site + + deploy: + needs: build + runs-on: ubuntu-latest + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - id: deployment + uses: actions/deploy-pages@v4 diff --git a/CHANGELOG.md b/CHANGELOG.md index 79a789c..c629c4f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,20 @@ All notable changes to this project are documented in this file. ## Unreleased +- Phase 9, docs, complete: + - Every public class and function across the package now has a docstring + explaining what it does and why you'd reach for it — not a restatement + of its name — matching the bar already set by the existing public API. + - A full API reference, generated straight from those docstrings with + [pdoc](https://pdoc.dev) (`pip install logquill[docs]`), is published to + GitHub Pages and rebuilt automatically on every push to `main` via + `.github/workflows/docs.yml`. + - README: a new "Kubernetes" section explains why `ConsoleTransport` + (stdout/stderr, captured by the node's log agent) belongs in a + container instead of `FileTransport` (writes to an ephemeral + filesystem nothing aggregates), and how to avoid losing queued records + to `SIGTERM` when `async_dispatch=True` is combined with a container's + termination grace period. - Phase 8, CLI, complete: - `logquill tail [--level=] [--json] [-f/--follow] [-n/--lines]` — a `logquill` console-script for tailing a JSONL log file in local dev. diff --git a/README.md b/README.md index c0e1765..a5f0d43 100644 --- a/README.md +++ b/README.md @@ -781,6 +781,33 @@ the same decorator under a name that reads naturally at each platform's own handler definition — the flush-before-return behavior is identical across all three. +### Kubernetes + +Default to `ConsoleTransport`, not `FileTransport`, for anything running in a +container. A container's filesystem is ephemeral and invisible to the rest of +the cluster — a log file written inside it disappears the moment the pod is +rescheduled, and nothing aggregates it in the meantime unless you also run a +sidecar to tail it back out. Writing to stdout/stderr instead costs nothing +extra: every major container runtime already captures both streams, and the +node-level log agent your cluster runs (Fluentd, Fluent Bit, Vector, or your +cloud provider's own) ships them to your aggregator without any code on your +side that needs to know that agent exists: + +```python +from logquill import ConsoleTransport, Logger + +logger = Logger("app", transports=[ConsoleTransport()]) +``` + +If you do reach for `async_dispatch=True` in a container, make sure +`logger.close()` (or `logger.flush()`) runs before the container actually +stops — Kubernetes sends `SIGTERM` and then kills the process after +`terminationGracePeriodSeconds` (30s by default) regardless of whether it's +finished shutting down, so a queued-but-undispatched record can be lost if +nothing catches the signal. A `SIGTERM` handler or `preStop` hook that calls +`logger.close(timeout=...)` closes that gap the same way `with_lambda` closes +it for a serverless freeze. + ## Context propagation, exception capture & the stdlib bridge `bind_context()` binds request-scoped values for a `with` block — every @@ -881,6 +908,19 @@ colors) when writing to a terminal; pass `--no-color` to disable that, or that isn't valid JSON, or isn't a JSON object, is skipped with a warning on stderr rather than aborting the whole tail. +## API reference + +Every public class and function is documented with a docstring; the full +reference, generated from those docstrings with [pdoc](https://pdoc.dev), is +published at +[nikhilvdev.github.io/logquill-python](https://nikhilvdev.github.io/logquill-python/) +and rebuilt on every push to `main`. To build it locally: + +```bash +pip install -e ".[docs]" +pdoc --docformat google logquill # opens a local server; add -o DIR to write static HTML instead +``` + ## Development ```bash diff --git a/logquill/adapters/autogen.py b/logquill/adapters/autogen.py index 87f9800..66a64c1 100644 --- a/logquill/adapters/autogen.py +++ b/logquill/adapters/autogen.py @@ -68,6 +68,9 @@ class AutoGenAdapter(LogQuillAdapter, logging.Handler): """ def __init__(self, agent_log: Logger) -> None: + """Attaches this adapter as a `logging.Handler` on AutoGen's + `EVENT_LOGGER_NAME` logger immediately — active as soon as it's + constructed, no separate "start" call needed.""" LogQuillAdapter.__init__(self, agent_log) logging.Handler.__init__(self) self._event_logger = logging.getLogger(EVENT_LOGGER_NAME) @@ -78,10 +81,16 @@ def __init__(self, agent_log: Logger) -> None: self._event_logger.addHandler(self) def close(self) -> None: + """Detach this handler from AutoGen's event logger, then run the + base `logging.Handler.close()`.""" self._event_logger.removeHandler(self) super().close() def emit(self, record: logging.LogRecord) -> None: + """Unpack an `autogen_core` structured event object off `record.msg` + and forward it as the matching `.action()`/`.observation()`/ + `.error()` call; silently ignored if `record.msg` isn't one of + those structured event objects (e.g. an ordinary log line).""" kwargs = getattr(record.msg, "kwargs", None) if not isinstance(kwargs, dict): return # not one of autogen_core.logging's structured event objects diff --git a/logquill/adapters/base.py b/logquill/adapters/base.py index 6299241..8c365cb 100644 --- a/logquill/adapters/base.py +++ b/logquill/adapters/base.py @@ -18,4 +18,6 @@ class LogQuillAdapter: """ def __init__(self, agent_log: Logger) -> None: + """`agent_log` is the `Logger` (typically `.child(...)` with a + `RunPlugin` attached) every translated event is forwarded onto.""" self.log = agent_log diff --git a/logquill/adapters/crewai.py b/logquill/adapters/crewai.py index 033a0f5..edb5d96 100644 --- a/logquill/adapters/crewai.py +++ b/logquill/adapters/crewai.py @@ -92,6 +92,9 @@ class CrewAIAdapter(LogQuillAdapter, BaseEventListener): # type: ignore[misc] """ def __init__(self, agent_log: Logger) -> None: + """Registers this listener's handlers on CrewAI's event bus + immediately (`BaseEventListener.__init__`'s own behavior) — keep a + reference alive for as long as you want it active.""" LogQuillAdapter.__init__(self, agent_log) self._open_spans: dict[str, SpanContext] = {} self._call_starts: dict[str, datetime] = {} @@ -132,6 +135,9 @@ def _step_end(self, name: str, event: Any, *, error: str | None = None) -> None: # `@`-applying an `Any`-typed decorator even though the wrapped method # itself is fully annotated. A plain call sidesteps that check. def setup_listeners(self, crewai_event_bus: Any) -> None: + """`BaseEventListener`'s required override: subscribes every CrewAI + event this adapter translates (crew/task/agent/tool/LLM + start/end/error) to its matching handler on `crewai_event_bus`.""" crewai_event_bus.on(CrewKickoffStartedEvent)(self._on_crew_started) crewai_event_bus.on(CrewKickoffCompletedEvent)(self._on_crew_completed) crewai_event_bus.on(CrewKickoffFailedEvent)(self._on_crew_failed) diff --git a/logquill/adapters/langchain.py b/logquill/adapters/langchain.py index 0e1fba2..0c0d55b 100644 --- a/logquill/adapters/langchain.py +++ b/logquill/adapters/langchain.py @@ -69,6 +69,8 @@ class LangChainAdapter(LogQuillAdapter, BaseCallbackHandler): # type: ignore[mi """ def __init__(self, agent_log: Logger) -> None: + """Pass the resulting instance into a chain/agent's `callbacks=[...]` + — no separate registration step needed.""" LogQuillAdapter.__init__(self, agent_log) BaseCallbackHandler.__init__(self) self._open_spans: dict[UUID, SpanContext] = {} @@ -91,6 +93,8 @@ def on_chain_start( parent_run_id: UUID | None = None, **kwargs: Any, ) -> Any: + """Opens a `span()` for this chain run, keyed by `run_id` so the + matching `on_chain_end`/`on_chain_error` can close it.""" name = _tool_name(serialized, "chain") span = self.log.span( name, @@ -108,6 +112,7 @@ def on_chain_end( parent_run_id: UUID | None = None, **kwargs: Any, ) -> Any: + """Closes the `span()` opened by the matching `on_chain_start`.""" span = self._open_spans.pop(run_id, None) if span is not None: span.__exit__(None, None, None) @@ -120,6 +125,9 @@ def on_chain_error( parent_run_id: UUID | None = None, **kwargs: Any, ) -> Any: + """Closes the `span()` opened by the matching `on_chain_start`, + propagating `error` into it so the span's own record captures the + failure (at `ERROR`, with `meta.error` set).""" span = self._open_spans.pop(run_id, None) if span is not None: span.__exit__(type(error), error, error.__traceback__) @@ -135,6 +143,8 @@ def on_llm_start( parent_run_id: UUID | None = None, **kwargs: Any, ) -> Any: + """Records the call's start time (for the matching `on_llm_end`'s + `duration_ms`) and emits `.action("llm_start")`.""" self._call_starts[run_id] = time.monotonic() self.log.action("llm_start", **_span_ids(run_id, parent_run_id)) @@ -146,6 +156,8 @@ def on_llm_end( parent_run_id: UUID | None = None, **kwargs: Any, ) -> Any: + """Emits `.observation("llm_end")` with `duration_ms` measured since + the matching `on_llm_start`.""" duration_ms = self._duration_ms(run_id) meta = _span_ids(run_id, parent_run_id) if duration_ms is not None: @@ -160,6 +172,9 @@ def on_llm_error( parent_run_id: UUID | None = None, **kwargs: Any, ) -> Any: + """Emits `.error("llm_error")`; discards the timing recorded by + `on_llm_start` without using it, since a failed call has no + meaningful `duration_ms` to report here.""" self._duration_ms(run_id) self.log.error("llm_error", error=str(error), **_span_ids(run_id, parent_run_id)) @@ -181,6 +196,11 @@ def on_agent_action( parent_run_id: UUID | None = None, **kwargs: Any, ) -> Any: + """Emits `.action(tool)` for the agent's chosen action; leaves + `span_id`/`parent_span_id` unset since `run_id` here is the + enclosing chain's own id, not a fresh one — the ambient-span + auto-stamp in `Logger._log` supplies the correct + `parent_span_id` instead.""" tool = getattr(action, "tool", "agent_action") self.log.action(tool) @@ -192,6 +212,8 @@ def on_agent_finish( parent_run_id: UUID | None = None, **kwargs: Any, ) -> Any: + """Emits `.decision("agent_finish")` for the agent's concluding + decision.""" self.log.decision("agent_finish") # -- tools: action (start) / observation (end) / error ------------------ @@ -205,6 +227,8 @@ def on_tool_start( parent_run_id: UUID | None = None, **kwargs: Any, ) -> Any: + """Records the call's start time (for the matching `on_tool_end`'s + `duration_ms`) and emits `.action(name)` for the tool call.""" name = _tool_name(serialized, "tool") self._call_starts[run_id] = time.monotonic() self.log.action(name, **_span_ids(run_id, parent_run_id)) @@ -217,6 +241,8 @@ def on_tool_end( parent_run_id: UUID | None = None, **kwargs: Any, ) -> Any: + """Emits `.observation("tool_end")` with `duration_ms` measured + since the matching `on_tool_start`.""" duration_ms = self._duration_ms(run_id) meta = _span_ids(run_id, parent_run_id) if duration_ms is not None: @@ -231,5 +257,8 @@ def on_tool_error( parent_run_id: UUID | None = None, **kwargs: Any, ) -> Any: + """Emits `.error("tool_error")`; discards the timing recorded by + `on_tool_start` without using it, since a failed call has no + meaningful `duration_ms` to report here.""" self._duration_ms(run_id) self.log.error("tool_error", error=str(error), **_span_ids(run_id, parent_run_id)) diff --git a/logquill/adapters/langgraph.py b/logquill/adapters/langgraph.py index dbc4f6c..3ee2e17 100644 --- a/logquill/adapters/langgraph.py +++ b/logquill/adapters/langgraph.py @@ -64,9 +64,14 @@ class exists for that reason alone; everything else is inherited """ def on_interrupt(self, event: Any) -> None: + """Emits `.observation("graph_interrupted")` when the graph pauses on + an `interrupt()` call, carrying each pending interrupt's `id`/`value` + plus the checkpoint fields.""" meta = _checkpoint_meta(event) meta["interrupts"] = [{"id": i.id, "value": i.value} for i in event.interrupts] self.log.observation("graph_interrupted", **meta) def on_resume(self, event: Any) -> None: + """Emits `.action("graph_resumed")` when the graph resumes from a + persisted checkpoint.""" self.log.action("graph_resumed", **_checkpoint_meta(event)) diff --git a/logquill/adapters/llamaindex.py b/logquill/adapters/llamaindex.py index d5556b9..3758f28 100644 --- a/logquill/adapters/llamaindex.py +++ b/logquill/adapters/llamaindex.py @@ -59,6 +59,9 @@ def prepare_to_exit_span( result: Any = None, **kwargs: Any, ) -> Any: + """Logs a successfully-completed span (`.info()`, with `duration_ms` + and `parent_span_id` if nested) after delegating to + `SimpleSpanHandler` for the actual span bookkeeping.""" span = super().prepare_to_exit_span(id_, bound_args, instance, result, **kwargs) self._log_close(id_, span) return span @@ -71,6 +74,9 @@ def prepare_to_drop_span( err: BaseException | None = None, **kwargs: Any, ) -> Any: + """Logs a span that exited via exception (`.error()`, with `err`'s + message) after delegating to `SimpleSpanHandler` for the actual span + bookkeeping.""" span = super().prepare_to_drop_span(id_, bound_args, instance, err, **kwargs) self._log_close(id_, span, error=err) return span @@ -101,6 +107,10 @@ class _EventLogger(BaseEventHandler): # type: ignore[misc] log: Logger def handle(self, event: Any, **kwargs: Any) -> Any: + """`BaseEventHandler`'s required override: classifies `event` by its + `class_name()` suffix (`*StartEvent`/`*EndEvent`/`*ErrorEvent`) and + forwards it as the matching `.action()`/`.observation()`/`.error()` + call; a few noisy progress/delta event types are skipped entirely.""" name = event.class_name() if name.endswith(_SKIPPED_SUFFIXES): return None @@ -149,6 +159,9 @@ class LlamaIndexAdapter(LogQuillAdapter): """ def __init__(self, agent_log: Logger) -> None: + """Registers a span handler and an event handler on LlamaIndex's + global instrumentation dispatcher immediately — active as soon as + it's constructed, no separate "start" call needed.""" super().__init__(agent_log) self._span_handler = _SpanLogger(log=agent_log) self._event_handler = _EventLogger(log=agent_log) diff --git a/logquill/cli.py b/logquill/cli.py index 05319d4..a3b04a2 100644 --- a/logquill/cli.py +++ b/logquill/cli.py @@ -24,6 +24,9 @@ def build_parser() -> argparse.ArgumentParser: + """Builds the `logquill` CLI's argument parser (currently just the + `tail` subcommand); split out from `main()` so tests can inspect/exercise + it without going through `sys.argv`.""" parser = argparse.ArgumentParser( prog="logquill", description="LogQuill command-line tools for local development." ) @@ -221,6 +224,9 @@ def _run_tail( def main(argv: Sequence[str] | None = None) -> int: + """The `logquill` console-script entry point: parses `argv` (defaulting + to `sys.argv`) and dispatches to the matching subcommand, returning the + process exit code.""" parser = build_parser() args = parser.parse_args(argv) diff --git a/logquill/formatter.py b/logquill/formatter.py index a1c7756..0731be6 100644 --- a/logquill/formatter.py +++ b/logquill/formatter.py @@ -9,11 +9,15 @@ class Formatter(Protocol): """`format(record) -> string`, per the transport contract shared with logquill-js.""" - def format(self, record: LogRecord) -> str: ... + def format(self, record: LogRecord) -> str: + """Render `record` to the string a transport will write.""" + ... class JSONFormatter: """Serializes a record to the canonical JSON line shape.""" def format(self, record: LogRecord) -> str: + """Serializes `record` to a single compact JSON line; non-JSON-native + values fall back to `str()` rather than raising.""" return json.dumps(record, separators=(",", ":"), default=str) diff --git a/logquill/handler.py b/logquill/handler.py index a5115d7..3d29796 100644 --- a/logquill/handler.py +++ b/logquill/handler.py @@ -82,10 +82,18 @@ class LogQuillHandler(logging.Handler): """ def __init__(self, logger: Logger, level: int = logging.NOTSET) -> None: + """`logger` is the LogQuill `Logger` every bridged stdlib record is + forwarded onto; `level` is this handler's own stdlib-level filter, + applied on top of `logger`'s own level.""" super().__init__(level) self._logger = logger def emit(self, record: logging.LogRecord) -> None: + """Translates a stdlib `logging.LogRecord` into a LogQuill call: + maps its level, folds any `extra=` fields into `meta`, formats + `exc_info` if present, and routes it through the wrapped `Logger`'s + own `_log` path. Delegates to `self.handleError()` (never raises) + if translation itself fails.""" try: meta: dict[str, Any] = { key: value diff --git a/logquill/logger.py b/logquill/logger.py index 7fd7917..02c933b 100644 --- a/logquill/logger.py +++ b/logquill/logger.py @@ -18,6 +18,14 @@ class Logger: + """A named, leveled logger that runs records through a plugin pipeline + before writing them to one or more transports. + + Construct directly, or via `.child()` to derive a namespaced logger that + shares this one's transports. See `__init__` for what `async_dispatch` + changes about ordering. + """ + def __init__( self, name: str, @@ -54,9 +62,12 @@ def __init__( @property def level(self) -> Level: + """The minimum level this logger currently accepts.""" return self._level def set_level(self, level: int | str | Level) -> None: + """Change the minimum level this logger accepts; accepts an int, a + level name, or a `Level` member.""" self._level = parse_level(level) def use(self, plugin: Plugin | MiddlewareFunc) -> Logger: @@ -200,15 +211,23 @@ def _log(self, level: Level, message: str, meta: dict[str, Any]) -> LogRecord | # caller — exactly what the plugin pipeline's hypothesis tests assert # never happens (see `tests/test_plugin_pipeline_properties.py`). def trace(self, message: str, /, **meta: Any) -> LogRecord | None: + """Log at `TRACE`. Returns the emitted record, or `None` if filtered + by level or dropped by a plugin.""" return self._log(Level.TRACE, message, meta) def debug(self, message: str, /, **meta: Any) -> LogRecord | None: + """Log at `DEBUG`. Returns the emitted record, or `None` if filtered + by level or dropped by a plugin.""" return self._log(Level.DEBUG, message, meta) def info(self, message: str, /, **meta: Any) -> LogRecord | None: + """Log at `INFO`. Returns the emitted record, or `None` if filtered + by level or dropped by a plugin.""" return self._log(Level.INFO, message, meta) def warn(self, message: str, /, **meta: Any) -> LogRecord | None: + """Log at `WARN`. Returns the emitted record, or `None` if filtered + by level or dropped by a plugin.""" return self._log(Level.WARN, message, meta) def error(self, message: str, /, **meta: Any) -> LogRecord | None: @@ -221,6 +240,8 @@ def error(self, message: str, /, **meta: Any) -> LogRecord | None: return self._log(Level.ERROR, message, meta) def fatal(self, message: str, /, **meta: Any) -> LogRecord | None: + """Log at `FATAL`. Returns the emitted record, or `None` if filtered + by level or dropped by a plugin.""" return self._log(Level.FATAL, message, meta) def thought(self, message: str, /, **meta: Any) -> LogRecord | None: diff --git a/logquill/plugins/alerting_plugin.py b/logquill/plugins/alerting_plugin.py index 4a189a1..107ba5b 100644 --- a/logquill/plugins/alerting_plugin.py +++ b/logquill/plugins/alerting_plugin.py @@ -10,9 +10,15 @@ class _Window: + """Tracks one open dedupe window: the first matching record, how many + matches have arrived since, and the timer that will flush it.""" + __slots__ = ("record", "count", "timer") def __init__(self, record: LogRecord, timer: threading.Timer) -> None: + """`record` is the first record seen for this dedupe key; `timer` + is the pending flush that will send a follow-up alert if `count` + ends up greater than 1 by the time it fires.""" self.record = record self.count = 1 self.timer = timer @@ -59,6 +65,12 @@ def __init__( dedupe_key: Callable[[LogRecord], str] | None = None, max_tracked_keys: int = 500, ) -> None: + """`dedupe_key` defaults to level+logger+message; pass a custom + function to group differently (e.g. by `meta["trace_id"]`). + `max_tracked_keys` bounds how many distinct dedupe windows are open + at once — beyond that, a new key is dropped from tracking rather + than alerted on, so alerting degrades under extreme cardinality + instead of growing memory without bound.""" self.threshold = parse_level(threshold) self.dedupe_window_seconds = dedupe_window_seconds self._dedupe_key = dedupe_key or self._default_dedupe_key @@ -71,6 +83,10 @@ def _default_dedupe_key(record: LogRecord) -> str: return f"{record['level']}:{record['logger']}:{record['message']}" def after_log(self, record: LogRecord) -> None: + """Fires `send_alert` on a new background thread for the first + record at or above `threshold` under a given dedupe key, and starts + that key's dedupe window; any further match within the window just + increments its count instead of sending again.""" if Level[record["level"]] < self.threshold: return diff --git a/logquill/plugins/context_plugin.py b/logquill/plugins/context_plugin.py index ee82923..5e21fc1 100644 --- a/logquill/plugins/context_plugin.py +++ b/logquill/plugins/context_plugin.py @@ -13,8 +13,12 @@ class ContextPlugin(Plugin): """ def __init__(self, **context: Any) -> None: + """`context` is the fixed set of key/value pairs injected into every + record this plugin sees.""" self.context = context def before_log(self, record: LogRecord) -> LogRecord | None: + """Merges the fixed `context` under the record's own `meta`, so any + key already present in `meta` is left untouched.""" record["meta"] = {**self.context, **record["meta"]} return record diff --git a/logquill/plugins/email_alert_plugin.py b/logquill/plugins/email_alert_plugin.py index 8b081a4..aff6787 100644 --- a/logquill/plugins/email_alert_plugin.py +++ b/logquill/plugins/email_alert_plugin.py @@ -29,6 +29,8 @@ def __init__( timeout: float = 10.0, **kwargs: Any, ) -> None: + """`kwargs` are forwarded to `AlertingPlugin.__init__` (`threshold`, + `dedupe_window_seconds`, etc.).""" super().__init__(**kwargs) self.smtp_host = smtp_host self.smtp_port = smtp_port @@ -40,6 +42,9 @@ def __init__( self.timeout = timeout def send_alert(self, record: LogRecord, occurrences: int) -> None: + """Sends one plaintext email summarizing `record`, opening a fresh + SMTP connection per alert (never raises to the caller — see + `AlertingPlugin`'s `_safe_send` wrapper).""" message = EmailMessage() subject = f"[{record['level']}] {record['logger']}" if occurrences > 1: diff --git a/logquill/plugins/pagerduty_alert_plugin.py b/logquill/plugins/pagerduty_alert_plugin.py index 7d2884f..5d249e7 100644 --- a/logquill/plugins/pagerduty_alert_plugin.py +++ b/logquill/plugins/pagerduty_alert_plugin.py @@ -20,11 +20,16 @@ class PagerDutyAlertPlugin(AlertingPlugin): """ def __init__(self, routing_key: str, *, timeout: float = 5.0, **kwargs: Any) -> None: + """`kwargs` are forwarded to `AlertingPlugin.__init__` (`threshold`, + `dedupe_window_seconds`, etc.).""" super().__init__(**kwargs) self.routing_key = routing_key self.timeout = timeout def send_alert(self, record: LogRecord, occurrences: int) -> None: + """POSTs one `trigger` event to PagerDuty's Events API v2; raises if + the API responds with an error status (caught by `AlertingPlugin`'s + `_safe_send` wrapper, so this never crashes the caller).""" summary = f"{record['logger']}: {record['message']}" if occurrences > 1: summary += f" (x{occurrences})" diff --git a/logquill/plugins/pii_redact_plugin.py b/logquill/plugins/pii_redact_plugin.py index 2b18625..cb9b259 100644 --- a/logquill/plugins/pii_redact_plugin.py +++ b/logquill/plugins/pii_redact_plugin.py @@ -59,6 +59,11 @@ def __init__( presidio_entities: Sequence[str] | None = None, presidio_language: str = "en", ) -> None: + """`patterns` overrides/extends `DEFAULT_PII_PATTERNS` entirely (not + merged) when given. `presidio_entities`/`presidio_language` are only + used when `use_presidio=True`; loading Presidio happens here, at + construction time, so a missing optional dependency fails fast + rather than on the first log call.""" self.patterns: dict[str, re.Pattern[str]] = ( dict(patterns) if patterns is not None else dict(DEFAULT_PII_PATTERNS) ) @@ -85,6 +90,8 @@ def _load_presidio() -> tuple[Any, Any]: return AnalyzerEngine(), AnonymizerEngine() def before_log(self, record: LogRecord) -> LogRecord | None: + """Recursively redacts PII-shaped substrings anywhere in `meta`'s + values, regardless of which key holds them.""" record["meta"] = self._redact_value(record["meta"], set(), 0) return record diff --git a/logquill/plugins/plugin.py b/logquill/plugins/plugin.py index a344e72..a2f52a0 100644 --- a/logquill/plugins/plugin.py +++ b/logquill/plugins/plugin.py @@ -37,7 +37,10 @@ class FunctionPlugin(Plugin): """ def __init__(self, func: MiddlewareFunc) -> None: + """`func` is the plain `before_log`-style function this instance + delegates to.""" self._func = func def before_log(self, record: LogRecord) -> LogRecord | None: + """Delegates to the wrapped function.""" return self._func(record) diff --git a/logquill/plugins/rate_limit_plugin.py b/logquill/plugins/rate_limit_plugin.py index 53eef02..f084cb7 100644 --- a/logquill/plugins/rate_limit_plugin.py +++ b/logquill/plugins/rate_limit_plugin.py @@ -43,6 +43,9 @@ def __init__( max_keys: int = 1000, clock: Callable[[], float] = time.monotonic, ) -> None: + """`clock` is injectable for deterministic testing of window + rollover; defaults to `time.monotonic` so wall-clock adjustments + can't shrink or extend a window.""" if max_records < 1: raise ValueError(f"max_records must be at least 1, got {max_records!r}") if per_seconds <= 0: @@ -56,6 +59,9 @@ def __init__( self._windows: OrderedDict[Hashable, tuple[float, int]] = OrderedDict() def before_log(self, record: LogRecord) -> LogRecord | None: + """Drops `record` if its key's current window has already reached + `max_records`; starts a fresh window for a key whose previous + window has expired.""" key = self.key_func(record) now = self._clock() window = self._windows.get(key) diff --git a/logquill/plugins/redact_plugin.py b/logquill/plugins/redact_plugin.py index 108f018..238f97c 100644 --- a/logquill/plugins/redact_plugin.py +++ b/logquill/plugins/redact_plugin.py @@ -16,10 +16,14 @@ def __init__( keys: Iterable[str] = DEFAULT_REDACTED_KEYS, replacement: str = "***", ) -> None: + """`keys` defaults to `DEFAULT_REDACTED_KEYS`; matching is + case-insensitive, so callers don't need to worry about casing.""" self.keys = {key.lower() for key in keys} self.replacement = replacement def before_log(self, record: LogRecord) -> LogRecord | None: + """Replaces the value of any `meta` key matching `keys` + (case-insensitively) with `replacement`.""" meta = record["meta"] record["meta"] = { key: self.replacement if key.lower() in self.keys else value diff --git a/logquill/plugins/run_plugin.py b/logquill/plugins/run_plugin.py index 280b0c6..d6b7e1f 100644 --- a/logquill/plugins/run_plugin.py +++ b/logquill/plugins/run_plugin.py @@ -27,10 +27,14 @@ class RunPlugin(Plugin): """ def __init__(self, run_id: str | None = None) -> None: + """`run_id` defaults to a freshly generated UUID hex string; pass an + explicit one to adopt an id handed in from elsewhere.""" self.run_id = run_id or uuid.uuid4().hex self._step = 0 def before_log(self, record: LogRecord) -> LogRecord | None: + """Stamps `meta.run_id` (if not already set) and this instance's + current step counter, then increments the counter.""" meta = record["meta"] meta.setdefault("run_id", self.run_id) meta["step"] = self._step diff --git a/logquill/plugins/sampling_plugin.py b/logquill/plugins/sampling_plugin.py index 0195e3f..95427b1 100644 --- a/logquill/plugins/sampling_plugin.py +++ b/logquill/plugins/sampling_plugin.py @@ -52,6 +52,13 @@ def __init__( max_buffered_records: int = 1000, max_traces: int = 200, ) -> None: + """`rng` is injectable for deterministic testing of the sample + decision; defaults to `random.random`. `transports` opts into + tail-based elevation — see the class docstring — and must be the + same transport list given to the `Logger` for buffered records to + actually reach the intended sinks. `max_buffered_records`/ + `max_traces` bound the tail-buffer's memory; the oldest trace is + evicted (unflushed) once either is exceeded.""" if not 0.0 <= rate <= 1.0: raise ValueError(f"rate must be between 0 and 1, got {rate!r}") self.rate = rate @@ -66,6 +73,10 @@ def __init__( self._elevated: OrderedDict[object, None] = OrderedDict() def before_log(self, record: LogRecord) -> LogRecord | None: + """Keeps `record` per the sample rate, buffers it under its trace id + if dropped and tail-based elevation is active, or — if this or an + earlier record for the same trace reached `elevate_at` — flushes the + whole buffered trace and lets `record` through unconditionally.""" transports = self.transports trace_id = record["meta"].get(self.trace_key) if transports is not None else None diff --git a/logquill/plugins/slack_alert_plugin.py b/logquill/plugins/slack_alert_plugin.py index 157c68a..2e0b916 100644 --- a/logquill/plugins/slack_alert_plugin.py +++ b/logquill/plugins/slack_alert_plugin.py @@ -16,11 +16,17 @@ class SlackAlertPlugin(AlertingPlugin): """ def __init__(self, webhook_url: str, *, timeout: float = 5.0, **kwargs: Any) -> None: + """`kwargs` are forwarded to `AlertingPlugin.__init__` (`threshold`, + `dedupe_window_seconds`, etc.).""" super().__init__(**kwargs) self.webhook_url = webhook_url self.timeout = timeout def send_alert(self, record: LogRecord, occurrences: int) -> None: + """POSTs a plain-text summary of `record` to the Slack webhook; + raises if Slack responds with an error status (caught by + `AlertingPlugin`'s `_safe_send` wrapper, so this never crashes the + caller).""" body = json.dumps({"text": _format_message(record, occurrences)}).encode("utf-8") request = urllib.request.Request( self.webhook_url, diff --git a/logquill/plugins/tamper_evident_plugin.py b/logquill/plugins/tamper_evident_plugin.py index 010706b..b08dcc7 100644 --- a/logquill/plugins/tamper_evident_plugin.py +++ b/logquill/plugins/tamper_evident_plugin.py @@ -30,10 +30,15 @@ class TamperEvidentPlugin(Plugin): """ def __init__(self, *, genesis_hash: str = GENESIS_HASH) -> None: + """`genesis_hash` is the `prev_hash` used for the very first record + in the chain; override it to start a new chain that continues from + a previously-recorded hash (e.g. across a log rotation).""" self._genesis_hash = genesis_hash self._last_hash = genesis_hash def before_log(self, record: LogRecord) -> LogRecord | None: + """Stamps `meta.prev_hash`/`meta.hash`, chaining this record onto + the previous one this plugin instance processed.""" prev_hash = self._last_hash digest = _compute_hash(record, prev_hash) record["meta"] = {**record["meta"], "prev_hash": prev_hash, "hash": digest} diff --git a/logquill/plugins/trace_context_plugin.py b/logquill/plugins/trace_context_plugin.py index b55cec8..91e0064 100644 --- a/logquill/plugins/trace_context_plugin.py +++ b/logquill/plugins/trace_context_plugin.py @@ -32,6 +32,8 @@ def set_traceparent(value: str | None) -> Token[str | None]: def reset_traceparent(token: Token[str | None]) -> None: + """Restore the trace header context to what it was before the matching + `set_traceparent` call, using the token that call returned.""" _current_traceparent.reset(token) @@ -90,10 +92,15 @@ class TraceContextPlugin(Plugin): """ def __init__(self, *, trace_key: str = "trace_id", traceparent: str | None = None) -> None: + """`traceparent`, if given, takes priority over whatever + `set_traceparent()` set for the current context — see the class + docstring's resolution order.""" self.trace_key = trace_key self._explicit_traceparent = traceparent def before_log(self, record: LogRecord) -> LogRecord | None: + """Stamps `meta[trace_key]` if not already set, resolving a trace id + per the class docstring's priority order.""" meta = record["meta"] if meta.get(self.trace_key) is not None: return record diff --git a/logquill/records.py b/logquill/records.py index 7cd3b36..227fa90 100644 --- a/logquill/records.py +++ b/logquill/records.py @@ -23,6 +23,9 @@ def utc_timestamp() -> str: def create_record(*, level: Level, logger: str, message: str, meta: dict[str, Any]) -> LogRecord: + """Build a `LogRecord` with the current UTC timestamp and the level's + string name (not its numeric value, per the cross-language record + shape).""" return LogRecord( timestamp=utc_timestamp(), level=level.name, diff --git a/logquill/serverless.py b/logquill/serverless.py index 8f5ad44..e4eb1c1 100644 --- a/logquill/serverless.py +++ b/logquill/serverless.py @@ -48,10 +48,15 @@ def with_lambda(loggers: LoggerOrLoggers, *, timeout: float | None = 5.0) -> Cal targets = _as_loggers(loggers) def decorator(func: F) -> F: + """Wraps `func`, choosing the async or sync flush path based on + whether `func` itself is a coroutine function.""" if inspect.iscoroutinefunction(func): @functools.wraps(func) async def async_wrapper(*args: Any, **kwargs: Any) -> Any: + """Awaits `func`, then `flush_async`es every target logger + in a `finally` block so a raised exception still ships its + logs.""" try: return await func(*args, **kwargs) finally: @@ -62,6 +67,8 @@ async def async_wrapper(*args: Any, **kwargs: Any) -> Any: @functools.wraps(func) def wrapper(*args: Any, **kwargs: Any) -> Any: + """Calls `func`, then flushes every target logger in a `finally` + block so a raised exception still ships its logs.""" try: return func(*args, **kwargs) finally: diff --git a/logquill/span.py b/logquill/span.py index ba3a1f4..3beb44a 100644 --- a/logquill/span.py +++ b/logquill/span.py @@ -57,6 +57,10 @@ def __init__( parent_span_id: str | None = None, **meta: Any, ) -> None: + """`span_id` defaults to a freshly generated id; `parent_span_id` + overrides the auto-nesting that would otherwise come from any + enclosing span active in this execution context — see the class + docstring for why a caller would pass either explicitly.""" self._logger = logger self._name = name self._meta = meta @@ -66,6 +70,8 @@ def __init__( self._start = 0.0 def __enter__(self) -> SpanContext: + """Push this span's id as the current span for this execution + context and start its duration timer.""" self._token = _current_span_id.set(self._span_id) self._start = time.monotonic() return self @@ -76,6 +82,10 @@ def __exit__( exc: BaseException | None, tb: TracebackType | None, ) -> None: + """Pop this span off the current execution context and emit its + record — at `ERROR` with `meta.error` set if the block raised, + `INFO` otherwise. Does not suppress the exception; it still + propagates after this returns.""" duration_ms = (time.monotonic() - self._start) * 1000 assert self._token is not None _current_span_id.reset(self._token) diff --git a/logquill/transports/batching_transport.py b/logquill/transports/batching_transport.py index ef060b9..541586c 100644 --- a/logquill/transports/batching_transport.py +++ b/logquill/transports/batching_transport.py @@ -33,6 +33,9 @@ def __init__( max_records: int = 100, max_bytes: int = 1_000_000, ) -> None: + """`max_records`/`max_bytes` are the two independent triggers for an + automatic flush — whichever is hit first, checked after every + `write()`.""" super().__init__(formatter) self.max_records = max_records self.max_bytes = max_bytes @@ -46,6 +49,8 @@ def _size_of(self, item: T) -> int: return len(json.dumps(item, separators=(",", ":"), default=str).encode("utf-8")) def write(self, formatted: str, record: LogRecord) -> None: + """Buffers `record` (converted via `_to_item`) and triggers a + `flush()` as soon as either `max_records` or `max_bytes` is reached.""" item = self._to_item(formatted, record) self._buffer.append(item) self._buffer_bytes += self._size_of(item) @@ -53,6 +58,10 @@ def write(self, formatted: str, record: LogRecord) -> None: self.flush() def flush(self) -> None: + """Sends whatever is currently buffered via `_send_batch`, clearing + the buffer first so a failing send doesn't retry the same batch on + the next flush; a send failure is caught and logged, never raised + to the caller. No-op if nothing is buffered.""" if not self._buffer: return batch, self._buffer = self._buffer, [] @@ -63,7 +72,12 @@ def flush(self) -> None: _logger.exception("%s: failed to send log batch", type(self).__name__) def close(self) -> None: + """Flushes any remaining buffered records.""" self.flush() @abstractmethod - def _send_batch(self, batch: Sequence[T]) -> None: ... + def _send_batch(self, batch: Sequence[T]) -> None: + """Send one batch of buffered items to the concrete sink. Called + only with a non-empty `batch`; a raised exception is caught by + `flush()` and logged, not propagated.""" + ... diff --git a/logquill/transports/cloud/app_insights_transport.py b/logquill/transports/cloud/app_insights_transport.py index 639e482..9fd0b6a 100644 --- a/logquill/transports/cloud/app_insights_transport.py +++ b/logquill/transports/cloud/app_insights_transport.py @@ -47,6 +47,9 @@ def __init__( max_bytes: int = 1_000_000, sender: Sender | None = None, ) -> None: + """`instrumentation_key` is the Application Insights resource's + instrumentation key. `sender` defaults to a stdlib `urllib`-based + POST; override for a fake in tests.""" super().__init__(formatter=formatter, max_records=max_records, max_bytes=max_bytes) self.instrumentation_key = instrumentation_key self._sender: Sender = sender or _urllib_sender diff --git a/logquill/transports/cloud/cloud_logging_transport.py b/logquill/transports/cloud/cloud_logging_transport.py index 983e06d..ca841a1 100644 --- a/logquill/transports/cloud/cloud_logging_transport.py +++ b/logquill/transports/cloud/cloud_logging_transport.py @@ -17,7 +17,13 @@ class CloudLoggingClientLike(Protocol): - def log_struct(self, info: dict[str, Any], severity: str) -> None: ... + """The subset of `google-cloud-logging`'s `Logger` client this transport + calls — implement this shape to inject a fake in tests without + installing the real driver.""" + + def log_struct(self, info: dict[str, Any], severity: str) -> None: + """Write one structured log entry at the given severity.""" + ... class CloudLoggingTransport(BatchingTransport[LogRecord]): @@ -38,6 +44,8 @@ def __init__( max_records: int = 100, max_bytes: int = 1_000_000, ) -> None: + """`log_name` names the Cloud Logging log this transport writes to + when it connects its own client (ignored if `client` is given).""" super().__init__(formatter=formatter, max_records=max_records, max_bytes=max_bytes) self._log_name = log_name self._injected = client diff --git a/logquill/transports/cloud/cloudwatch_transport.py b/logquill/transports/cloud/cloudwatch_transport.py index 0aec29a..ecb311d 100644 --- a/logquill/transports/cloud/cloudwatch_transport.py +++ b/logquill/transports/cloud/cloudwatch_transport.py @@ -8,12 +8,18 @@ class CloudWatchClientLike(Protocol): + """The subset of `boto3`'s CloudWatch Logs client this transport calls — + implement this shape to inject a fake in tests without installing the + real driver.""" + def put_log_events( self, logGroupName: str, logStreamName: str, logEvents: Sequence[dict[str, Any]], # noqa: N803 - ) -> object: ... + ) -> object: + """Send one batch of chronologically-ordered log events.""" + ... def _to_millis(iso_timestamp: str) -> int: @@ -43,6 +49,8 @@ def __init__( max_records: int = 100, max_bytes: int = 1_000_000, ) -> None: + """`region` is the AWS region this transport connects its own + `boto3` client to (ignored if `client` is given).""" super().__init__(formatter=formatter, max_records=max_records, max_bytes=max_bytes) self.log_group = log_group self.log_stream = log_stream diff --git a/logquill/transports/cloud/datadog_transport.py b/logquill/transports/cloud/datadog_transport.py index abb70ea..170d2ec 100644 --- a/logquill/transports/cloud/datadog_transport.py +++ b/logquill/transports/cloud/datadog_transport.py @@ -46,6 +46,9 @@ def __init__( max_bytes: int = 1_000_000, sender: DatadogSender | None = None, ) -> None: + """`site` picks the region-specific intake endpoint — see the class + docstring for valid values. `sender` defaults to a stdlib + `urllib`-based POST; override for a fake in tests.""" super().__init__(formatter=formatter, max_records=max_records, max_bytes=max_bytes) self.api_key = api_key self.site = site diff --git a/logquill/transports/cloud/elasticsearch_transport.py b/logquill/transports/cloud/elasticsearch_transport.py index ef6a115..251362e 100644 --- a/logquill/transports/cloud/elasticsearch_transport.py +++ b/logquill/transports/cloud/elasticsearch_transport.py @@ -46,6 +46,8 @@ def __init__( max_bytes: int = 1_000_000, sender: ElasticsearchSender | None = None, ) -> None: + """`sender` defaults to a stdlib `urllib`-based POST; override for a + fake in tests.""" super().__init__(formatter=formatter, max_records=max_records, max_bytes=max_bytes) self.index = index self.bulk_url = f"{url.rstrip('/')}/_bulk" diff --git a/logquill/transports/cloud/new_relic_transport.py b/logquill/transports/cloud/new_relic_transport.py index ac3efc0..fa2af7e 100644 --- a/logquill/transports/cloud/new_relic_transport.py +++ b/logquill/transports/cloud/new_relic_transport.py @@ -24,6 +24,10 @@ class NewRelicSenderResult(TypedDict): + """What a `NewRelicSender` reports back about one send attempt: whether + it succeeded, the HTTP status, and (on a 429) the raw `Retry-After` + header value, if any.""" + ok: bool status: int retry_after: str | None @@ -92,6 +96,10 @@ def __init__( sender: NewRelicSender | None = None, clock: Callable[[], float] | None = None, ) -> None: + """`region` selects the US or EU ingestion endpoint. `sender` + defaults to a stdlib `urllib`-based POST; override for a fake in + tests. `clock` defaults to `time.time`; inject a fake to test the + 429 backoff window deterministically.""" super().__init__(formatter=formatter, max_records=max_records, max_bytes=max_bytes) self.license_key = license_key self.region = region diff --git a/logquill/transports/cloud/syslog_transport.py b/logquill/transports/cloud/syslog_transport.py index 32928f6..ee9f756 100644 --- a/logquill/transports/cloud/syslog_transport.py +++ b/logquill/transports/cloud/syslog_transport.py @@ -57,6 +57,9 @@ def __init__( formatter: Formatter | None = None, sender: SyslogSender | None = None, ) -> None: + """`protocol` must be `"udp"` or `"tcp"`; `sender` defaults to + sending over a real socket, opened lazily on first use — inject a + fake for tests.""" super().__init__(formatter) if protocol not in ("udp", "tcp"): raise ValueError(f"SyslogTransport: protocol must be 'udp' or 'tcp', got {protocol!r}") @@ -89,6 +92,9 @@ def _send_via_socket(self, data: bytes) -> None: self._sock.sendall(data) def write(self, formatted: str, record: LogRecord) -> None: + """Frames `formatted` as one RFC 5424 message and sends it + immediately (newline-terminated for TCP, as a single datagram for + UDP) — never batched, unlike the HTTP-API transports.""" severity = _SEVERITY_BY_LEVEL.get(parse_level(record["level"]), 6) pri = self.facility * 8 + severity app_name = self.app_name or record["logger"] @@ -102,6 +108,7 @@ def write(self, formatted: str, record: LogRecord) -> None: self._resolved_sender()(data) def close(self) -> None: + """Closes the underlying socket, if one was opened.""" if self._sock is not None: self._sock.close() self._sock = None diff --git a/logquill/transports/console_transport.py b/logquill/transports/console_transport.py index 7d69691..3bc75be 100644 --- a/logquill/transports/console_transport.py +++ b/logquill/transports/console_transport.py @@ -30,12 +30,17 @@ def __init__( stdout: TextIO | None = None, stderr: TextIO | None = None, ) -> None: + """`stdout`/`stderr` are injectable (defaulting to `sys.stdout`/ + `sys.stderr`) so tests can capture output without redirecting the + real streams.""" super().__init__(formatter) self.colorize = colorize self._stdout: TextIO = stdout if stdout is not None else sys.stdout self._stderr: TextIO = stderr if stderr is not None else sys.stderr def write(self, formatted: str, record: LogRecord) -> None: + """Writes `formatted` to stdout, or stderr for `ERROR`/`FATAL`, + colorized by level when `colorize` is set.""" level = parse_level(record["level"]) stream = self._stderr if level >= Level.ERROR else self._stdout line = self._colorize(formatted, level) if self.colorize else formatted diff --git a/logquill/transports/file_transport.py b/logquill/transports/file_transport.py index 36fab22..a326980 100644 --- a/logquill/transports/file_transport.py +++ b/logquill/transports/file_transport.py @@ -50,6 +50,12 @@ def __init__( backup_count: int = 5, encrypt_key: bytes | str | None = None, ) -> None: + """`max_bytes` triggers rotation once the file reaches that size + (0 disables rotation); `backup_count` bounds how many rotated files + are kept (`path.1`, `path.2`, ...) — the oldest is deleted once that + many already exist, or, if `backup_count` is 0, the file is simply + truncated on rotation rather than kept. `encrypt_key` opts into + per-line Fernet encryption — see the class docstring.""" super().__init__(formatter) self.path = Path(path) self.max_bytes = max_bytes @@ -64,6 +70,9 @@ def _open(self) -> TextIO | BinaryIO: return self.path.open("a", encoding="utf-8") def write(self, formatted: str, record: LogRecord) -> None: + """Appends `formatted` as one line (encrypted first if `encrypt_key` + was set), flushes to disk immediately, and rotates if `max_bytes` + has been reached.""" if self._fernet is not None: cast(BinaryIO, self._file).write( self._fernet.encrypt(formatted.encode("utf-8")) + b"\n" @@ -91,4 +100,5 @@ def _rotate(self) -> None: self._file = self._open() def close(self) -> None: + """Closes the underlying file handle.""" self._file.close() diff --git a/logquill/transports/http_transport.py b/logquill/transports/http_transport.py index f6574bd..015e427 100644 --- a/logquill/transports/http_transport.py +++ b/logquill/transports/http_transport.py @@ -38,6 +38,8 @@ def __init__( batch_size: int = 50, sender: Sender | None = None, ) -> None: + """`sender` defaults to a stdlib `urllib`-based POST; override for a + fake in tests or an alternate HTTP backend.""" super().__init__(formatter) self.url = url self.batch_size = batch_size @@ -45,15 +47,20 @@ def __init__( self._batch: list[str] = [] def write(self, formatted: str, record: LogRecord) -> None: + """Buffers `formatted` and triggers a `flush()` once `batch_size` is + reached.""" self._batch.append(formatted) if len(self._batch) >= self.batch_size: self.flush() def flush(self) -> None: + """Sends whatever is currently buffered via `sender`, clearing the + buffer first. No-op if nothing is buffered.""" if not self._batch: return batch, self._batch = self._batch, [] self._sender(self.url, batch) def close(self) -> None: + """Flushes any remaining buffered records.""" self.flush() diff --git a/logquill/transports/nosql/dynamodb_transport.py b/logquill/transports/nosql/dynamodb_transport.py index 205319c..bb2e42a 100644 --- a/logquill/transports/nosql/dynamodb_transport.py +++ b/logquill/transports/nosql/dynamodb_transport.py @@ -8,11 +8,24 @@ class DynamoBatchWriterLike(Protocol): - def put_item(self, Item: dict[str, Any]) -> object: ... # noqa: N803 — matches boto3's kwarg + """The subset of `boto3`'s DynamoDB batch-writer context this transport + calls — implement this shape to inject a fake in tests without + installing the real driver.""" + + def put_item(self, Item: dict[str, Any]) -> object: # noqa: N803 — matches boto3's kwarg + """Queue one item for the batch write.""" + ... class DynamoTableLike(Protocol): - def batch_writer(self) -> ContextManager[DynamoBatchWriterLike]: ... + """The subset of `boto3`'s DynamoDB `Table` resource this transport + calls — implement this shape to inject a fake in tests without + installing the real driver.""" + + def batch_writer(self) -> ContextManager[DynamoBatchWriterLike]: + """Open a batch-writer context that auto-chunks and auto-retries + queued items.""" + ... def _partition_key(record: LogRecord) -> str: @@ -64,6 +77,8 @@ def __init__( max_records: int = 100, max_bytes: int = 1_000_000, ) -> None: + """`table_name`/`region` are used only when this transport connects + its own `boto3` resource (ignored if `table` is given).""" super().__init__(formatter=formatter, max_records=max_records, max_bytes=max_bytes) self._injected = table self._table_name = table_name diff --git a/logquill/transports/nosql/mongodb_transport.py b/logquill/transports/nosql/mongodb_transport.py index cbaf956..b2a56db 100644 --- a/logquill/transports/nosql/mongodb_transport.py +++ b/logquill/transports/nosql/mongodb_transport.py @@ -8,11 +8,23 @@ class MongoCollectionLike(Protocol): - def insert_many(self, documents: Sequence[dict[str, Any]]) -> object: ... + """The subset of `pymongo`'s `Collection` this transport calls — + implement this shape to inject a fake in tests without installing the + real driver.""" + + def insert_many(self, documents: Sequence[dict[str, Any]]) -> object: + """Insert a batch of documents in one call.""" + ... class MongoClientLike(Protocol): - def close(self) -> None: ... + """The subset of `pymongo`'s `MongoClient` this transport calls to + release its connection on shutdown — implement this shape to inject a + fake in tests without installing the real driver.""" + + def close(self) -> None: + """Release the client's connection resources.""" + ... class MongoDBTransport(BatchingTransport[LogRecord]): @@ -34,6 +46,9 @@ def __init__( max_records: int = 100, max_bytes: int = 1_000_000, ) -> None: + """`uri`/`database`/`collection_name` are used only when this + transport connects its own `pymongo` client (ignored if + `collection` is given).""" super().__init__(formatter=formatter, max_records=max_records, max_bytes=max_bytes) self._injected = collection self._uri = uri @@ -62,6 +77,9 @@ def _import_collection(self) -> MongoCollectionLike: return cast(MongoCollectionLike, client[self._database][self._collection_name]) def close(self) -> None: + """Flushes any remaining buffered records, then closes the + self-connected client — never a client passed in as `collection`, + since this transport doesn't own that connection's lifecycle.""" super().close() if self._injected is None and self._client is not None: self._client.close() diff --git a/logquill/transports/nosql/redis_transport.py b/logquill/transports/nosql/redis_transport.py index 14c7650..e1efd75 100644 --- a/logquill/transports/nosql/redis_transport.py +++ b/logquill/transports/nosql/redis_transport.py @@ -9,8 +9,17 @@ class RedisClientLike(Protocol): - def xadd(self, name: str, fields: dict[str, Any]) -> object: ... - def close(self) -> None: ... + """The subset of `redis-py`'s `Redis` client this transport calls — + implement this shape to inject a fake in tests without installing the + real driver.""" + + def xadd(self, name: str, fields: dict[str, Any]) -> object: + """Append one entry to a Redis Stream.""" + ... + + def close(self) -> None: + """Release the client's connection resources.""" + ... class RedisTransport(BatchingTransport[LogRecord]): @@ -32,6 +41,8 @@ def __init__( max_records: int = 100, max_bytes: int = 1_000_000, ) -> None: + """`url` is used only when this transport connects its own `redis` + client (ignored if `client` is given).""" super().__init__(formatter=formatter, max_records=max_records, max_bytes=max_bytes) self._injected = client self._stream = stream @@ -57,6 +68,9 @@ def _import_client(self) -> RedisClientLike: return client def close(self) -> None: + """Flushes any remaining buffered records, then closes the + self-connected client — never a client passed in as `client`, since + this transport doesn't own that connection's lifecycle.""" super().close() if self._injected is None and self._client is not None: self._client.close() diff --git a/logquill/transports/queue/base_queue_transport.py b/logquill/transports/queue/base_queue_transport.py index 0441e7c..1bb400f 100644 --- a/logquill/transports/queue/base_queue_transport.py +++ b/logquill/transports/queue/base_queue_transport.py @@ -24,6 +24,9 @@ def __init__( max_records: int = 100, max_bytes: int = 1_000_000, ) -> None: + """`topic` names the queue/topic this transport publishes to — + interpreted per concrete backend (a Kafka topic name, an SQS queue + URL, a fully-qualified Pub/Sub topic path, ...).""" super().__init__(formatter=formatter, max_records=max_records, max_bytes=max_bytes) self.topic = topic diff --git a/logquill/transports/queue/kafka_transport.py b/logquill/transports/queue/kafka_transport.py index e50a0c0..e570795 100644 --- a/logquill/transports/queue/kafka_transport.py +++ b/logquill/transports/queue/kafka_transport.py @@ -9,9 +9,21 @@ class KafkaProducerLike(Protocol): - def send(self, topic: str, value: bytes, key: bytes | None = None) -> object: ... - def flush(self) -> None: ... - def close(self) -> None: ... + """The subset of `kafka-python`'s `KafkaProducer` this transport calls — + implement this shape to inject a fake in tests without installing the + real driver.""" + + def send(self, topic: str, value: bytes, key: bytes | None = None) -> object: + """Publish one message, optionally keyed for partition affinity.""" + ... + + def flush(self) -> None: + """Block until all previously sent messages are acknowledged.""" + ... + + def close(self) -> None: + """Release the producer's connection resources.""" + ... def _partition_key(record: LogRecord) -> bytes | None: @@ -43,6 +55,8 @@ def __init__( max_records: int = 100, max_bytes: int = 1_000_000, ) -> None: + """`bootstrap_servers` is used only when this transport connects its + own `KafkaProducer` (ignored if `producer` is given).""" super().__init__( topic=topic, formatter=formatter, max_records=max_records, max_bytes=max_bytes ) @@ -71,6 +85,9 @@ def _import_producer(self) -> KafkaProducerLike: ) def close(self) -> None: + """Flushes any remaining buffered records, then closes the + self-connected producer — never a producer passed in as `producer`, + since this transport doesn't own that connection's lifecycle.""" super().close() if self._injected is None and self._producer is not None: self._producer.close() diff --git a/logquill/transports/queue/pubsub_transport.py b/logquill/transports/queue/pubsub_transport.py index 4dc365c..5ca92e5 100644 --- a/logquill/transports/queue/pubsub_transport.py +++ b/logquill/transports/queue/pubsub_transport.py @@ -9,11 +9,24 @@ class PubSubFutureLike(Protocol): - def result(self) -> object: ... + """The subset of the future `google-cloud-pubsub`'s `publish()` returns + this transport calls — implement this shape to inject a fake in tests + without installing the real driver.""" + + def result(self) -> object: + """Block until the publish completes, raising on failure.""" + ... class PubSubTopicLike(Protocol): - def publish(self, topic: str, data: bytes) -> PubSubFutureLike: ... + """The subset of `google-cloud-pubsub`'s `PublisherClient` this + transport calls — implement this shape to inject a fake in tests + without installing the real driver.""" + + def publish(self, topic: str, data: bytes) -> PubSubFutureLike: + """Publish one message to `topic`, returning a future for its + result.""" + ... class PubSubTransport(BaseQueueTransport): @@ -34,6 +47,8 @@ def __init__( max_records: int = 100, max_bytes: int = 1_000_000, ) -> None: + """Connects its own `PublisherClient` lazily on first use unless + `client` is given.""" super().__init__( topic=topic, formatter=formatter, max_records=max_records, max_bytes=max_bytes ) diff --git a/logquill/transports/queue/rabbitmq_transport.py b/logquill/transports/queue/rabbitmq_transport.py index 83181b2..a82a902 100644 --- a/logquill/transports/queue/rabbitmq_transport.py +++ b/logquill/transports/queue/rabbitmq_transport.py @@ -9,11 +9,22 @@ class AMQPChannelLike(Protocol): - def basic_publish(self, exchange: str, routing_key: str, body: bytes) -> object: ... + """The subset of `pika`'s channel this transport calls — implement this + shape to inject a fake in tests without installing the real driver.""" + + def basic_publish(self, exchange: str, routing_key: str, body: bytes) -> object: + """Publish one message to `exchange` with the given routing key.""" + ... class AMQPConnectionLike(Protocol): - def close(self) -> None: ... + """The subset of `pika`'s connection this transport calls to release + its resources on shutdown — implement this shape to inject a fake in + tests without installing the real driver.""" + + def close(self) -> None: + """Close the AMQP connection.""" + ... class RabbitMQTransport(BaseQueueTransport): @@ -33,6 +44,8 @@ def __init__( max_records: int = 100, max_bytes: int = 1_000_000, ) -> None: + """`url` is used only when this transport connects its own `pika` + connection (ignored if `channel` is given).""" super().__init__( topic=topic, formatter=formatter, max_records=max_records, max_bytes=max_bytes ) @@ -63,6 +76,9 @@ def _import_channel(self) -> AMQPChannelLike: return cast(AMQPChannelLike, channel) def close(self) -> None: + """Flushes any remaining buffered records, then closes the + self-opened connection — never a channel passed in as `channel`, + since this transport doesn't own that connection's lifecycle.""" super().close() if self._injected is None and self._connection is not None: self._connection.close() diff --git a/logquill/transports/queue/sqs_transport.py b/logquill/transports/queue/sqs_transport.py index 2652555..0033133 100644 --- a/logquill/transports/queue/sqs_transport.py +++ b/logquill/transports/queue/sqs_transport.py @@ -11,7 +11,13 @@ class SQSClientLike(Protocol): - def send_message_batch(self, QueueUrl: str, Entries: Sequence[dict[str, str]]) -> object: ... # noqa: N803 + """The subset of `boto3`'s SQS client this transport calls — implement + this shape to inject a fake in tests without installing the real + driver.""" + + def send_message_batch(self, QueueUrl: str, Entries: Sequence[dict[str, str]]) -> object: # noqa: N803 + """Send up to 10 messages to `QueueUrl` in one call.""" + ... class SQSTransport(BaseQueueTransport): @@ -36,6 +42,8 @@ def __init__( max_records: int = 100, max_bytes: int = 1_000_000, ) -> None: + """`region` is used only when this transport connects its own + `boto3` client (ignored if `client` is given).""" super().__init__( topic=topic, formatter=formatter, max_records=max_records, max_bytes=max_bytes ) diff --git a/logquill/transports/sql/base_sql_transport.py b/logquill/transports/sql/base_sql_transport.py index 194a2a5..5dc5f5c 100644 --- a/logquill/transports/sql/base_sql_transport.py +++ b/logquill/transports/sql/base_sql_transport.py @@ -50,6 +50,10 @@ def __init__( table_name: str = "logs", ensure_schema: bool = False, ) -> None: + """`ensure_schema=True` runs `create_table_sql()`'s DDL once, before + the first batch send — a dev/test-only convenience, never enabled + by default, since schema creation in production is the caller's + responsibility (see the class docstring).""" super().__init__(formatter=formatter, max_records=max_records, max_bytes=max_bytes) self.table_name = table_name self.ensure_schema = ensure_schema diff --git a/logquill/transports/sql/mysql_transport.py b/logquill/transports/sql/mysql_transport.py index 2596006..013741d 100644 --- a/logquill/transports/sql/mysql_transport.py +++ b/logquill/transports/sql/mysql_transport.py @@ -7,13 +7,31 @@ class MySQLCursorLike(Protocol): - def execute(self, sql: str, parameters: Sequence[object] = ()) -> object: ... + """The subset of `pymysql`'s cursor this transport calls — implement + this shape to inject a fake in tests without installing the real + driver.""" + + def execute(self, sql: str, parameters: Sequence[object] = ()) -> object: + """Execute one parameterized SQL statement.""" + ... class MySQLConnectionLike(Protocol): - def cursor(self) -> MySQLCursorLike: ... - def commit(self) -> None: ... - def close(self) -> None: ... + """The subset of `pymysql`'s connection this transport calls — + implement this shape to inject a fake in tests without installing the + real driver.""" + + def cursor(self) -> MySQLCursorLike: + """Return a new cursor on this connection.""" + ... + + def commit(self) -> None: + """Commit the current transaction.""" + ... + + def close(self) -> None: + """Close the connection.""" + ... class MySQLTransport(BaseSQLTransport): @@ -38,6 +56,9 @@ def __init__( table_name: str = "logs", ensure_schema: bool = False, ) -> None: + """`host`/`port`/`user`/`password`/`database` are used only when + this transport connects its own `pymysql` connection (ignored if + `connection` is given).""" super().__init__( formatter=formatter, max_records=max_records, @@ -54,6 +75,8 @@ def __init__( self._connection: MySQLConnectionLike | None = None def create_table_sql(self) -> str: + """MySQL-dialect DDL: `AUTO_INCREMENT` primary key and a native + `JSON` column for `meta`, in place of the generic base class DDL.""" return ( f"CREATE TABLE IF NOT EXISTS {self.table_name} (" "id INT AUTO_INCREMENT PRIMARY KEY, " @@ -96,6 +119,10 @@ def _import_connection(self) -> MySQLConnectionLike: ) def close(self) -> None: + """Flushes any remaining buffered records, then closes the + self-connected connection — never a connection passed in as + `connection`, since this transport doesn't own that connection's + lifecycle.""" super().close() if self._injected is None and self._connection is not None: self._connection.close() diff --git a/logquill/transports/sql/postgres_transport.py b/logquill/transports/sql/postgres_transport.py index 7358b28..2d2efa5 100644 --- a/logquill/transports/sql/postgres_transport.py +++ b/logquill/transports/sql/postgres_transport.py @@ -7,13 +7,31 @@ class PostgresCursorLike(Protocol): - def execute(self, sql: str, parameters: Sequence[object] = ()) -> object: ... + """The subset of `psycopg2`'s cursor this transport calls — implement + this shape to inject a fake in tests without installing the real + driver.""" + + def execute(self, sql: str, parameters: Sequence[object] = ()) -> object: + """Execute one parameterized SQL statement.""" + ... class PostgresConnectionLike(Protocol): - def cursor(self) -> PostgresCursorLike: ... - def commit(self) -> None: ... - def close(self) -> None: ... + """The subset of `psycopg2`'s connection this transport calls — + implement this shape to inject a fake in tests without installing the + real driver.""" + + def cursor(self) -> PostgresCursorLike: + """Return a new cursor on this connection.""" + ... + + def commit(self) -> None: + """Commit the current transaction.""" + ... + + def close(self) -> None: + """Close the connection.""" + ... class PostgresTransport(BaseSQLTransport): @@ -34,6 +52,8 @@ def __init__( table_name: str = "logs", ensure_schema: bool = False, ) -> None: + """`dsn` is used only when this transport connects its own + `psycopg2` connection (ignored if `connection` is given).""" super().__init__( formatter=formatter, max_records=max_records, @@ -46,6 +66,8 @@ def __init__( self._connection: PostgresConnectionLike | None = None def create_table_sql(self) -> str: + """Postgres-dialect DDL: `SERIAL` primary key and a native `JSONB` + column for `meta`, in place of the generic base class DDL.""" return ( f"CREATE TABLE IF NOT EXISTS {self.table_name} (" "id SERIAL PRIMARY KEY, " @@ -80,6 +102,10 @@ def _import_connection(self) -> PostgresConnectionLike: return cast(PostgresConnectionLike, psycopg2.connect(self._dsn)) def close(self) -> None: + """Flushes any remaining buffered records, then closes the + self-connected connection — never a connection passed in as + `connection`, since this transport doesn't own that connection's + lifecycle.""" super().close() if self._injected is None and self._connection is not None: self._connection.close() diff --git a/logquill/transports/sql/sqlite_transport.py b/logquill/transports/sql/sqlite_transport.py index bea301f..4c95f0b 100644 --- a/logquill/transports/sql/sqlite_transport.py +++ b/logquill/transports/sql/sqlite_transport.py @@ -8,10 +8,24 @@ class SQLiteConnectionLike(Protocol): - def execute(self, sql: str, parameters: Sequence[object] = ()) -> object: ... - def executemany(self, sql: str, seq_of_parameters: Iterable[Sequence[object]]) -> object: ... - def commit(self) -> None: ... - def close(self) -> None: ... + """The subset of stdlib `sqlite3`'s connection this transport calls — + implement this shape to inject a fake in tests.""" + + def execute(self, sql: str, parameters: Sequence[object] = ()) -> object: + """Execute one parameterized SQL statement.""" + ... + + def executemany(self, sql: str, seq_of_parameters: Iterable[Sequence[object]]) -> object: + """Execute one SQL statement against many parameter sequences.""" + ... + + def commit(self) -> None: + """Commit the current transaction.""" + ... + + def close(self) -> None: + """Close the connection.""" + ... class SQLiteTransport(BaseSQLTransport): @@ -31,6 +45,9 @@ def __init__( table_name: str = "logs", ensure_schema: bool = False, ) -> None: + """`filename` is used only when this transport connects its own + `sqlite3` connection (ignored if `connection` is given); defaults to + an in-memory database.""" super().__init__( formatter=formatter, max_records=max_records, @@ -50,6 +67,10 @@ def _resolved_connection(self) -> SQLiteConnectionLike: return self._connection def close(self) -> None: + """Flushes any remaining buffered records, then closes the + self-connected connection — never a connection passed in as + `connection`, since this transport doesn't own that connection's + lifecycle.""" super().close() if self._injected is None and self._connection is not None: self._connection.close() diff --git a/logquill/transports/transport.py b/logquill/transports/transport.py index d47806a..b3b87f9 100644 --- a/logquill/transports/transport.py +++ b/logquill/transports/transport.py @@ -12,13 +12,21 @@ class Transport(ABC): """ def __init__(self, formatter: Formatter | None = None) -> None: + """`formatter` defaults to `JSONFormatter` — the canonical JSON line + shape shared with logquill-js.""" self.formatter: Formatter = formatter or JSONFormatter() def format(self, record: LogRecord) -> str: + """Render `record` via this transport's configured `formatter`.""" return self.formatter.format(record) @abstractmethod - def write(self, formatted: str, record: LogRecord) -> None: ... + def write(self, formatted: str, record: LogRecord) -> None: + """Send the already-formatted string (and the original `record`, for + transports that need structured fields rather than the formatted + text) to this transport's sink. Must not raise for a single bad + record — see each concrete transport for its own failure handling.""" + ... def flush(self) -> None: # noqa: B027 — intentionally optional to override """Push any internally buffered records out now, without releasing @@ -38,14 +46,18 @@ class CollectingTransport(Transport): """In-memory transport for tests: collects every (formatted, record) pair written to it.""" def __init__(self, formatter: Formatter | None = None) -> None: + """Starts with empty `formatted`/`records` lists and `closed=False`.""" super().__init__(formatter) self.formatted: list[str] = [] self.records: list[LogRecord] = [] self.closed = False def write(self, formatted: str, record: LogRecord) -> None: + """Appends `formatted` and `record` to this transport's in-memory + lists, for tests to assert against.""" self.formatted.append(formatted) self.records.append(record) def close(self) -> None: + """Marks this transport `closed`, for tests to assert shutdown ran.""" self.closed = True diff --git a/logquill/worker.py b/logquill/worker.py index 4670edc..21dc645 100644 --- a/logquill/worker.py +++ b/logquill/worker.py @@ -54,6 +54,10 @@ def __init__( max_queue_size: int = 10_000, backpressure: BackpressurePolicy = "drop_oldest", ) -> None: + """Starts the background dispatch thread immediately — a worker is + active as soon as it's constructed, no separate "start" call + needed. See the class docstring for what `backpressure` does once + `max_queue_size` is reached.""" if max_queue_size < 1: raise ValueError(f"max_queue_size must be >= 1, got {max_queue_size}") if backpressure not in _VALID_POLICIES: diff --git a/pyproject.toml b/pyproject.toml index b865f33..8b6a042 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -71,6 +71,11 @@ hooks = [ # CI matrix still tests 3.8, and pre-commit is only needed locally. "pre-commit>=3.7", ] +docs = [ + # Renders the API reference published to GitHub Pages; not needed to + # use or develop logquill itself. + "pdoc>=16", +] [project.scripts] logquill = "logquill.cli:main"