feat: Add limit processor

Limit text or tool output by approximate token count to manage content size.
Increases version to 0.2.2.

Author: iovdin

README.md

@@ -41,6 +41,7 @@ Basic toolset for [Tune](https://github.com/iovdin/tune).
   - [slice](#slice) take lines from <start> to <finish> of a file
   - [random](#random) random selection, sampling, shuffling, uniform ranges
   - [curry](#curry) change a tool by setting a parameter
+  - [limit](#limit) limit output size by approximate token count
 
 
 ## Setup
@@ -843,6 +844,31 @@ Notes:
 - choices and sample output multiple lines (one item per line).
 
 
+### `limit`
+Limit text or tool output by approximate token count.
+
+Useful when including large files or command output into a prompt.
+The processor estimates tokens as roughly `content.length / 4`.
+
+Arguments:
+- `tokens` max token limit, default `10000`
+- `hit` what to do when the limit is exceeded:
+  - `hard_err` throw an error (default)
+  - `soft_err` return a short message instead of the content
+  - `cut` truncate the content and append a warning
+
+`tokens` also supports shorthand values like `2k`, `1.5k`, `3m`.
+Invalid, non-finite, or non-positive values fall back to `10000`.
+
+```chat
+system:
+@{ README.md | limit tokens=2k }
+
+@{ sh | limit tokens=1500 hit=soft_err }
+@{ sh | limit tokens=1.5k hit=cut }
+@{| proc sh tree | limit tokens=800 }
+```
+
 ### `curry`
 Modify a tool by setting parameter or name or description. 
 Narrow possible usage of a tool so that LLM wont mess up

package.json

@@ -1,6 +1,6 @@
 {
   "name": "tune-basic-toolset",
-  "version": "0.2.1",
+  "version": "0.2.2",
   "description": "Basic toolset for tune",
   "main": "src/index.js",
   "files": [

src/limit.proc.js

@@ -0,0 +1,81 @@
+const { parseArgs } = require('./utils.js')
+
+
+const parseMaxTokens = (value) => {
+  if (value == null || value === '') {
+    return 10000
+  }
+  if (typeof value === 'number') {
+    return Number.isFinite(value) && value > 0 ? Math.floor(value) : 10000
+  }
+  if (typeof value === 'string') {
+    const normalized = value.trim().toLowerCase()
+    const match = normalized.match(/^([+-]?\d+(?:\.\d+)?)\s*([kmb])?$/)
+    if (!match) {
+      return 10000
+    }
+
+    const num = Number(match[1])
+    if (!Number.isFinite(num) || num <= 0) {
+      return 10000
+    }
+
+    const multipliers = {
+      k: 1e3,
+      m: 1e6,
+      b: 1e9,
+    }
+    const multiplier = multipliers[match[2]] || 1
+    const parsed = Math.floor(num * multiplier)
+    return Number.isFinite(parsed) && parsed > 0 ? parsed : 10000
+  }
+  return 10000
+}
+
+module.exports = async function limit(node, args) {
+  /*
+   @{ sh | limit tokens=2000 hit=hard_err|soft_err|cut }
+   @{ filename | limit tokens=2k }
+   @{ filename | limit tokens=1.5K }
+   @{| proc sh tree | limit }
+  */
+
+  if (!node) {
+    return
+  }
+
+  const params = parseArgs(args)
+
+  const maxTokens = parseMaxTokens(params.tokens)
+
+  const handleContent = (content) => {
+    if (!content || (content.length / 4) <= maxTokens) {
+      return content
+    }
+    switch (params.hit) {
+      case "cut":
+        //TODO: binary
+        return content.slice(0, maxTokens * 4) + `\n warning: the rest of the content is cut because it hit max token limit ${maxTokens} `
+      case "soft_err":
+        return `Content is too big to be shown, context limit ${maxTokens} tokens`
+      default: // hard_err
+        throw Error(`Content is too big to be shown, context limit ${maxTokens} tokens`)
+    }
+  }
+
+  if (node.type === "text") {
+    return {
+      ...node,
+      read: async () => handleContent(await node.read())
+    }
+  }
+
+  if (node.type === "tool") {
+    return {
+      ...node,
+      exec: async (params, ctx) => handleContent(await node.exec.call(ctx, params, ctx))
+    }
+  }
+
+  throw Error(`limit processor can only handle 'text' and 'tool' nodes, got '${node.type}'`)
+}