feat(open_ai): image content parts, response_format and token limits

drivers/open_ai/gpt.cr

@@ -16,7 +16,7 @@ class OpenAI::GPT < PlaceOS::Driver
     openai_org = setting?(String, :openai_org)
 
     transport.before_request do |request|
-      logger.debug { "requesting #{request.method} #{request.path}?#{request.query}\n#{request.headers}\n#{request.body}" }
+      logger.debug { "requesting #{request.method} #{request.path}?#{request.query}\n#{request.headers}\n#{redact_blobs(request.body)}" }
 
       request.headers["Authorization"] = "Bearer #{openai_key}"
       request.headers["OpenAI-Organization"] = openai_org if openai_org
@@ -36,6 +36,14 @@ class OpenAI::GPT < PlaceOS::Driver
   getter prompt_tokens : Int64 = 0
   getter completion_tokens : Int64 = 0
 
+  # any long run of base64 / hex characters, i.e. an encoded image or file
+  BINARY_BLOB = /[A-Za-z0-9+\/=_-]{256,}/
+
+  # encoded blobs are large and of no use in the logs
+  protected def redact_blobs(body) : String
+    body.to_s.gsub(BINARY_BLOB, "<binary data redacted>")
+  end
+
   protected def check(response)
     raise "unexpected response #{response.status_code}\n#{response.body}" unless response.success?
     response
@@ -75,9 +83,19 @@ class OpenAI::GPT < PlaceOS::Driver
   end
 
   # creates a completion for the chat message
-  def chat(model : String, message : Message | Array(Message))
+  #
+  # message content is either a string or a list of parts, images being
+  # base64 encoded: `[{"type": "image_url", "image": "iVBORw0...", "media_type": "image/png"}]`
+  def chat(
+    model : String,
+    message : Message | Array(Message),
+    response_format : JSON::Any? = nil,
+    max_completion_tokens : Int32? = nil,
+  )
     messages = message.is_a?(Array) ? message : [message]
     chat = CreateChatCompletion.new(model, messages)
+    chat.response_format = response_format
+    chat.max_completion_tokens = max_completion_tokens
     response = check post("/v1/chat/completions", body: chat.to_json)
     chat = ChatCompletion.from_json response.body
     update_token chat.usage

drivers/open_ai/gpt_spec.cr

@@ -2,4 +2,73 @@ require "placeos-driver/spec"
 require "./models/*"
 
 DriverSpecs.mock_driver "OpenAI::GPT" do
+  # long enough that the driver redacts it from the debug logs
+  image = "aVZCT1J3MEs" * 30
+
+  it "sends text and base64 image content parts" do
+    message = OpenAI::Message.new(
+      :user,
+      [
+        OpenAI::TextContent.new("what is in this image?"),
+        OpenAI::ImageContent.new(image, "image/png"),
+      ] of OpenAI::Content
+    )
+
+    resp = exec(:chat, model: "gpt-5.1", message: message, response_format: {type: "json_object"}, max_completion_tokens: 500)
+
+    expect_http_request do |request, response|
+      body = JSON.parse(request.body.not_nil!.gets_to_end)
+      body["max_completion_tokens"].should eq 500
+      body["response_format"]["type"].should eq "json_object"
+
+      content = body["messages"][0]["content"]
+      content[0]["type"].should eq "text"
+      content[0]["text"].should eq "what is in this image?"
+      content[1]["type"].should eq "image_url"
+      content[1]["image_url"]["url"].should eq "data:image/png;base64,#{image}"
+
+      response.status_code = 200
+      response << {
+        id:      "chatcmpl-123",
+        object:  "chat.completion",
+        created: 1_677_652_288,
+        choices: [{
+          index:         0,
+          message:       {role: "assistant", content: "a cat"},
+          finish_reason: "stop",
+        }],
+        usage: {prompt_tokens: 9, completion_tokens: 12, total_tokens: 21},
+      }.to_json
+    end
+
+    choices = Array(OpenAI::MessageChoice).from_json resp.get.not_nil!.to_json
+    choices.first.message.text.should eq "a cat"
+    status[:usage]["total_tokens"].should eq 21
+  end
+
+  it "omits the optional fields when not provided" do
+    resp = exec(:chat, model: "gpt-5.1", message: OpenAI::Message.new(:user, "hello"))
+
+    expect_http_request do |request, response|
+      body = JSON.parse(request.body.not_nil!.gets_to_end)
+      body["messages"][0]["content"].should eq "hello"
+      body.as_h.has_key?("response_format").should be_false
+      body.as_h.has_key?("max_completion_tokens").should be_false
+
+      response.status_code = 200
+      response << {
+        id:      "chatcmpl-124",
+        object:  "chat.completion",
+        created: 1_677_652_288,
+        choices: [{
+          index:         0,
+          message:       {role: "assistant", content: "hi"},
+          finish_reason: "stop",
+        }],
+        usage: {prompt_tokens: 1, completion_tokens: 1, total_tokens: 2},
+      }.to_json
+    end
+
+    resp.get.should_not be_nil
+  end
 end

drivers/open_ai/models/chat_completion.cr

@@ -11,16 +11,83 @@ module OpenAI
     Assistant
   end
 
+  struct TextContent
+    include JSON::Serializable
+
+    def initialize(@text : String)
+    end
+
+    getter type : String = "text"
+    getter text : String
+  end
+
+  # Images are provided as base64 encoded data, they are sent to OpenAI
+  # inline as a data URI. Both forms parse, so this round trips its own output.
+  struct ImageContent
+    def initialize(@image : String, @media_type : String = "image/jpeg", @detail : String? = nil)
+    end
+
+    getter type : String = "image_url"
+
+    # base64 encoded image data
+    getter image : String
+    getter media_type : String = "image/jpeg"
+
+    # one of "auto", "low" or "high"
+    getter detail : String? = nil
+
+    def self.new(pull : JSON::PullParser)
+      json = JSON::Any.new(pull)
+      detail = json.dig?("image_url", "detail") || json["detail"]?
+
+      if url = json.dig?("image_url", "url").try(&.as_s)
+        media_type, _, image = url.lchop("data:").partition(";base64,")
+      else
+        image = json["image"]?.try(&.as_s)
+        media_type = json["media_type"]?.try(&.as_s) || "image/jpeg"
+      end
+
+      raise JSON::ParseException.new("expected base64 encoded image data", 0, 0) if image.nil? || image.empty?
+      new(image, media_type, detail.try(&.as_s))
+    end
+
+    def to_json(json : JSON::Builder) : Nil
+      json.object do
+        json.field "type", "image_url"
+        json.field "image_url" do
+          json.object do
+            json.field "url", "data:#{media_type};base64,#{image}"
+            if detail = @detail
+              json.field "detail", detail
+            end
+          end
+        end
+      end
+    end
+  end
+
+  alias Content = TextContent | ImageContent
+
   # Typically, a conversation is formatted with a system message first,
   # followed by alternating user and assistant messages.
   struct Message
     include JSON::Serializable
 
-    def initialize(@role : Role, @content : String)
+    def initialize(@role : Role, @content : String | Array(Content))
     end
 
     getter role : Role
-    getter content : String
+
+    # either plain text or a list of text / image parts
+    getter content : String | Array(Content)
+
+    # the text of the message, ignoring any images
+    def text : String
+      case content = @content
+      in String         then content
+      in Array(Content) then String.build { |io| content.each { |part| io << part.text if part.is_a?(TextContent) } }
+      end
+    end
   end
 
   # POST https://api.openai.com/v1/chat/completions
@@ -73,6 +140,13 @@ module OpenAI
 
     # A unique identifier representing your end-user, which can help OpenAI to monitor and detect abuse.
     property user : String? = nil
+
+    # i.e. `{"type": "json_object"}` or a `{"type": "json_schema", "json_schema": {...}}` spec
+    property response_format : JSON::Any? = nil
+
+    # An upper bound for the number of tokens that can be generated,
+    # including both visible output and reasoning tokens.
+    property max_completion_tokens : Int32? = nil
   end
 
   struct MessageChoice