From bed47993d342d3f9edb2e1c2f7ed41efed890edb Mon Sep 17 00:00:00 2001 From: Alex Sorafumo Date: Mon, 3 Aug 2026 14:58:01 +1000 Subject: [PATCH] feat(open_ai): image content parts, response_format and token limits Message content is now either a plain string or a list of typed text / image parts. Images are provided as base64 encoded data and sent to OpenAI inline as a data URI, so drivers never hand the API a URL to fetch. ImageContent parses both forms, allowing it to round trip its own serialized output across the exec boundary. Adds the optional response_format and max_completion_tokens fields to CreateChatCompletion, exposed as arguments on chat. Request bodies are also scrubbed of long base64 / hex runs before they are logged, keeping encoded images out of the debug logs. --- drivers/open_ai/gpt.cr | 22 ++++++- drivers/open_ai/gpt_spec.cr | 69 ++++++++++++++++++++ drivers/open_ai/models/chat_completion.cr | 78 ++++++++++++++++++++++- 3 files changed, 165 insertions(+), 4 deletions(-) diff --git a/drivers/open_ai/gpt.cr b/drivers/open_ai/gpt.cr index 2a4b2f31f2a..884104e72f1 100644 --- a/drivers/open_ai/gpt.cr +++ b/drivers/open_ai/gpt.cr @@ -16,7 +16,7 @@ class OpenAI::GPT < PlaceOS::Driver openai_org = setting?(String, :openai_org) transport.before_request do |request| - logger.debug { "requesting #{request.method} #{request.path}?#{request.query}\n#{request.headers}\n#{request.body}" } + logger.debug { "requesting #{request.method} #{request.path}?#{request.query}\n#{request.headers}\n#{redact_blobs(request.body)}" } request.headers["Authorization"] = "Bearer #{openai_key}" request.headers["OpenAI-Organization"] = openai_org if openai_org @@ -36,6 +36,14 @@ class OpenAI::GPT < PlaceOS::Driver getter prompt_tokens : Int64 = 0 getter completion_tokens : Int64 = 0 + # any long run of base64 / hex characters, i.e. an encoded image or file + BINARY_BLOB = /[A-Za-z0-9+\/=_-]{256,}/ + + # encoded blobs are large and of no use in the logs + protected def redact_blobs(body) : String + body.to_s.gsub(BINARY_BLOB, "") + end + protected def check(response) raise "unexpected response #{response.status_code}\n#{response.body}" unless response.success? response @@ -75,9 +83,19 @@ class OpenAI::GPT < PlaceOS::Driver end # creates a completion for the chat message - def chat(model : String, message : Message | Array(Message)) + # + # message content is either a string or a list of parts, images being + # base64 encoded: `[{"type": "image_url", "image": "iVBORw0...", "media_type": "image/png"}]` + def chat( + model : String, + message : Message | Array(Message), + response_format : JSON::Any? = nil, + max_completion_tokens : Int32? = nil, + ) messages = message.is_a?(Array) ? message : [message] chat = CreateChatCompletion.new(model, messages) + chat.response_format = response_format + chat.max_completion_tokens = max_completion_tokens response = check post("/v1/chat/completions", body: chat.to_json) chat = ChatCompletion.from_json response.body update_token chat.usage diff --git a/drivers/open_ai/gpt_spec.cr b/drivers/open_ai/gpt_spec.cr index 46b5dd8570f..bf5418d6575 100644 --- a/drivers/open_ai/gpt_spec.cr +++ b/drivers/open_ai/gpt_spec.cr @@ -2,4 +2,73 @@ require "placeos-driver/spec" require "./models/*" DriverSpecs.mock_driver "OpenAI::GPT" do + # long enough that the driver redacts it from the debug logs + image = "aVZCT1J3MEs" * 30 + + it "sends text and base64 image content parts" do + message = OpenAI::Message.new( + :user, + [ + OpenAI::TextContent.new("what is in this image?"), + OpenAI::ImageContent.new(image, "image/png"), + ] of OpenAI::Content + ) + + resp = exec(:chat, model: "gpt-5.1", message: message, response_format: {type: "json_object"}, max_completion_tokens: 500) + + expect_http_request do |request, response| + body = JSON.parse(request.body.not_nil!.gets_to_end) + body["max_completion_tokens"].should eq 500 + body["response_format"]["type"].should eq "json_object" + + content = body["messages"][0]["content"] + content[0]["type"].should eq "text" + content[0]["text"].should eq "what is in this image?" + content[1]["type"].should eq "image_url" + content[1]["image_url"]["url"].should eq "data:image/png;base64,#{image}" + + response.status_code = 200 + response << { + id: "chatcmpl-123", + object: "chat.completion", + created: 1_677_652_288, + choices: [{ + index: 0, + message: {role: "assistant", content: "a cat"}, + finish_reason: "stop", + }], + usage: {prompt_tokens: 9, completion_tokens: 12, total_tokens: 21}, + }.to_json + end + + choices = Array(OpenAI::MessageChoice).from_json resp.get.not_nil!.to_json + choices.first.message.text.should eq "a cat" + status[:usage]["total_tokens"].should eq 21 + end + + it "omits the optional fields when not provided" do + resp = exec(:chat, model: "gpt-5.1", message: OpenAI::Message.new(:user, "hello")) + + expect_http_request do |request, response| + body = JSON.parse(request.body.not_nil!.gets_to_end) + body["messages"][0]["content"].should eq "hello" + body.as_h.has_key?("response_format").should be_false + body.as_h.has_key?("max_completion_tokens").should be_false + + response.status_code = 200 + response << { + id: "chatcmpl-124", + object: "chat.completion", + created: 1_677_652_288, + choices: [{ + index: 0, + message: {role: "assistant", content: "hi"}, + finish_reason: "stop", + }], + usage: {prompt_tokens: 1, completion_tokens: 1, total_tokens: 2}, + }.to_json + end + + resp.get.should_not be_nil + end end diff --git a/drivers/open_ai/models/chat_completion.cr b/drivers/open_ai/models/chat_completion.cr index ac35bbbb37c..6e23fdf589a 100644 --- a/drivers/open_ai/models/chat_completion.cr +++ b/drivers/open_ai/models/chat_completion.cr @@ -11,16 +11,83 @@ module OpenAI Assistant end + struct TextContent + include JSON::Serializable + + def initialize(@text : String) + end + + getter type : String = "text" + getter text : String + end + + # Images are provided as base64 encoded data, they are sent to OpenAI + # inline as a data URI. Both forms parse, so this round trips its own output. + struct ImageContent + def initialize(@image : String, @media_type : String = "image/jpeg", @detail : String? = nil) + end + + getter type : String = "image_url" + + # base64 encoded image data + getter image : String + getter media_type : String = "image/jpeg" + + # one of "auto", "low" or "high" + getter detail : String? = nil + + def self.new(pull : JSON::PullParser) + json = JSON::Any.new(pull) + detail = json.dig?("image_url", "detail") || json["detail"]? + + if url = json.dig?("image_url", "url").try(&.as_s) + media_type, _, image = url.lchop("data:").partition(";base64,") + else + image = json["image"]?.try(&.as_s) + media_type = json["media_type"]?.try(&.as_s) || "image/jpeg" + end + + raise JSON::ParseException.new("expected base64 encoded image data", 0, 0) if image.nil? || image.empty? + new(image, media_type, detail.try(&.as_s)) + end + + def to_json(json : JSON::Builder) : Nil + json.object do + json.field "type", "image_url" + json.field "image_url" do + json.object do + json.field "url", "data:#{media_type};base64,#{image}" + if detail = @detail + json.field "detail", detail + end + end + end + end + end + end + + alias Content = TextContent | ImageContent + # Typically, a conversation is formatted with a system message first, # followed by alternating user and assistant messages. struct Message include JSON::Serializable - def initialize(@role : Role, @content : String) + def initialize(@role : Role, @content : String | Array(Content)) end getter role : Role - getter content : String + + # either plain text or a list of text / image parts + getter content : String | Array(Content) + + # the text of the message, ignoring any images + def text : String + case content = @content + in String then content + in Array(Content) then String.build { |io| content.each { |part| io << part.text if part.is_a?(TextContent) } } + end + end end # POST https://api.openai.com/v1/chat/completions @@ -73,6 +140,13 @@ module OpenAI # A unique identifier representing your end-user, which can help OpenAI to monitor and detect abuse. property user : String? = nil + + # i.e. `{"type": "json_object"}` or a `{"type": "json_schema", "json_schema": {...}}` spec + property response_format : JSON::Any? = nil + + # An upper bound for the number of tokens that can be generated, + # including both visible output and reasoning tokens. + property max_completion_tokens : Int32? = nil end struct MessageChoice