Apply the attachments refactoring. Rebase on origin/main

Author: OriPekelman

lib/ruby_llm/providers/mistral/chat.rb

@@ -59,29 +59,15 @@ def render_payload(messages, tools:, temperature:, model:, stream: nil,
 
         def render_message(message)
           result = {}
-          result[:role] = message.role
+          result[:role] = message.role.to_s # Ensure role is a string for API compliance
 
           # If the message content is a RubyLLM::Content with attachments, convert to multimodal array
           if message.content.is_a?(RubyLLM::Content)
             content = message.content
-            if content.attachments.any?
-              multimodal = []
-              multimodal << { type: "text", text: content.text } if content.text
-              content.attachments.each do |attachment|
-                case attachment
-                when RubyLLM::Attachments::Image
-                  multimodal << { type: "image", source: attachment.url? ? { url: attachment.source } : { type: 'base64', media_type: attachment.mime_type, data: attachment.encoded } }
-                # Add more attachment types if Mistral supports them
-                end
-              end
-              # Always format multimodal for Mistral
-              result[:content] = Mistral::Media.format_content(multimodal)
-            else
-              result[:content] = content.text
-            end
+            result[:content] = Mistral::Media.format_content(content)
           elsif message.content.is_a?(Array)
             # Multimodal content: format each part
-            formatted_content = Mistral::Media.format_content(message.content.compact)
+            formatted_content = message.content.compact
             result[:content] = formatted_content unless formatted_content.empty?
           else
             # Simple text content
@@ -96,6 +82,7 @@ def render_message(message)
             tool_calls = message.tool_calls.is_a?(Hash) ? message.tool_calls.values : message.tool_calls
             result[:tool_calls] = tool_calls.map { |tc| render_tool_call(tc) }
           end
+          
           result.compact
         end
 

lib/ruby_llm/providers/mistral/media.rb

@@ -2,80 +2,70 @@ module RubyLLM
   module Providers
     module Mistral
       # Media handling for Mistral models
-      #
-      # NOTE: There's currently an issue with Pixtral vision capabilities in the test suite.
-      # The content array contains nil values when an image is attached, which causes the API to return errors.
-      # This might be due to how images are being attached or formatted in the Content class.
-      # The test in chat_content_spec.rb for 'pixtral-12b-latest can understand images' currently fails.
-      # The debug output shows: content: [{type: "text", text: "..."}, nil] where the second element should be the image.
-      # This likely requires fixes in the core library's Content class or how it interacts with provider-specific formatting.
+
       module Media
         module_function
 
-        def supports_image?(model_id)
-          # Check if the model supports vision according to the model capabilities
-          capabilities.supports_vision?(model_id)
-        end
-
-        def supports_audio?(model_id)
-          # Check if the model supports audio according to the model capabilities
-          capabilities.supports_audio?(model_id)
-        end
-
-        # Moved from chat.rb
         def format_content(content)
-          return content unless content.is_a?(Array)
+          return content unless content.is_a?(Content)
 
-          RubyLLM.logger.debug "Formatting multimodal content: #{content.inspect}"
+          parts = []
+          parts << { type: "text", text: content.text } if content.text
 
-          # Filter out nil values
-          filtered_content = content.compact
-
-          RubyLLM.logger.debug "Filtered content: #{filtered_content.inspect}"
-
-          filtered_content.map do |item|
-            if item.is_a?(Hash) && item[:type] == "image"
-              format_image_content(item)
+          content.attachments.each do |attachment|
+            case attachment.type
+            when :image
+              parts << format_image(attachment)
+            when :pdf
+              # Mistral doesn't currently support PDFs other than
+              # through the OCR API
+              raise UnsupportedAttachmentError, attachment.type
+            when :audio
+              # Mistral doesn't currently support audio
+              raise UnsupportedAttachmentError, attachment.type
+            when :text
+              # Mistral doesn't support text files as attachments, so we'll append to the text
+              if parts.first && parts.first[:type] == "text"
+                parts.first[:text] += "\n\n" + Utils.format_text_file_for_llm(attachment)
+              else
+                parts << { type: "text", text: Utils.format_text_file_for_llm(attachment) }
+              end
             else
-              item # Pass through text or other types
+              raise UnsupportedAttachmentError, attachment.type
             end
           end
+
+          # Filter out nil values and return the formatted content array
+          parts.compact
         end
 
         # Format image according to Mistral API requirements
-        # @param image [Hash] Image data hash from Content class
+        # @param image [Attachment] Image attachment
         # @return [Hash] Formatted image data for Mistral API
-        def format_image_content(image)
-          RubyLLM.logger.debug "Formatting image content: #{image.inspect}"
-          
-          if image[:source].is_a?(Hash)
-            if image[:source][:url]
-              # Direct URL from source hash
-              {
-                type: "image_url",
-                image_url: image[:source][:url]
-              }
-            elsif image[:source][:type] == 'base64'
-              # Base64 data from source hash
-              data_uri = "data:#{image[:source][:media_type]};base64,#{image[:source][:data]}"
-              {
-                type: "image_url",
-                image_url: data_uri
-              }
-            else
-              RubyLLM.logger.warn "Invalid image source format: #{image[:source]}"
-              nil
-            end
-          elsif image[:source].is_a?(String)
-            # Direct URL string
-            {
-              type: "image_url",
-              image_url: image[:source]
-            }
+        def format_image(attachment)
+          url = if attachment.respond_to?(:source) && attachment.source.to_s.match?(/^https?:\/\//)
+            attachment.source.to_s
           else
-            RubyLLM.logger.warn "Invalid image format: #{image}"
-            nil
+            "data:#{attachment.mime_type};base64,#{attachment.encoded}"
           end
+          {
+            type: "image_url",
+            image_url: url
+          }
+        end
+
+        def supports_image?(model_id)
+          # Check if the model supports vision according to the model capabilities
+          capabilities.supports_vision?(model_id)
+        end
+
+        def supports_audio?(model_id)
+          # Check if the model supports audio according to the model capabilities
+          capabilities.supports_audio?(model_id)
+        end
+
+        def capabilities
+          RubyLLM::Providers::Mistral::Capabilities
         end
       end
     end

lib/ruby_llm/providers/mistral/models.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'ruby_llm/model/info'
+
 module RubyLLM
   module Providers
     module Mistral
@@ -20,7 +22,7 @@ def parse_list_models_response(response, slug, capabilities)
         def parse_model(model, slug, capabilities)
           id = model["id"]
 
-          ModelInfo.new(
+          ModelInfoWithFunctions.new(
             id: id,
             created_at: model["created"] ? Time.at(model["created"]) : nil,
             display_name: capabilities.format_display_name(id),
@@ -36,6 +38,12 @@ def parse_model(model, slug, capabilities)
             capabilities: capabilities.capabilities_for(id),
           )
         end
+
+        class ModelInfoWithFunctions < RubyLLM::Model::Info
+          def supports_functions?
+            RubyLLM::Providers::Mistral::Capabilities.supports_functions?(id)
+          end
+        end
       end
     end
   end