Merge branch 'main' of https://github.com/byerlikaya/SmartRAG

Author: byerlikaya

README.md

@@ -18,7 +18,7 @@ SmartRAG is a **production-ready** .NET 9.0 library that provides a complete **R
 - 🧠 **Advanced RAG Pipeline**: Smart chunking, semantic retrieval, AI-powered answer generation
 - ⚡ **Lightning Fast**: Optimized vector search with context-aware answer synthesis
 - 🔌 **Plug & Play**: Single-line integration with dependency injection
-- 📄 **Multi-Format**: PDF, Word, text files with intelligent parsing
+- 📄 **Multi-Format**: PDF, Word, Excel, text files with intelligent parsing
 - 🎯 **Enhanced Semantic Search**: Advanced hybrid scoring with 80% semantic + 20% keyword relevance
 - 🔍 **Smart Document Chunking**: Word boundary validation and optimal break points for context preservation
 - ✅ **Enterprise Grade**: Zero Warnings Policy, SOLID principles, comprehensive logging, XML documentation
@@ -38,7 +38,7 @@ SmartRAG is a **production-ready** .NET 9.0 library that provides a complete **R
 - **Language-Agnostic Design**: No hardcoded language patterns - works globally with any language
 - **Multiple Storage Options**: From in-memory to enterprise vector databases
 - **AI Provider Flexibility**: Switch between providers without code changes
-- **Document Intelligence**: Advanced parsing for PDF, Word, and text formats
+- **Document Intelligence**: Advanced parsing for PDF, Word, Excel, and text formats
 - **Configuration-First**: Environment-based configuration with sensible defaults
 - **Dependency Injection**: Full DI container integration
 - **Enhanced Semantic Search**: Advanced hybrid scoring combining semantic similarity and keyword relevance
@@ -134,6 +134,42 @@ dotnet add package SmartRAG
 <PackageReference Include="SmartRAG" Version="1.1.0" />
 ```
 
+## 📄 Supported Document Formats
+
+SmartRAG supports a wide range of document formats with intelligent parsing and text extraction:
+
+### **📊 Excel Files (.xlsx, .xls)**
+- **Advanced Parsing**: Extracts text from all worksheets and cells
+- **Structured Data**: Preserves table structure with tab-separated values
+- **Worksheet Names**: Includes worksheet names for context
+- **Cell Content**: Extracts all non-empty cell values
+- **Format Preservation**: Maintains data organization for better context
+
+### **📝 Word Documents (.docx, .doc)**
+- **Rich Text Extraction**: Preserves formatting and structure
+- **Table Support**: Extracts content from tables and lists
+- **Paragraph Handling**: Maintains paragraph breaks and flow
+- **Metadata Preservation**: Keeps document structure intact
+
+### **📋 PDF Documents (.pdf)**
+- **Multi-Page Support**: Processes all pages with text extraction
+- **Layout Preservation**: Maintains document structure and flow
+- **Text Quality**: High-quality text extraction for analysis
+- **Page Separation**: Clear page boundaries for context
+
+### **📄 Text Files (.txt, .md, .json, .xml, .csv, .html, .htm)**
+- **Universal Support**: Handles all text-based formats
+- **Encoding Detection**: Automatic UTF-8 and encoding detection
+- **Structure Preservation**: Maintains original formatting
+- **Fast Processing**: Optimized for text-based content
+
+### **🔍 Content Type Support**
+SmartRAG automatically detects file types using both file extensions and MIME content types:
+- **Excel**: `application/vnd.openxmlformats-officedocument.spreadsheetml.sheet`, `application/vnd.ms-excel`
+- **Word**: `application/vnd.openxmlformats-officedocument.wordprocessingml.document`, `application/msword`
+- **PDF**: `application/pdf`
+- **Text**: `text/*`, `application/json`, `application/xml`, `application/csv`
+
 ## 🚀 Quick Start
 
 ### 1. **Development Setup**

src/SmartRAG/Providers/AnthropicProvider.cs

@@ -144,6 +144,15 @@ public override async Task<List<List<float>>> GenerateEmbeddingsBatchAsync(IEnum
         if (inputList.Count == 0)
             return [];
 
+        // Filter out empty or null strings to prevent Voyage AI API errors
+        var validInputs = inputList.Where(text => !string.IsNullOrWhiteSpace(text)).ToList();
+        
+        if (validInputs.Count == 0)
+        {
+            ProviderLogMessages.LogAnthropicEmbeddingValidationError(Logger, "All input texts are empty or null", null);
+            return Enumerable.Range(0, inputList.Count).Select(_ => new List<float>()).ToList();
+        }
+
         var voyageApiKey = config.EmbeddingApiKey ?? config.ApiKey;
         var voyageModel = config.EmbeddingModel ?? DefaultVoyageModel;
 
@@ -156,7 +165,7 @@ public override async Task<List<List<float>>> GenerateEmbeddingsBatchAsync(IEnum
 
         var payload = new
         {
-            input = inputList.ToArray(),
+            input = validInputs.ToArray(),
             model = voyageModel,
             input_type = VoyageInputType
         };
@@ -173,7 +182,8 @@ public override async Task<List<List<float>>> GenerateEmbeddingsBatchAsync(IEnum
 
         try
         {
-            return ParseVoyageBatchEmbeddingResponse(response, inputList.Count);
+            var validEmbeddings = ParseVoyageBatchEmbeddingResponse(response, validInputs.Count);
+            return MapEmbeddingsToOriginalInputs(validEmbeddings, inputList, validInputs);
         }
         catch (Exception ex)
         {
@@ -295,5 +305,39 @@ private static List<List<float>> ParseVoyageBatchEmbeddingResponse(string respon
             .ToList();
     }
 
+    /// <summary>
+    /// Maps valid embeddings back to original input positions, filling empty positions with empty embeddings
+    /// </summary>
+    private static List<List<float>> MapEmbeddingsToOriginalInputs(List<List<float>> validEmbeddings, List<string> originalInputs, List<string> validInputs)
+    {
+        var result = new List<List<float>>();
+        var validIndex = 0;
+
+        for (int i = 0; i < originalInputs.Count; i++)
+        {
+            if (string.IsNullOrWhiteSpace(originalInputs[i]))
+            {
+                // Empty input gets empty embedding
+                result.Add(new List<float>());
+            }
+            else
+            {
+                // Valid input gets corresponding embedding
+                if (validIndex < validEmbeddings.Count)
+                {
+                    result.Add(validEmbeddings[validIndex]);
+                    validIndex++;
+                }
+                else
+                {
+                    // Fallback: empty embedding if something goes wrong
+                    result.Add(new List<float>());
+                }
+            }
+        }
+
+        return result;
+    }
+
     #endregion
 }

src/SmartRAG/Providers/BaseAIProvider.cs

@@ -210,8 +210,9 @@ protected static HttpClient CreateHttpClientWithoutAuth(Dictionary<string, strin
                 if (success)
                     return (true, response, string.Empty);
 
-                // Handle rate limiting
-                if (error.Contains("429") || error.Contains("TooManyRequests"))
+                // Handle rate limiting and server overload
+                if (error.Contains("429") || error.Contains("TooManyRequests") || 
+                    error.Contains("529") || error.Contains("Overloaded"))
                 {
                     attempt++;
                     if (attempt < maxRetries)
@@ -299,11 +300,12 @@ private static void AddAdditionalHeaders(HttpClient client, Dictionary<string, s
     }
 
     /// <summary>
-    /// Calculates retry delay for rate limiting
+    /// Calculates retry delay for rate limiting and server overload
     /// </summary>
     private static int CalculateRetryDelay(int attempt)
     {
-        return MinRetryDelayMs;
+        // For server overload (529), use exponential backoff starting from 60 seconds
+        return MinRetryDelayMs * attempt;
     }
 
     /// <summary>

src/SmartRAG/Services/DocumentParserService.cs

@@ -12,6 +12,7 @@
 using SmartRAG.Services;
 using System.Text;
 using System.Text.RegularExpressions;
+using OfficeOpenXml;
 
 namespace SmartRAG.Services;
 
@@ -22,11 +23,19 @@ public class DocumentParserService(
     IOptions<SmartRagOptions> options,
     ILogger<DocumentParserService> logger) : IDocumentParserService
 {
+    /// <summary>
+    /// Static constructor to set EPPlus license once for the application
+    /// </summary>
+    static DocumentParserService()
+    {
+        // Set EPPlus 8+ license for non-commercial organization use
+        ExcelPackage.License.SetNonCommercialOrganization("SmartRAG");
+    }
     #region Constants
 
     // Content validation constants
-    private const int MinContentLength = 10;
-    private const double MinMeaningfulTextRatio = 0.3;
+    private const int MinContentLength = 5; // Reduced for Excel files
+    private const double MinMeaningfulTextRatio = 0.1; // Reduced for Excel files
 
     // Chunk boundary search constants
     private const int DefaultDynamicSearchRange = 500;
@@ -36,6 +45,7 @@ public class DocumentParserService(
     // File extension constants
     private static readonly string[] WordExtensions = [".docx", ".doc"];
     private static readonly string[] PdfExtensions = [".pdf"];
+    private static readonly string[] ExcelExtensions = [".xlsx", ".xls"];
     private static readonly string[] TextExtensions = [".txt", ".md", ".json", ".xml", ".csv", ".html", ".htm"];
 
     // Content type constants
@@ -45,6 +55,12 @@ public class DocumentParserService(
         "application/vnd.ms-word"
     ];
 
+    private static readonly string[] ExcelContentTypes = [
+        "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+        "application/vnd.ms-excel",
+        "application/vnd.ms-excel.sheet.macroEnabled.12"
+    ];
+    
     private static readonly string[] TextContentTypes = ["text/", "application/json", "application/xml", "application/csv"];
 
     // Sentence ending constants
@@ -91,12 +107,12 @@ public class DocumentParserService(
     /// <summary>
     /// Gets supported file types
     /// </summary>
-    public IEnumerable<string> GetSupportedFileTypes() => TextExtensions.Concat(WordExtensions).Concat(PdfExtensions);
+    public IEnumerable<string> GetSupportedFileTypes() => TextExtensions.Concat(WordExtensions).Concat(PdfExtensions).Concat(ExcelExtensions);
 
     /// <summary>
     /// Gets supported content types
     /// </summary>
-    public IEnumerable<string> GetSupportedContentTypes() => TextContentTypes.Concat(WordContentTypes).Append("application/pdf");
+    public IEnumerable<string> GetSupportedContentTypes() => TextContentTypes.Concat(WordContentTypes).Append("application/pdf").Concat(ExcelContentTypes);
 
     #endregion
 
@@ -153,6 +169,15 @@ private static bool IsPdfDocument(string fileName, string contentType)
                contentType == "application/pdf";
     }
 
+    /// <summary>
+    /// Checks if file is an Excel document
+    /// </summary>
+    private static bool IsExcelDocument(string fileName, string contentType)
+    {
+        return ExcelExtensions.Any(ext => fileName.EndsWith(ext, StringComparison.OrdinalIgnoreCase)) || 
+               ExcelContentTypes.Any(ct => contentType.Contains(ct));
+    }
+
     /// <summary>
     /// Checks if file is text-based
     /// </summary>
@@ -175,6 +200,10 @@ private static async Task<string> ExtractTextAsync(Stream fileStream, string fil
         {
             return await ParsePdfDocumentAsync(fileStream);
         }
+        else if (IsExcelDocument(fileName, contentType))
+        {
+            return await ParseExcelDocumentAsync(fileStream);
+        }
         else if (IsTextBasedFile(fileName, contentType))
         {
             return await ParseTextDocumentAsync(fileStream);
@@ -213,6 +242,104 @@ private static async Task<string> ParseWordDocumentAsync(Stream fileStream)
         }
     }
 
+    /// <summary>
+    /// Parses Excel document and extracts text content
+    /// </summary>
+    private static async Task<string> ParseExcelDocumentAsync(Stream fileStream)
+    {
+        try
+        {
+            var memoryStream = await CreateMemoryStreamCopy(fileStream);
+            
+            // EPPlus license already set in static constructor
+            using var package = new ExcelPackage(memoryStream);
+            var textBuilder = new StringBuilder();
+
+            // Check if workbook has any worksheets
+            if (package.Workbook.Worksheets.Count == 0)
+            {
+                return "Excel file contains no worksheets";
+            }
+
+            foreach (var worksheet in package.Workbook.Worksheets)
+            {
+                if (worksheet.Dimension != null)
+                {
+                    textBuilder.AppendLine(string.Format(System.Globalization.CultureInfo.InvariantCulture, "Worksheet: {0}", worksheet.Name));
+                    
+                    var rowCount = worksheet.Dimension.Rows;
+                    var colCount = worksheet.Dimension.Columns;
+
+                    // Add header row if exists
+                    var hasData = false;
+                    for (int row = 1; row <= rowCount; row++)
+                    {
+                        var rowBuilder = new StringBuilder();
+                        var rowHasData = false;
+                        
+                        for (int col = 1; col <= colCount; col++)
+                        {
+                            var cellValue = worksheet.Cells[row, col].Value;
+                            if (cellValue != null)
+                            {
+                                var cellText = cellValue.ToString();
+                                if (!string.IsNullOrWhiteSpace(cellText))
+                                {
+                                    rowBuilder.Append(cellText);
+                                    rowHasData = true;
+                                    if (col < colCount) rowBuilder.Append('\t');
+                                }
+                                else
+                                {
+                                    rowBuilder.Append(' '); // Empty cell gets space
+                                    if (col < colCount) rowBuilder.Append('\t');
+                                }
+                            }
+                            else
+                            {
+                                rowBuilder.Append(' '); // Null cell gets space
+                                if (col < colCount) rowBuilder.Append('\t');
+                            }
+                        }
+                        
+                        if (rowHasData)
+                        {
+                            textBuilder.AppendLine(rowBuilder.ToString());
+                            hasData = true;
+                        }
+                    }
+                    
+                    if (!hasData)
+                    {
+                        textBuilder.AppendLine("Worksheet contains no data");
+                    }
+                    
+                    textBuilder.AppendLine();
+                }
+                else
+                {
+                    textBuilder.AppendLine(string.Format(System.Globalization.CultureInfo.InvariantCulture, "Worksheet: {0} (empty)", worksheet.Name));
+                }
+            }
+
+            var content = textBuilder.ToString();
+            var cleanedContent = CleanContent(content);
+            
+            // If content is still empty after cleaning, return a fallback message
+            if (string.IsNullOrWhiteSpace(cleanedContent))
+            {
+                return "Excel file processed but no text content extracted";
+            }
+            
+            return cleanedContent;
+        }
+        catch (Exception ex)
+        {
+            // Return error message instead of empty string for debugging
+            return $"Error parsing Excel file: {ex.Message}";
+        }
+    }
+
     /// <summary>
     /// Creates a memory stream copy for processing
     /// </summary>
@@ -369,7 +496,15 @@ private static bool IsContentValid(string content)
             return false;
         }
 
+        // For Excel files, be more lenient with content validation
         var meaningfulTextRatio = content.Count(c => char.IsLetterOrDigit(c)) / (double)content.Length;
+        
+        // If content contains worksheet markers, it's likely valid Excel content
+        if (content.Contains("Worksheet:") || content.Contains("Excel file"))
+        {
+            return true;
+        }
+        
         return meaningfulTextRatio >= MinMeaningfulTextRatio;
     }
 
@@ -747,3 +882,4 @@ private static int CalculateNextStartPosition(string content, int currentStart,
 
     #endregion
 }
+

src/SmartRAG/Services/DocumentService.cs

@@ -3,7 +3,6 @@
 using SmartRAG.Entities;
 using SmartRAG.Interfaces;
 using SmartRAG.Models;
-using SmartRAG.Services;
 using System.Globalization;
 using System.Text;
 
@@ -30,7 +29,7 @@ public class DocumentService(
     private const string NoFileNamesMessage = "No file names provided";
     private const string NoContentTypesMessage = "No content types provided";
     private const string MismatchedCountsMessage = "Number of file streams, names, and content types must match";
-    
+
     // CompositeFormat for repeated formatting (CA1863)
     private static readonly CompositeFormat UnsupportedFileTypeFormat = CompositeFormat.Parse("Unsupported file type: {0}. Supported types: {1}");
     private static readonly CompositeFormat UnsupportedContentTypeFormat = CompositeFormat.Parse("Unsupported content type: {0}. Supported types: {1}");