Merge pull request #2 from byerlikaya/feature/document-improvements

feat: Complete document service refactoring and logging migration

## Extended Description

This PR introduces comprehensive improvements to the SmartRAG system:

### �� Core Architecture
- Complete refactoring of DocumentService into focused, SOLID-compliant services
- Separation of concerns: DocumentService (CRUD) vs DocumentSearchService (AI/Search)
- Professional structured logging with ILogger throughout the codebase
- Migration from Console.WriteLine to structured logging

### 🚀 New Features
- Multi-document upload capability with parallel processing
- Smart query intent detection (general conversation vs document search)
- Enhanced embedding generation with fallback mechanisms
- Batch processing support for VoyageAI embeddings

### 📊 Performance & Reliability
- Intelligent rate limiting for AI providers
- Fallback strategies for embedding generation failures
- Optimized batch processing for large documents
- Enhanced error handling and retry mechanisms

### 🛡️ Security & Quality
- SanitizeForLog helper to prevent log forging security issues
- Centralized logging with ServiceLogMessages
- Comprehensive XML documentation for all interfaces
- SOLID and DRY principles implementation

### 🧪 Testing & Validation
- Unit tests pass successfully
- Integration tests completed
- Manual testing verified
- 0 errors, 0 warnings, 0 messages achieved

This refactoring establishes a solid foundation for future SmartRAG enhancements while maintaining backward compatibility.

Author: byerlikaya

README.md

@@ -547,11 +547,7 @@ We welcome contributions!
 
 This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.
 
-## 🌟 Star History
 
-[![Star History Chart](https://api.star-history.com/svg?repos=byerlikaya/SmartRAG&type=Date)](https://star-history.com/#byerlikaya/SmartRAG&Date)
-
----
 
 **Built with ❤️ by Barış Yerlikaya**
 

SmartRAG.sln

@@ -4,7 +4,7 @@ VisualStudioVersion = 17.0.31903.59
 MinimumVisualStudioVersion = 10.0.40219.1
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SmartRAG", "src\SmartRAG\SmartRAG.csproj", "{DECA885F-8815-4A0F-A12C-30563827C255}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SmartRAG.API", "src\SmartRAG.API\SmartRAG.API.csproj", "{E7606EAF-F26D-441F-B5A4-34A72A70DD6C}"
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SmartRAG.API", "examples\WebAPI\SmartRAG.API.csproj", "{E7606EAF-F26D-441F-B5A4-34A72A70DD6C}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution

examples/README.md

@@ -0,0 +1,54 @@
+# SmartRAG Examples
+
+This directory contains example projects demonstrating how to use SmartRAG in different scenarios.
+
+## 📁 Available Examples
+
+### **WebAPI** - ASP.NET Core Web API Example
+- **Location**: `WebAPI/`
+- **Description**: Complete web API implementation showing document upload, search, and RAG operations
+- **Features**: 
+  - Multi-document upload
+  - AI-powered question answering
+  - Smart query intent detection
+  - Multiple storage providers
+  - Comprehensive API documentation
+
+## 🚀 Running Examples
+
+### WebAPI Example
+```bash
+cd examples/WebAPI
+dotnet restore
+dotnet run
+```
+
+Browse to `https://localhost:5001/scalar/v1` for interactive API documentation.
+
+## 🔧 Configuration
+
+Each example includes its own configuration files. Copy and modify the template files as needed:
+
+```bash
+# Copy development configuration template
+cp appsettings.Development.template.json appsettings.Development.json
+
+# Edit with your API keys and configuration
+```
+
+## 📚 Documentation
+
+- **Main Documentation**: [SmartRAG README](../../README.md)
+- **API Reference**: [API Documentation](../../docs/api-reference.md)
+- **Configuration Guide**: [Configuration Guide](../../docs/configuration.md)
+
+## 🤝 Contributing
+
+Want to add more examples? Create a new directory and submit a pull request!
+
+### Example Types to Consider:
+- **Console Application** - Command-line interface
+- **Blazor WebAssembly** - Client-side web app
+- **WPF Application** - Desktop application
+- **Azure Functions** - Serverless implementation
+- **Minimal API** - Lightweight web API

src/SmartRAG.API/Contracts/SearchRequestIDto.cs renamed to examples/WebAPI/Contracts/SearchRequestIDto.cs

@@ -13,7 +13,9 @@ namespace SmartRAG.API.Controllers;
 [Route("api/[controller]")]
 [Produces("application/json")]
 [ProducesResponseType(StatusCodes.Status500InternalServerError)]
-public class DocumentsController(IDocumentService documentService, IDocumentParserService documentParser) : ControllerBase
+public class DocumentsController(
+    IDocumentService documentService,
+    IDocumentParserService documentParser) : ControllerBase
 {
     /// <summary>
     /// Gets supported file types and content types
@@ -58,6 +60,37 @@ public IActionResult GetSupportedTypes()
         }
     }
 
+    /// <summary>
+    /// Upload multiple documents to the system
+    /// </summary>
+    [HttpPost("upload-multiple")]
+    [Consumes("multipart/form-data")]
+    [RequestSizeLimit(100 * 1024 * 1024)] // 100 MB
+    [RequestFormLimits(MultipartBodyLengthLimit = 100 * 1024 * 1024)]
+    public async Task<ActionResult<List<Entities.Document>>> UploadDocuments([FromForm] List<IFormFile> files)
+    {
+        if (files == null || files.Count == 0)
+            return BadRequest("No files provided");
+
+        try
+        {
+            var fileStreams = files.Select(f => f.OpenReadStream());
+            var fileNames = files.Select(f => f.FileName);
+            var contentTypes = files.Select(f => f.ContentType);
+
+            var documents = await documentService.UploadDocumentsAsync(
+                fileStreams,
+                fileNames,
+                contentTypes,
+                "system");
+
+            return CreatedAtAction(nameof(GetAllDocuments), documents);
+        }
+        catch (Exception ex)
+        {
+            return StatusCode(500, $"Internal server error: {ex.Message}");
+        }
+    }
     /// <summary>
     /// Get a document by ID
     /// </summary>
@@ -83,7 +116,6 @@ public IActionResult GetSupportedTypes()
         return Ok(documents);
     }
 
-
     /// <summary>
     /// Delete a document
     /// </summary>

src/SmartRAG.API/Controllers/DocumentsController.cs renamed to examples/WebAPI/Controllers/DocumentsController.cs

@@ -10,7 +10,7 @@ namespace SmartRAG.API.Controllers;
 [Route("api/[controller]")]
 [Produces("application/json")]
 [ProducesResponseType(StatusCodes.Status500InternalServerError)]
-public class SearchController(IDocumentService documentService) : ControllerBase
+public class SearchController(IDocumentSearchService documentSearchService) : ControllerBase
 {
     /// <summary>
     /// Search documents using RAG (Retrieval-Augmented Generation)
@@ -28,7 +28,7 @@ public async Task<ActionResult<object>> Search([FromBody] Contracts.SearchReques
 
         try
         {
-            var response = await documentService.GenerateRagAnswerAsync(query, maxResults);
+            var response = await documentSearchService.GenerateRagAnswerAsync(query, maxResults);
             return Ok(response);
         }
         catch (Exception ex)

src/SmartRAG.API/Controllers/SearchController.cs renamed to examples/WebAPI/Controllers/SearchController.cs

@@ -0,0 +1,53 @@
+using Microsoft.OpenApi.Models;
+using Swashbuckle.AspNetCore.SwaggerGen;
+
+namespace SmartRAG.API.Filters;
+
+public class MultipartFileUploadFilter : IOperationFilter
+{
+    public void Apply(OpenApiOperation operation, OperationFilterContext context)
+    {
+        // Check if this is the upload-multiple endpoint
+        if (context.MethodInfo.Name == "UploadDocuments" && 
+            context.MethodInfo.DeclaringType?.Name == "DocumentsController")
+        {
+            // Remove any existing request body
+            operation.RequestBody = null;
+
+            // Add the multipart form data request body with multiple files
+            operation.RequestBody = new OpenApiRequestBody
+            {
+                Content = new Dictionary<string, OpenApiMediaType>
+                {
+                    ["multipart/form-data"] = new OpenApiMediaType
+                    {
+                        Schema = new OpenApiSchema
+                        {
+                            Type = "object",
+                            Properties = new Dictionary<string, OpenApiSchema>
+                            {
+                                ["files"] = new OpenApiSchema
+                                {
+                                    Type = "array",
+                                    Items = new OpenApiSchema
+                                    {
+                                        Type = "string",
+                                        Format = "binary"
+                                    }
+                                }
+                            },
+                            Required = new HashSet<string> { "files" }
+                        }
+                    }
+                }
+            };
+
+            // Add external documentation link for simple upload page
+            if (operation.ExternalDocs == null)
+                operation.ExternalDocs = new OpenApiExternalDocs();
+            
+            operation.ExternalDocs.Description = "Simple multiple file upload page";
+            operation.ExternalDocs.Url = new Uri("/upload.html", UriKind.Relative);
+        }
+    }
+}

src/SmartRAG.API/Controllers/StorageController.cs renamed to examples/WebAPI/Controllers/StorageController.cs

@@ -0,0 +1,87 @@
+using Scalar.AspNetCore;
+using SmartRAG.Enums;
+using SmartRAG.Extensions;
+using Microsoft.OpenApi.Models;
+using SmartRAG.API.Filters;
+
+var builder = WebApplication.CreateBuilder(args);
+
+// Configure Kestrel server options for file uploads
+builder.WebHost.UseKestrel(options =>
+{
+    options.Limits.MaxRequestBodySize = 100 * 1024 * 1024; // 100 MB
+});
+
+RegisterServices(builder.Services, builder.Configuration);
+
+var app = builder.Build();
+ConfigureMiddleware(app, builder.Environment);
+
+app.Run();
+
+static void RegisterServices(IServiceCollection services, IConfiguration configuration)
+{
+    // Configure logging
+    services.AddLogging(builder =>
+    {
+        builder.ClearProviders();
+        builder.AddConsole();
+        builder.AddDebug();
+        builder.SetMinimumLevel(LogLevel.Debug);
+    });
+
+    services.AddControllers();
+    services.AddEndpointsApiExplorer();
+    services.AddOpenApi();
+    services.AddSwaggerGen(c =>
+    {
+        c.SwaggerDoc("v1", new OpenApiInfo { Title = "SmartRAG API", Version = "v1" });
+        
+        // Configure multipart file upload for multiple files
+        c.OperationFilter<MultipartFileUploadFilter>();
+    });
+    
+    // Configure form options for file uploads
+    services.Configure<Microsoft.AspNetCore.Http.Features.FormOptions>(options =>
+    {
+        options.MultipartBodyLengthLimit = 100 * 1024 * 1024; // 100 MB
+        options.ValueLengthLimit = int.MaxValue;
+        options.ValueCountLimit = int.MaxValue;
+        options.KeyLengthLimit = int.MaxValue;
+    });
+
+    // Add SmartRag services with minimal configuration
+    services.UseSmartRag(configuration,
+        storageProvider: StorageProvider.InMemory,  // Default: InMemory
+        aiProvider: AIProvider.Gemini               // Use OpenAI provider
+    );
+
+    services.AddCors(options =>
+    {
+        options.AddPolicy("AllowAll", policy =>
+        {
+            policy.AllowAnyOrigin()
+                  .AllowAnyMethod()
+                  .AllowAnyHeader();
+        });
+    });
+}
+
+static void ConfigureMiddleware(WebApplication app, IWebHostEnvironment environment)
+{
+
+    if (environment.IsDevelopment())
+    {
+        app.MapOpenApi();
+        app.MapSwagger();
+        app.UseSwaggerUI();
+    }
+
+    // Serve static files for simple upload page
+    app.UseStaticFiles();
+
+    app.UseHttpsRedirection();
+    app.UseCors("AllowAll");
+    app.UseAuthorization();
+    app.MapControllers();
+}