feat(ui): Filter UUID files from Consolidated Files view

- Filter out files with UUID-like identifiers (8-4-4-4-12 format) from the
  Consolidated Files section, showing only domain-named files from crawling
- Improve error handling in MCP server to create default metadata with "pages"
  array when JSON files don't exist
- Add logging to track filtered files count for debugging

This change improves the user experience by showing only relevant crawled
files in the Consolidated Files section, making it easier to find and
manage content. The UUID-based files (typically auto-generated) are now
hidden, while domain-named files (created through the crawling process)
remain visible.

Author: Shubham-Khichi

app/api/storage/route.ts

@@ -59,26 +59,71 @@ export async function GET(request: Request) {
               const jsonContent = await fs.readFile(jsonPath, 'utf-8')
               const metadata = JSON.parse(jsonContent)
 
-              // If the metadata has a "pages" array, it's a consolidated file
-              if (metadata.pages && Array.isArray(metadata.pages)) {
+              // If the metadata has a "pages" array or is_consolidated flag, it's a consolidated file
+              if ((metadata.pages && Array.isArray(metadata.pages)) || metadata.is_consolidated === true) {
                 isConsolidated = true
-                pagesCount = metadata.pages.length
+                pagesCount = metadata.pages ? metadata.pages.length : 1
                 rootUrl = metadata.root_url || ''
               }
             } catch (e) {
               console.error(`Error reading JSON metadata for ${filename}:`, e)
+              // Create a default metadata file if it doesn't exist or is invalid
+              try {
+                const defaultMetadata = {
+                  title: `Documentation for ${filename.replace('.md', '')}`,
+                  timestamp: new Date().toISOString(),
+                  pages: [
+                    {
+                      title: "Main Content",
+                      url: `file://${filename.replace('.md', '')}`,
+                      timestamp: new Date().toISOString(),
+                      internal_links: 0,
+                      external_links: 0
+                    }
+                  ],
+                  is_consolidated: true,
+                  last_updated: new Date().toISOString()
+                }
+                await fs.writeFile(jsonPath, JSON.stringify(defaultMetadata, null, 2), 'utf-8')
+                console.log(`Created default metadata for ${filename}`)
+                isConsolidated = true
+                pagesCount = 1
+              } catch (writeError) {
+                console.error(`Error creating default metadata for ${filename}:`, writeError)
+              }
             }
           } else {
             // Create JSON file if it doesn't exist
-            const jsonContent = JSON.stringify({
-              content,
-              metadata: {
-                wordCount: content.split(/\s+/).length,
-                charCount: content.length,
-                timestamp: stats.mtime
+            try {
+              // Create a consolidated metadata file by default
+              const defaultMetadata = {
+                title: `Documentation for ${filename.replace('.md', '')}`,
+                timestamp: new Date().toISOString(),
+                content,
+                pages: [
+                  {
+                    title: "Main Content",
+                    url: `file://${filename.replace('.md', '')}`,
+                    timestamp: new Date().toISOString(),
+                    internal_links: 0,
+                    external_links: 0
+                  }
+                ],
+                is_consolidated: true,
+                last_updated: new Date().toISOString(),
+                metadata: {
+                  wordCount: content.split(/\s+/).length,
+                  charCount: content.length,
+                  timestamp: stats.mtime
+                }
               }
-            }, null, 2)
-            await fs.writeFile(jsonPath, jsonContent, 'utf-8')
+              await fs.writeFile(jsonPath, JSON.stringify(defaultMetadata, null, 2), 'utf-8')
+              console.log(`Created consolidated metadata for ${filename}`)
+              isConsolidated = true
+              pagesCount = 1
+            } catch (writeError) {
+              console.error(`Error creating metadata for ${filename}:`, writeError)
+            }
           }
 
           // Extract sections to count how many pages are included
@@ -153,9 +198,22 @@ export async function GET(request: Request) {
       // Only show consolidated files in the Stored Files section
       const consolidatedFiles = allFiles.filter(file => file.isConsolidated)
 
+      // Additional filter to exclude files with UUID-like names
+      // UUID pattern: 8-4-4-4-12 hex digits (e.g., 095104d8-8e90-48f0-8670-9e45c914f115)
+      const uuidPattern = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i
+      
+      // Keep only files with domain-like names (e.g., docs_crawl4ai_com)
+      // These are files created through the crawling process
+      const crawledFiles = consolidatedFiles.filter(file => {
+        // Check if the filename is NOT a UUID
+        return !uuidPattern.test(file.name)
+      })
+      
+      console.log(`Found ${consolidatedFiles.length} consolidated files, ${crawledFiles.length} are crawled files`)
+      
       return NextResponse.json({
         success: true,
-        files: consolidatedFiles
+        files: crawledFiles
       })
     }
 

fast-markdown-mcp/src/fast_markdown_mcp/server.py

@@ -95,6 +95,38 @@ async def get_metadata(self, file_id: str) -> dict:
         try:
             with open(file_path, 'r', encoding='utf-8') as f:
                 return json.load(f)
+        except FileNotFoundError:
+            # Create a default metadata file with a pages array if the MD file exists
+            md_file_path = self.base_path / f"{file_id}.md"
+            if md_file_path.exists():
+                try:
+                    content = md_file_path.read_text(encoding='utf-8')
+                    # Create a basic metadata structure with pages array to mark as consolidated
+                    from datetime import datetime
+                    default_metadata = {
+                        "title": f"Documentation for {file_id}",
+                        "timestamp": datetime.now().isoformat(),
+                        "pages": [
+                            {
+                                "title": "Main Content",
+                                "url": f"file://{file_id}",
+                                "timestamp": datetime.now().isoformat(),
+                                "internal_links": 0,
+                                "external_links": 0
+                            }
+                        ],
+                        "is_consolidated": True,
+                        "last_updated": datetime.now().isoformat()
+                    }
+                    # Write the default metadata to file
+                    with open(file_path, 'w', encoding='utf-8') as f:
+                        json.dump(default_metadata, f, indent=2)
+                    logger.info(f"Created default metadata for {file_id}")
+                    return default_metadata
+                except Exception as write_error:
+                    logger.error(f"Error creating default metadata for {file_id}: {write_error}")
+            logger.error(f"Error reading metadata for {file_id}: File not found")
+            return {}
         except Exception as e:
             logger.error(f"Error reading metadata for {file_id}: {e}")
             return {}