adding fallback to the content type (URLs only) into the Rabbit Hole (#15)

Author: matteocacciola

core/cat/rabbit_hole.py

@@ -161,7 +161,10 @@ async def __file_to_docs(
                 request = httpx.get(file, headers={"User-Agent": "Magic Browser"})
 
                 # Define mime type and source of url
-                content_type = request.headers["Content-Type"].split(";")[0]
+                # Add fallback for empty/None content_type
+                content_type = request.headers.get(
+                    "Content-Type", "text/html" if file.startswith(("http://", "https://")) else "text/plain"
+                ).split(";")[0]
                 source = file
 
                 try:
@@ -183,6 +186,10 @@ async def __file_to_docs(
         if not file_bytes:
             raise ValueError(f"Something went wrong with the file {source}")
 
+        log.debug(f"Attempting to parse file: {source}")
+        log.debug(f"Detected MIME type: {content_type}")
+        log.debug(f"Available handlers: {list(stray.file_handlers.keys())}")
+
         # Load the bytes in the Blob schema
         blob = Blob(data=file_bytes, mimetype=content_type).from_data(
             data=file_bytes, mime_type=content_type, path=source

core/pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "Cheshire-Cat"
 description = "Production ready AI assistant framework"
-version = "2.1.2"
+version = "2.1.3"
 requires-python = ">=3.11"
 license = { file = "LICENSE" }
 authors = [