neo4j-labs
diff --git a/‎backend/score.py
Lines changed: 91 additions & 9 deletions b/‎backend/score.py
Lines changed: 91 additions & 9 deletions
diff --git a/‎backend/src/communities.py
Lines changed: 33 additions & 16 deletions b/‎backend/src/communities.py
Lines changed: 33 additions & 16 deletions
diff --git a/‎backend/src/graphDB_dataAccess.py
Lines changed: 1 addition & 1 deletion b/‎backend/src/graphDB_dataAccess.py
Lines changed: 1 addition & 1 deletion
diff --git a/‎backend/src/graph_query.py
Lines changed: 32 additions & 1 deletion b/‎backend/src/graph_query.py
Lines changed: 32 additions & 1 deletion
diff --git a/‎backend/src/neighbours.py
Lines changed: 2 additions & 1 deletion b/‎backend/src/neighbours.py
Lines changed: 2 additions & 1 deletion
diff --git a/‎backend/src/shared/constants.py
Lines changed: 13 additions & 0 deletions b/‎backend/src/shared/constants.py
Lines changed: 13 additions & 0 deletions
diff --git a/‎docker-compose.yml
Lines changed: 1 addition & 1 deletion b/‎docker-compose.yml
Lines changed: 1 addition & 1 deletion
diff --git a/‎example.env
Lines changed: 0 additions & 1 deletion b/‎example.env
Lines changed: 0 additions & 1 deletion
diff --git a/‎frontend/Dockerfile
Lines changed: 0 additions & 1 deletion b/‎frontend/Dockerfile
Lines changed: 0 additions & 1 deletion
diff --git a/‎frontend/src/components/ChatBot/ChunkInfo.tsx
Lines changed: 1 addition & 1 deletion b/‎frontend/src/components/ChatBot/ChunkInfo.tsx
Lines changed: 1 addition & 1 deletion
diff --git a/‎frontend/src/components/ChatBot/CommunitiesInfo.tsx
Lines changed: 1 addition & 1 deletion b/‎frontend/src/components/ChatBot/CommunitiesInfo.tsx
Lines changed: 1 addition & 1 deletion
diff --git a/‎frontend/src/components/ChatBot/EntitiesInfo.tsx
Lines changed: 1 addition & 1 deletion b/‎frontend/src/components/ChatBot/EntitiesInfo.tsx
Lines changed: 1 addition & 1 deletion
diff --git a/‎frontend/src/components/ChatBot/SourcesInfo.tsx
Lines changed: 1 addition & 1 deletion b/‎frontend/src/components/ChatBot/SourcesInfo.tsx
Lines changed: 1 addition & 1 deletion
diff --git a/‎frontend/src/components/ChatBot/chatInfo.ts
Lines changed: 1 addition & 1 deletion b/‎frontend/src/components/ChatBot/chatInfo.ts
Lines changed: 1 addition & 1 deletion
@@ -12,14 +12,14 @@
 from langchain_google_vertexai import ChatVertexAI
 from src.api_response import create_api_response
 from src.graphDB_dataAccess import graphDBdataAccess
-from src.graph_query import get_graph_results
+from src.graph_query import get_graph_results,get_chunktext_results
 from src.chunkid_entities import get_entities_from_chunkids
 from src.post_processing import create_vector_fulltext_indexes, create_entity_embedding
 from sse_starlette.sse import EventSourceResponse
 from src.communities import create_communities
 from src.neighbours import get_neighbour_nodes
 import json
-from typing import List, Mapping
+from typing import List, Mapping, Union
 from starlette.middleware.sessions import SessionMiddleware
 import google_auth_oauthlib.flow
 from google.oauth2.credentials import Credentials
@@ -33,8 +33,10 @@
 from Secweb.ContentSecurityPolicy import ContentSecurityPolicy
 from Secweb.XContentTypeOptions import XContentTypeOptions
 from Secweb.XFrameOptions import XFrame
-
+from fastapi.middleware.gzip import GZipMiddleware
 from src.ragas_eval import *
+from starlette.types import ASGIApp, Message, Receive, Scope, Send
+import gzip
 
 logger = CustomLogger()
 CHUNK_DIR = os.path.join(os.path.dirname(__file__), "chunks")
@@ -49,14 +51,42 @@ def healthy():
 
 def sick():
     return False
-
+class CustomGZipMiddleware:
+    def __init__(
+        self,
+        app: ASGIApp,
+        paths: List[str],
+        minimum_size: int = 1000,
+        compresslevel: int = 5
+    ):
+        self.app = app
+        self.paths = paths
+        self.minimum_size = minimum_size
+        self.compresslevel = compresslevel
+    
+    async def __call__(self, scope: Scope, receive: Receive, send: Send):
+        if scope["type"] != "http":
+            return await self.app(scope, receive, send)
+ 
+        path = scope["path"]
+        should_compress = any(path.startswith(gzip_path) for gzip_path in self.paths)
+        
+        if not should_compress:
+            return await self.app(scope, receive, send)
+        
+        gzip_middleware = GZipMiddleware(
+            app=self.app,
+            minimum_size=self.minimum_size,
+            compresslevel=self.compresslevel
+        )
+        await gzip_middleware(scope, receive, send)
 app = FastAPI()
 # SecWeb(app=app, Option={'referrer': False, 'xframe': False})
-# app.add_middleware(HSTS, Option={'max-age': 4})
-# app.add_middleware(ContentSecurityPolicy, Option={'default-src': ["'self'"], 'base-uri': ["'self'"], 'block-all-mixed-content': []}, script_nonce=False, style_nonce=False, report_only=False)
-# app.add_middleware(XContentTypeOptions)
-# app.add_middleware(XFrame, Option={'X-Frame-Options': 'DENY'})
-
+app.add_middleware(ContentSecurityPolicy, Option={'default-src': ["'self'"], 'base-uri': ["'self'"], 'block-all-mixed-content': []}, script_nonce=False, style_nonce=False, report_only=False)
+app.add_middleware(XContentTypeOptions)
+app.add_middleware(XFrame, Option={'X-Frame-Options': 'DENY'})
+#app.add_middleware(GZipMiddleware, minimum_size=1000, compresslevel=5)
+app.add_middleware(CustomGZipMiddleware, minimum_size=1000, compresslevel=5,paths=["/sources_list","/url/scan","/extract","/chat_bot","/chunk_entities","/get_neighbours","/graph_query","/schema","/populate_graph_schema","/get_unconnected_nodes_list","/get_duplicate_nodes","/fetch_chunktext"])
 app.add_middleware(
     CORSMiddleware,
     allow_origins=["*"],
@@ -818,5 +848,57 @@ async def calculate_metric(question: str = Form(),
    finally:
        gc.collect()
 
+@app.post("/fetch_chunktext")
+async def fetch_chunktext(
+   uri: str = Form(),
+   database: str = Form(),
+   userName: str = Form(),
+   password: str = Form(),
+   document_name: str = Form(),
+   page_no: int = Form(1)
+):
+   try:
+       payload_json_obj = {
+           'api_name': 'fetch_chunktext',
+           'db_url': uri,
+           'userName': userName,
+           'database': database,
+           'document_name': document_name,
+           'page_no': page_no,
+           'logging_time': formatted_time(datetime.now(timezone.utc))
+       }
+       logger.log_struct(payload_json_obj, "INFO")
+       start = time.time()
+       result = await asyncio.to_thread(
+           get_chunktext_results,
+           uri=uri,
+           username=userName,
+           password=password,
+           database=database,
+           document_name=document_name,
+           page_no=page_no
+       )
+       end = time.time()
+       elapsed_time = end - start
+       json_obj = {
+           'api_name': 'fetch_chunktext',
+           'db_url': uri,
+           'document_name': document_name,
+           'page_no': page_no,
+           'logging_time': formatted_time(datetime.now(timezone.utc)),
+           'elapsed_api_time': f'{elapsed_time:.2f}'
+       }
+       logger.log_struct(json_obj, "INFO")
+       return create_api_response('Success', data=result, message=f"Total elapsed API time {elapsed_time:.2f}")
+   except Exception as e:
+       job_status = "Failed"
+       message = "Unable to get chunk text response"
+       error_message = str(e)
+       logging.exception(f'Exception in fetch_chunktext: {error_message}')
+       return create_api_response(job_status, message=message, error=error_message)
+   finally:
+       gc.collect()
+
+
 if __name__ == "__main__":
     uvicorn.run(app)
@@ -107,24 +107,38 @@
 STORE_COMMUNITY_SUMMARIES = """
 UNWIND $data AS row
 MERGE (c:__Community__ {id:row.community})
-SET c.summary = row.summary
+SET c.summary = row.summary,
+    c.title = row.title
 """ 
 
+
 COMMUNITY_SYSTEM_TEMPLATE = "Given input triples, generate the information summary. No pre-amble."
 
-COMMUNITY_TEMPLATE = """Based on the provided nodes and relationships that belong to the same graph community,
-generate a natural language summary of the provided information:
-{community_info}
 
-Summary:""" 
+COMMUNITY_TEMPLATE = """
+Based on the provided nodes and relationships that belong to the same graph community,
+generate following output in exact format
+title: A concise title, no more than 4 words,
+summary: A natural language summary of the information
+{community_info}
+Example output:
+title: Example Title,
+summary: This is an example summary that describes the key information of this community.
+"""
 
 PARENT_COMMUNITY_SYSTEM_TEMPLATE = "Given an input list of community summaries, generate a summary of the information"
 
 PARENT_COMMUNITY_TEMPLATE = """Based on the provided list of community summaries that belong to the same graph community, 
-generate a natural language summary of the information.Include all the necessary information as possible
+generate following output in exact format
+title: A concise title, no more than 4 words,
+summary: A natural language summary of the information. Include all the necessary information as much as possible.
+
 {community_info}
 
-Summary:""" 
+Example output:
+title: Example Title,
+summary: This is an example summary that describes the key information of this community.
+""" 
 
 
 GET_COMMUNITY_DETAILS = """
@@ -277,8 +291,17 @@ def process_community_info(community, chain, is_parent=False):
             combined_text = " ".join(f"Summary {i+1}: {summary}" for i, summary in enumerate(community.get("texts", [])))
         else:
             combined_text = prepare_string(community)
-        summary = chain.invoke({'community_info': combined_text})
-        return {"community": community['communityId'], "summary": summary}
+        summary_response = chain.invoke({'community_info': combined_text})
+        lines = summary_response.splitlines()
+        title = "Untitled Community"
+        summary = ""
+        for line in lines:
+            if line.lower().startswith("title"):
+                title = line.split(":", 1)[-1].strip()
+            elif line.lower().startswith("summary"):
+                summary = line.split(":", 1)[-1].strip()     
+        logging.info(f"Community Title : {title}")
+        return {"community": community['communityId'], "title":title, "summary": summary}
     except Exception as e:
         logging.error(f"Failed to process community {community.get('communityId', 'unknown')}: {e}")
         return None
@@ -291,7 +314,7 @@ def create_community_summaries(gds, model):
         summaries = []
         with ThreadPoolExecutor() as executor:
             futures = [executor.submit(process_community_info, community, community_chain) for community in community_info_list.to_dict(orient="records")]
-            
+   
             for future in as_completed(futures):
                 result = future.result()
                 if result:
@@ -482,9 +505,3 @@ def create_communities(uri, username, password, database,model=COMMUNITY_CREATIO
             logging.warning("Failed to write communities. Constraint was not applied.")
     except Exception as e:
         logging.error(f"Failed to create communities: {e}")
-
-
-
-
-
-
 
@@ -354,7 +354,7 @@ def get_duplicate_nodes_list(self):
         score_value = float(os.environ.get('DUPLICATE_SCORE_VALUE'))
         text_distance = int(os.environ.get('DUPLICATE_TEXT_DISTANCE'))
         query_duplicate_nodes = """
-                MATCH (n:!Chunk&!Session&!Document&!`__Community__`) with n 
+                MATCH (n:!Chunk&!Session&!Document&!`__Community__`&!`__Entity__`) with n 
                 WHERE n.embedding is not null and n.id is not null // and size(toString(n.id)) > 3
                 WITH n ORDER BY count {{ (n)--() }} DESC, size(toString(n.id)) DESC // updated
                 WITH collect(n) as nodes
 
@@ -3,7 +3,7 @@
 from neo4j import GraphDatabase
 import os
 import json
-from src.shared.constants import GRAPH_CHUNK_LIMIT,GRAPH_QUERY
+from src.shared.constants import GRAPH_CHUNK_LIMIT,GRAPH_QUERY,CHUNK_TEXT_QUERY,COUNT_CHUNKS_QUERY
 # from neo4j.debug import watch
 
 # watch("neo4j")
@@ -226,3 +226,34 @@ def get_graph_results(uri, username, password,database,document_names):
         driver.close()
 
 
+def get_chunktext_results(uri, username, password, database, document_name, page_no):
+   """Retrieves chunk text, position, and page number from graph data with pagination."""
+   try:
+       logging.info("Starting chunk text query process")
+       offset = 10
+       skip = (page_no - 1) * offset
+       limit = offset
+       driver = GraphDatabase.driver(uri, auth=(username, password))
+       with driver.session(database=database) as session:
+           total_chunks_result = session.run(COUNT_CHUNKS_QUERY, file_name=document_name)
+           total_chunks = total_chunks_result.single()["total_chunks"]
+           total_pages = (total_chunks + offset - 1) // offset  # Calculate total pages
+           records = session.run(CHUNK_TEXT_QUERY, file_name=document_name, skip=skip, limit=limit)
+           pageitems = [
+               {
+                   "text": record["chunk_text"],
+                   "position": record["chunk_position"],
+                   "pagenumber": record["page_number"]
+               }
+               for record in records
+           ]
+           logging.info(f"Query process completed with {len(pageitems)} chunks retrieved")
+           return {
+               "pageitems": pageitems,
+               "total_pages": total_pages
+           }
+   except Exception as e:
+       logging.error(f"An error occurred in get_chunktext_results. Error: {str(e)}")
+       raise Exception("An error occurred in get_chunktext_results. Please check the logs for more details.") from e
+   finally:
+       driver.close()
@@ -20,7 +20,8 @@
             labels: [coalesce(apoc.coll.removeAll(labels(node), ['__Entity__'])[0], "*")],
             element_id: elementId(node),
             properties: { 
-                id: CASE WHEN node.id IS NOT NULL THEN node.id ELSE node.fileName END
+                id: CASE WHEN node.id IS NOT NULL THEN node.id ELSE node.fileName END,
+                title:  CASE WHEN node.title IS NOT NULL THEN node.title ELSE " " END
             }
         }
     ] AS nodes,
 
@@ -161,6 +161,19 @@
     ] AS entities
 """
 
+COUNT_CHUNKS_QUERY = """
+MATCH (d:Document {fileName: $file_name})<-[:PART_OF]-(c:Chunk)
+RETURN count(c) AS total_chunks
+"""
+
+CHUNK_TEXT_QUERY = """
+MATCH (d:Document {fileName: $file_name})<-[:PART_OF]-(c:Chunk)
+RETURN c.text AS chunk_text, c.position AS chunk_position, c.page_number AS page_number
+ORDER BY c.position
+SKIP $skip
+LIMIT $limit
+"""
+
 ## CHAT SETUP
 CHAT_MAX_TOKENS = 1000
 CHAT_SEARCH_KWARG_SCORE_THRESHOLD = 0.5
 
@@ -53,7 +53,6 @@ services:
       args:
         - VITE_BACKEND_API_URL=${VITE_BACKEND_API_URL-http://localhost:8000}
         - VITE_REACT_APP_SOURCES=${VITE_REACT_APP_SOURCES-local,wiki,s3}
-        - VITE_LLM_MODELS=${VITE_LLM_MODELS-}
         - VITE_GOOGLE_CLIENT_ID=${VITE_GOOGLE_CLIENT_ID-}
         - VITE_BLOOM_URL=${VITE_BLOOM_URL-https://workspace-preview.neo4j.io/workspace/explore?connectURL={CONNECT_URL}&search=Show+me+a+graph&featureGenAISuggestions=true&featureGenAISuggestionsInternal=true}
         - VITE_TIME_PER_PAGE=${VITE_TIME_PER_PAGE-50}
@@ -62,6 +61,7 @@ services:
         - VITE_ENV=${VITE_ENV-DEV}
         - VITE_CHAT_MODES=${VITE_CHAT_MODES-}
         - VITE_BATCH_SIZE=${VITE_BATCH_SIZE-2}
+        - VITE_LLM_MODELS=${VITE_LLM_MODELS-}
         - VITE_LLM_MODELS_PROD=${VITE_LLM_MODELS_PROD-openai_gpt_4o,openai_gpt_4o_mini,diffbot,gemini_1.5_flash}
     volumes:
       - ./frontend:/app
 
@@ -24,7 +24,6 @@ ENTITY_EMBEDDING=True
 VITE_BACKEND_API_URL="http://localhost:8000"
 VITE_BLOOM_URL="https://workspace-preview.neo4j.io/workspace/explore?connectURL={CONNECT_URL}&search=Show+me+a+graph&featureGenAISuggestions=true&featureGenAISuggestionsInternal=true"
 VITE_REACT_APP_SOURCES="local,youtube,wiki,s3,web"
-VITE_LLM_MODELS="diffbot,openai-gpt-3.5,openai-gpt-4o" # ",ollama_llama3"
 VITE_ENV="DEV"
 VITE_TIME_PER_PAGE=50
 VITE_CHUNK_SIZE=5242880
 
@@ -20,7 +20,6 @@ RUN yarn install
 COPY . ./
 RUN VITE_BACKEND_API_URL=$VITE_BACKEND_API_URL \
     VITE_REACT_APP_SOURCES=$VITE_REACT_APP_SOURCES \
-    VITE_LLM_MODELS=$VITE_LLM_MODELS \
     VITE_GOOGLE_CLIENT_ID=$VITE_GOOGLE_CLIENT_ID \
     VITE_BLOOM_URL=$VITE_BLOOM_URL \
     VITE_CHUNK_SIZE=$VITE_CHUNK_SIZE \
 
@@ -261,4 +261,4 @@ const ChunkInfo: FC<ChunkProps> = ({ loading, chunks, mode }) => {
     </>
   );
 };
-export default ChunkInfo;
+export default ChunkInfo;
@@ -75,4 +75,4 @@ const CommunitiesInfo: FC<CommunitiesProps> = ({ loading, communities, mode }) =
   );
 };
 
-export default CommunitiesInfo;
+export default CommunitiesInfo;
@@ -146,4 +146,4 @@ const EntitiesInfo: FC<EntitiesProps> = ({ loading, mode, graphonly_entities, in
     </>
   );
 };
-export default EntitiesInfo;
+export default EntitiesInfo;
@@ -106,7 +106,7 @@ const SourcesInfo: FC<SourcesProps> = ({ loading, mode, chunks, sources }) => {
                       </>
                     )}
                     {!link?.startsWith('s3://') &&
-                      !isAllowedHost(link, ['storage.googleapis.com', 'wikipedia.org', 'youtube.com']) && (
+                      !isAllowedHost(link, ['storage.googleapis.com', 'wikipedia.org', 'www.youtube.com']) && (
                         <div className='flex flex-row inline-block justify-between items-center'>
                           <GlobeAltIconOutline className='n-size-token-7' />
                           <TextLink href={link} externalLink={true}>
 
@@ -37,4 +37,4 @@ export const handleGraphNodeClick = async (
       setLoadingGraphView(false);
     }
   }
-};
+};
Original file line number	Diff line number	Diff line change
`@@ -20,7 +20,8 @@`
`20`	`20`	`labels: [coalesce(apoc.coll.removeAll(labels(node), ['__Entity__'])[0], "*")],`
`21`	`21`	`element_id: elementId(node),`
`22`	`22`	`properties: {`
`23`		`- id: CASE WHEN node.id IS NOT NULL THEN node.id ELSE node.fileName END`
	`23`	`+ id: CASE WHEN node.id IS NOT NULL THEN node.id ELSE node.fileName END,`
	`24`	`+ title: CASE WHEN node.title IS NOT NULL THEN node.title ELSE " " END`
`24`	`25`	`}`
`25`	`26`	`}`
`26`	`27`	`] AS nodes,`
Original file line number	Diff line number	Diff line change
`@@ -37,4 +37,4 @@ export const handleGraphNodeClick = async (`
`37`	`37`	`setLoadingGraphView(false);`
`38`	`38`	`}`
`39`	`39`	`}`
`40`		`-};`
	`40`	`+};`