Merge pull request #2 from FalkorDB/git-graph

Git graph

Author: swilly22

.env

@@ -0,0 +1,3 @@
+FALKORDB_HOST=localhost
+FALKORDB_PORT=6379
+

README.md

@@ -0,0 +1,11 @@
+flask --app code_graph run --debug
+
+Process local git repository, ignoring specific folder(s)
+
+curl -X POST http://127.0.0.1:5000/process_local_repo -H "Content-Type: application/json" -d '{"repo": "/Users/roilipman/Dev/FalkorDB", "ignore": ["./.github", "./sbin", "./.git","./deps", "./bin", "./build"]}'
+
+Process code coverage
+curl -X POST http://127.0.0.1:5000/process_code_coverage -H "Content-Type: application/json" -d '{"lcov": "/Users/roilipman/Dev/code_graph/code_graph/code_coverage/lcov/falkordb.lcov", "repo": "FalkorDB"}'
+
+Process git information
+curl -X POST http://127.0.0.1:5000/process_git_history -H "Content-Type: application/json" -d '{"repo": "/Users/roilipman/Dev/falkorDB"}'

code_graph/__init__.py

@@ -1,3 +1,10 @@
-from .analyzers.source_analyzer import *
+from .info import *
+from .llm import ask
+from .graph import *
+from .project import *
 from .entities import *
-from .graph import Graph
+from .git_utils import *
+from .app import create_app
+from .code_coverage import *
+from .analyzers.source_analyzer import *
+from .auto_complete import prefix_search

code_graph/analyzers/c/analyzer.py

@@ -1,9 +1,10 @@
 import io
-from typing import Union, Optional
-from pathlib import Path
+import os
 from ..utils import *
+from pathlib import Path
 from ...entities import *
 from ...graph import Graph
+from typing import Union, Optional
 from ..analyzer import AbstractAnalyzer
 
 import tree_sitter_c as tsc
@@ -347,13 +348,17 @@ def first_pass(self, path: Path, f: io.TextIOWrapper, graph:Graph) -> None:
         logger.info(f"Processing {path}")
 
         # Create file entity
-        file = File(str(path.parent), path.name, path.suffix)
+        file = File(os.path.dirname(path), path.name, path.suffix)
         graph.add_file(file)
 
         # Parse file
         source_code = f.read()
         tree = self.parser.parse(source_code)
-        source_code = source_code.decode('utf-8')
+        try:
+            source_code = source_code.decode('utf-8')
+        except Exception as e:
+            logger.error(f"Failed decoding source code: {e}")
+            source_code = ''
 
         # Process function definitions
         query = C_LANGUAGE.query("(function_definition) @function")
@@ -412,7 +417,7 @@ def second_pass(self, path: Path, f: io.TextIOWrapper, graph: Graph) -> None:
         logger.info(f"Processing {path}")
 
         # Get file entity
-        file = graph.get_file(str(path.parent), path.name, path.suffix)
+        file = graph.get_file(os.path.dirname(path), path.name, path.suffix)
         if file is None:
             logger.error(f"File entity not found for: {path}")
             return

code_graph/analyzers/python/analyzer.py

@@ -1,9 +1,10 @@
 import io
-from typing import Union, Optional
-from pathlib import Path
+import os
 from ..utils import *
+from pathlib import Path
 from ...entities import *
 from ...graph import Graph
+from typing import Union, Optional
 from ..analyzer import AbstractAnalyzer
 
 import tree_sitter_python as tspython
@@ -74,7 +75,7 @@ def process_class_definition(self, node: Node, path: Path) -> tuple[Class, list[
 
         return (c, inherited_classes)
 
-    def process_function_definition(self, node: Node, path: Path) -> Function:
+    def process_function_definition(self, node: Node, path: Path, source_code: str) -> Function:
         """
         Processes a function definition node from the syntax tree and extracts relevant information.
 
@@ -153,7 +154,8 @@ def process_function_definition(self, node: Node, path: Path) -> Function:
         ret_type = return_type.text.decode('utf-8') if return_type else None
 
         # Create Function object
-        f = Function(str(path), function_name, docstring, ret_type, '', start_line, end_line)
+        src = source_code[node.start_byte:node.end_byte]
+        f = Function(str(path), function_name, docstring, ret_type, src, start_line, end_line)
 
         # Add arguments to Function object
         for arg in args:
@@ -162,7 +164,7 @@ def process_function_definition(self, node: Node, path: Path) -> Function:
         return f
 
     def first_pass_traverse(self, parent: Union[File,Class,Function], node: Node,
-                            path: Path, graph: Graph) -> None:
+                            path: Path, graph: Graph, source_code: str) -> None:
         """
         Recursively traverses a syntax tree node, processes class and function definitions,
         and connects them in a graph representation.
@@ -197,7 +199,7 @@ def first_pass_traverse(self, parent: Union[File,Class,Function], node: Node,
             graph.add_class(entity)
 
         elif node.type == "function_definition":
-            entity = self.process_function_definition(node, path)
+            entity = self.process_function_definition(node, path, source_code)
             # Add Function object to the graph
             graph.add_function(entity)
 
@@ -208,7 +210,7 @@ def first_pass_traverse(self, parent: Union[File,Class,Function], node: Node,
 
         # Recursivly visit child nodes
         for child in node.children:
-            self.first_pass_traverse(parent, child, path, graph)
+            self.first_pass_traverse(parent, child, path, graph, source_code)
 
     def first_pass(self, path: Path, f: io.TextIOWrapper, graph:Graph) -> None:
         """
@@ -226,15 +228,20 @@ def first_pass(self, path: Path, f: io.TextIOWrapper, graph:Graph) -> None:
         logger.info(f"Python Processing {path}")
 
         # Create file entity
-        file = File(str(path.parent), path.name, path.suffix)
+        file = File(os.path.dirname(path), path.name, path.suffix)
         graph.add_file(file)
 
         # Parse file
-        content = f.read()
-        tree = self.parser.parse(content)
+        source_code = f.read()
+        tree = self.parser.parse(source_code)
+        try:
+            source_code = source_code.decode('utf-8')
+        except Exception as e:
+            logger.error(f"Failed decoding source code: {e}")
+            source_code = ''
 
         # Walk thought the AST
-        self.first_pass_traverse(file, tree.root_node, path, graph)
+        self.first_pass_traverse(file, tree.root_node, path, graph, source_code)
 
     def process_function_call(self, node) -> Optional[str]:
         """
@@ -323,7 +330,7 @@ def process_inheritance(self, cls: Class, super_classes: list[str],
             graph.connect_entities('INHERITS', cls.id, _super_class.id)
 
     def second_pass_traverse(self, parent: Union[File, Class, Function],
-                             node: Node, path: Path, graph: Graph) -> None:
+                             node: Node, path: Path, graph: Graph, source_code: str) -> None:
         """
         Traverse the AST nodes during the second pass and process each node accordingly.
 
@@ -340,7 +347,9 @@ def second_pass_traverse(self, parent: Union[File, Class, Function],
             parent = cls
 
         elif node.type == "function_definition":
-            func = self.process_function_definition(node, path)
+            # TODO: simply extract function name, no need to parse entire function
+            # see C analyzer
+            func = self.process_function_definition(node, path, source_code)
             parent = graph.get_function_by_name(func.name)
         elif node.type == "call":
             callee = self.process_function_call(node)
@@ -349,7 +358,7 @@ def second_pass_traverse(self, parent: Union[File, Class, Function],
 
         # Recursivly visit child nodes
         for child in node.children:
-            self.second_pass_traverse(parent, child, path, graph)
+            self.second_pass_traverse(parent, child, path, graph, source_code)
 
     def second_pass(self, path: Path, f: io.TextIOWrapper, graph: Graph) -> None:
         """
@@ -367,17 +376,17 @@ def second_pass(self, path: Path, f: io.TextIOWrapper, graph: Graph) -> None:
         logger.info(f"Processing {path}")
 
         # Get file entity
-        file = graph.get_file(str(path.parent), path.name, path.suffix)
+        file = graph.get_file(os.path.dirname(path), path.name, path.suffix)
         if file is None:
             logger.error(f"File entity not found for: {path}")
             return
 
         try:
             # Parse file
-            content = f.read()
-            tree = self.parser.parse(content)
+            source_code = f.read()
+            tree = self.parser.parse(source_code)
 
             # Walk thought the AST
-            self.second_pass_traverse(file, tree.root_node, path, graph)
+            self.second_pass_traverse(file, tree.root_node, path, graph, source_code)
         except Exception as e:
             logger.error(f"Failed to process file {path}: {e}")