Merge pull request #35 from FalkorDB/process-local-dir

Process local folder

Author: swilly22

.gitignore

@@ -1,77 +1,44 @@
-# Python
+# Byte-compiled files
 *.pyc
 *.pyo
 *.pyd
 __pycache__/
-*.env
-.venv/
-env/
-venv/
-ENV/
-*.egg-info/
-*.egg
-pip-log.txt
-
-# Flask
-instance/
-*.sqlite3
-*.db
-*.bak
-instance/config.py
-instance/*.secret
-
-# Environment Variables
-.env
-.env.*
 
-# Logs
-logs/
-*.log
-*.out
-*.err
+# Virtual environments
+venv/
+env/
+.virtualenv/
 
-# Pytest and Coverage
-.pytest_cache/
-.coverage
-.tox/
-nosetests.xml
-coverage.xml
-*.cover
-.cache
+# Distribution/build files
+build/
+dist/
+*.egg-info/
+.eggs/
 
-# IDEs and Editors
+# IDE settings
 .vscode/
-.idea/.env
-.vercel
-
-# ignore the .pyc files
-*.pyc
-
-dist/
-# Node (if using npm/yarn for assets)
-node_modules/
-*.lock
+.idea/
+*.swp
 
-# Static assets (if generated)
-static/
-dist/
-build/
+# Logs and debugging
+*.log
+*.trace
 
-# Docker
-*.pid
-docker-compose.override.yml
+# OS-specific files
+.DS_Store
+Thumbs.db
 
-# Heroku
-*.buildpacks
-*.env.local
-*.env.production
-*.env.*.local
+# Testing and coverage
+htmlcov/
+*.cover
+.coverage
+.cache/
+pytest_cache/
 
-# Miscellaneous
-*.bak
-*.tmp
-*.log.*
-Thumbs.db
+# Jupyter Notebook checkpoints
+.ipynb_checkpoints/
 
-# Vercel
-.vercel
+# Custom settings
+.env
+*.sqlite3
+.vercel

README.md

@@ -5,25 +5,44 @@
 ## Getting Started
 [Live Demo](https://code-graph.falkordb.com/)
 
+## Running locally
 
+### Run FalkorDB
+Free cloud instance: https://app.falkordb.cloud/signup
+
+Or by running locally with docker:
 ```bash
-flask --app code_graph run --debug
+docker run -p 6379:6379 -p 3000:3000 -it --rm falkordb/falkordb:latest
 ```
 
-Process local git repository, ignoring specific folder(s)
+### Config
+Create your own `.env` file from the `.env.template` file
 
+Start the server:
 ```bash
-curl -X POST http://127.0.0.1:5000/process_local_repo -H "Content-Type: application/json" -d '{"repo": "/Users/roilipman/Dev/FalkorDB", "ignore": ["./.github", "./sbin", "./.git","./deps", "./bin", "./build"]}'
+flask --app api/index.py run --debug
 ```
 
-Process code coverage
-
+### Creating a graph
+Process a local source folder:
 ```bash
-curl -X POST http://127.0.0.1:5000/process_code_coverage -H "Content-Type: application/json" -d '{"lcov": "/Users/roilipman/Dev/code_graph/code_graph/code_coverage/lcov/falkordb.lcov", "repo": "FalkorDB"}'
+curl -X POST http://127.0.0.1:5000/analyze_folder -H "Content-Type: application/json" -d '{"path": "<FULL_PATH_TO_FOLDER>", "ignore": [<OPTIONAL_IGNORE_LIST>]}' -H "Authorization: <.ENV_SECRET_TOKEN>"
 ```
 
-Process git information
-
+For example:
 ```bash
-curl -X POST http://127.0.0.1:5000/process_git_history -H "Content-Type: application/json" -d '{"repo": "/Users/roilipman/Dev/falkorDB"}'
+curl -X POST http://127.0.0.1:5000/analyze_folder -H "Content-Type: application/json" -d '{"path": "/Users/roilipman/Dev/GraphRAG-SDK", "ignore": ["./.github", "./build"]}' -H "Authorization: OpenSesame"
 ```
+## Working with your graph
+Once the source code analysis completes your FalkorDB DB will be populated with
+a graph representation of your source code, the graph name should be the same as
+the name of the folder you've requested to analyze, for the example above a graph named:
+"GraphRAG-SDK".
+
+At the moment only the Python and C languages are supported, we do intend to support additional languages.
+
+At this point you can explore and query your source code using various tools
+Here are several options:
+1. FalkorDB built-in UI
+2. One of FalkorDB's [clients](https://docs.falkordb.com/clients.html)
+3. Use FalkorDB [GraphRAG-SDK](https://github.com/FalkorDB/GraphRAG-SDK) to connect an LLM for natural language exploration.

api/analyzers/source_analyzer.py

@@ -135,7 +135,7 @@ def analyze_sources(self, ignore: List[str]) -> None:
             # Second pass analysis of the source code
             self.second_pass(ignore, executor)
 
-    def analyze(self, path: str, g: Graph, ignore: Optional[List[str]] = []) -> None:
+    def analyze_local_folder(self, path: str, g: Graph, ignore: Optional[List[str]] = []) -> None:
         """
         Analyze path.
 
@@ -144,6 +144,8 @@ def analyze(self, path: str, g: Graph, ignore: Optional[List[str]] = []) -> None
             ignore (List(str)): List of paths to skip
         """
 
+        logging.info(f"Analyzing local folder {path}")
+
         # Save original working directory for later restore
         original_dir = Path.cwd()
 
@@ -179,4 +181,4 @@ def analyze_local_repository(self, path: str, ignore: Optional[List[str]] = [])
         self.graph.set_graph_commit(head.hexsha)
 
         return self.graph
-
+    

api/index.py

@@ -1,6 +1,7 @@
 import os
 import datetime
 from api import *
+from pathlib import Path
 from typing import Optional
 from functools import wraps
 from falkordb import FalkorDB
@@ -309,3 +310,57 @@ def chat():
     response = { 'status': 'success', 'response': answer }
 
     return jsonify(response), 200
+
+@app.route('/analyze_folder', methods=['POST'])
+@token_required  # Apply token authentication decorator
+def analyze_folder():
+    """
+    Endpoint to analyze local source code
+    Expects 'path' and optionally an ignore list.
+
+    Returns:
+        JSON response with status and error message if applicable
+        Status codes:
+            200: Success
+            400: Invalid input
+            500: Internal server error
+    """
+
+    # Get JSON data from the request
+    data = request.get_json()
+
+    # Get query parameters
+    path      = data.get('path')
+    ignore    = data.get('ignore', [])
+
+    # Validate input parameters
+    if not path:
+        logging.error("'path' is missing from the request.")
+        return jsonify({"status": "'path' is required."}), 400
+
+    # Validate path exists and is a directory
+    if not os.path.isdir(path):
+        logging.error(f"Path '{path}' does not exist or is not a directory")
+        return jsonify({"status": "Invalid path: must be an existing directory"}), 400
+
+    # Validate ignore list contains valid paths
+    if not isinstance(ignore, list):
+        logging.error("'ignore' must be a list of paths")
+        return jsonify({"status": "'ignore' must be a list of paths"}), 400
+
+    proj_name = Path(path).name
+
+    # Initialize the graph with the provided project name
+    g = Graph(proj_name)
+
+    # Analyze source code within given folder
+    analyzer = SourceAnalyzer()
+    analyzer.analyze_local_folder(path, g, ignore)
+
+    # Return response
+    response = {
+            'status': 'success',
+            'project': proj_name
+        }
+    return jsonify(response), 200
+