process_repo end point clone and build both code and git graphs

Author: swilly22

code_graph/analyzers/source_analyzer.py

@@ -1,5 +1,6 @@
 import os
-import tempfile
+import shutil
+import subprocess
 import concurrent.futures
 
 from git import Repo
@@ -19,13 +20,11 @@
              '.py': PythonAnalyzer()}
 
 class SourceAnalyzer():
-    def __init__(self, host: str = 'localhost', port: int = 6379,
-                 username: Optional[str] = None, password: Optional[str] = None) -> None:
-
-        self.host      = host
-        self.port      = port
-        self.username  = username
-        self.password  = password
+    def __init__(self) -> None:
+        self.host      = os.getenv('FALKORDB_HOST')
+        self.port      = os.getenv('FALKORDB_PORT')
+        self.username  = os.getenv('FALKORDB_USERNAME')
+        self.password  = os.getenv('FALKORDB_PASSWORD')
 
     def first_pass(self, ignore: List[str], executor: concurrent.futures.Executor) -> None:
         """
@@ -126,34 +125,59 @@ def analyze_sources(self, ignore: List[str]) -> None:
             # Second pass analysis of the source code
             self.second_pass(ignore, executor)
 
-    def analyze_github_repository(self, url: str) -> None:
+    def analyze_github_repository(
+        self,
+        url: str,
+        repo_path: Path,
+        repo_name: str,
+        ignore: Optional[List[str]] = []
+    ) -> None:
         """
         Analyze a Git repository given its URL.
 
         Args:
-            url (str): The URL of the Git repository to analyze.
+            url: The URL of the Git repository to analyze
+            ignore_patterns: List of patterns to ignore during analysis
+
+        Raises:
+            subprocess.SubprocessError: If git clone fails
+            OSError: If there are filesystem operation errors
         """
 
-        # Extract repository name from the URL
-        components = url[:url.rfind('.')].split('/')
-        n = len(components)
-        repo_name = f'{components[n-2]}/{components[-1]}'
-        logger.debug(f'repo_name: {repo_name}')
-        #repo_name = url[url.rfind('/')+1:url.rfind('.')]
+        # Extract repository name more reliably
+        # Delete local repository if exists
+        if repo_path.exists():
+            shutil.rmtree(repo_path)
 
-        # Initialize the graph and analyzer
-        self.graph = Graph(repo_name, self.host, self.port, self.username,
-                           self.password)
+        # Create directory
+        repo_path.mkdir(parents=True, exist_ok=True)
+
+        # Clone repository
+        # Prepare the git clone command
+        command = ["git", "clone", url, repo_path]
+
+        # Run the git clone command and wait for it to finish
+        result = subprocess.run(command, check=True, capture_output=True, text=True)
+
+        # Store original working directory
+        original_dir = Path.cwd()
+
+        # change working directory to local repository
+        os.chdir(repo_path)
+
+        try:
+            # Initialize the graph and analyzer
+            self.graph = Graph(repo_name, self.host, self.port, self.username,
+                               self.password)
 
-        # Create a temporary directory for cloning the repository
-        with tempfile.TemporaryDirectory() as temp_dir:
-            logger.info(f"Cloning repository {url} to {temp_dir}")
-            repo = Repo.clone_from(url, temp_dir)
+            # Analyze repository
+            self.analyze_sources(ignore)
 
-            # Analyze source files
-            self.analyze_sources(temp_dir)
+            logging.info(f"Successfully processed repository: {repo_name}")
 
-        logger.info("Done processing repository")
+        finally:
+            # Ensure we always return to the original directory
+            os.chdir(original_dir)
 
     def analyze_local_folder(self, path: str, ignore: Optional[List[str]] = []) -> Graph:
         """

code_graph/git_utils/git_utils.py

@@ -108,14 +108,9 @@ def stop_monitor_effects():
     print("monitor thread exited")
 
 # build a graph capturing the git commit history
-def build_commit_graph(path: str, ignore_list: Optional[List[str]] = []) -> GitGraph:
-    print(f"Processing git history at: {path}")
-    print(f"ignoring the following paths: {ignore_list}")
-
+def build_commit_graph(path: str, repo_name: str, ignore_list: Optional[List[str]] = []) -> GitGraph:
     repo = Repo(path)
 
-    repo_name = os.path.split(os.path.normpath(path))[-1]
-
     # Clone graph into a temporary graph
     g =         Graph(repo_name).clone(repo_name + "_tmp")
     git_graph = GitGraph(GitRepoName(repo_name))

main.py

@@ -112,48 +112,56 @@ def get_neighbors():
 
 @app.route('/process_repo', methods=['POST'])
 def process_repo():
-    # Get JSON data from the request
-    data = request.get_json()
+    """
+    Process a GitHub repository.
 
-    # Process the data
+    Expected JSON payload:
+    {
+        "repo_url": "string",
+        "ignore": ["string"]  # optional
+    }
+
+    Returns:
+        JSON response with processing status
+    """
+
+    data = request.get_json()
     repo_url = data.get('repo_url')
     if repo_url is None:
         return jsonify({'status': f'Missing mandatory parameter "repo_url"'}), 400
     logger.debug(f'Received repo_url: {repo_url}')
 
-    # Validate URL
+    ignore = data.get('ignore', [])
+
+    # Validate and normalize URL
     try:
         urlparse(repo_url)
     except ValueError:
         return jsonify({'status': 'Invalid repository URL'}), 400
 
-    # Extract Organization and Repo name from URL
-    res = extract_org_name_from_url(repo_url)
-    if res is None:
-        return jsonify({'status': f'Failed to process repo_url: {repo_url}'}), 400
-
-    org, name = extract_org_name_from_url(repo_url)
-    logger.debug(f'Org: {org}, name: {name}')
-
     # Convert repo_url to git URL
     git_url = repo_url + '.git'
-    logger.debug(f'git_url: {git_url}')
+    parsed_url = urlparse(git_url)
+    logging.debug(f"Processing git URL: {git_url}")
+
+    repo_name = parsed_url.path.rstrip('.git').split('/')[-1]
+    if not repo_name:
+        raise ValueError(f"Could not extract repository name from URL: {url}")
+
+    base_path = Path("./repositories")
+    repo_path = base_path / repo_name
+    logging.debug(f"Repository name: {repo_name}")
 
     # Create source code analyzer
-    analyzer = SourceAnalyzer(host     = FALKORDB_HOST,
-                              port     = FALKORDB_PORT,
-                              username = FALKORDB_USERNAME,
-                              password = FALKORDB_PASSWORD)
+    analyzer = SourceAnalyzer()
 
     try:
-        analyzer.analyze_repository(git_url)
+        analyzer.analyze_github_repository(git_url, repo_path, repo_name, ignore)
+        build_commit_graph(repo_path, repo_name, ignore)
     except Exception as e:
         logger.error(f'An error occurred: {e}')
         return jsonify({'status': f'Failed to process repository: {git_url}'}), 400
 
-    repo_name = f'{org}/{name}'
-    save_repository_metadata(git_url, repo_name)
-
     # Create a response
     response = {
         'status': 'success',