Merge pull request #214 from MathisVerstrepen/feat/gitlab

feat: Expand Github node support for Gitlab

Author: MathisVerstrepen

api/app/database/pg/token_ops/provider_token_crud.py

@@ -10,33 +10,41 @@
 logger = logging.getLogger("uvicorn.error")
 
 
-async def store_github_token_for_user(
-    pg_engine: SQLAlchemyAsyncEngine, user_id: str, encrypted_token: str
+async def store_provider_token(
+    pg_engine: SQLAlchemyAsyncEngine, user_id: str, provider: str, encrypted_token: str
 ):
     async with AsyncSession(pg_engine) as session:
-        db_token = ProviderToken(user_id=user_id, provider="github", access_token=encrypted_token)
+        db_token = ProviderToken(user_id=user_id, provider=provider, access_token=encrypted_token)
         session.add(db_token)
         await session.commit()
         await session.refresh(db_token)
         return db_token
 
 
 async def get_provider_token(
-    pg_engine: SQLAlchemyAsyncEngine, user_id: str, provider: str
+    pg_engine: SQLAlchemyAsyncEngine, user_id: str, provider_prefix: str
 ) -> Optional[ProviderToken]:
     async with AsyncSession(pg_engine) as session:
         stmt = select(ProviderToken).where(
-            and_(ProviderToken.user_id == user_id, ProviderToken.provider == provider)
+            and_(
+                ProviderToken.user_id == user_id,
+                ProviderToken.provider.like(f"{provider_prefix}%"),  # type: ignore
+            )
         )
         result = await session.exec(stmt)  # type: ignore
         provider_token: ProviderToken = result.scalar_one_or_none()
         return provider_token if provider_token else None
 
 
-async def delete_provider_token(pg_engine: SQLAlchemyAsyncEngine, user_id: str, provider: str):
+async def delete_provider_token(
+    pg_engine: SQLAlchemyAsyncEngine, user_id: str, provider_prefix: str
+):
     async with AsyncSession(pg_engine) as session:
         stmt = select(ProviderToken).where(
-            and_(ProviderToken.user_id == user_id, ProviderToken.provider == provider)
+            and_(
+                ProviderToken.user_id == user_id,
+                ProviderToken.provider.like(f"{provider_prefix}%"),  # type: ignore
+            )
         )
         result = await session.exec(stmt)  # type: ignore
         token = result.scalar_one_or_none()

api/app/main.py

@@ -17,7 +17,7 @@
 from fastapi.responses import JSONResponse
 from fastapi.staticfiles import StaticFiles
 from models.usersDTO import SettingsDTO
-from routers import chat, files, github, graph, models, users
+from routers import chat, files, github, gitlab, graph, models, repository, users
 from sentry_sdk.integrations.fastapi import FastApiIntegration
 from sentry_sdk.integrations.httpx import HttpxIntegration
 from sentry_sdk.integrations.sqlalchemy import SqlalchemyIntegration
@@ -182,6 +182,8 @@ async def generic_exception_handler(request: Request, exc: Exception):
 app.include_router(models.router)
 app.include_router(users.router)
 app.include_router(github.router)
+app.include_router(gitlab.router)
+app.include_router(repository.router)
 app.include_router(files.router)
 
 app.mount("/static", StaticFiles(directory="data"), name="data")

api/app/models/github.py

@@ -25,15 +25,3 @@ class FileTreeNode(BaseModel):
     type: str  # "file" or "directory"
     path: str
     children: list["FileTreeNode"] = []
-
-
-class GithubCommitInfo(BaseModel):
-    hash: str
-    author: str
-    date: datetime
-
-
-class GithubCommitState(BaseModel):
-    latest_local: GithubCommitInfo
-    latest_online: GithubCommitInfo
-    is_up_to_date: bool

api/app/models/repository.py

@@ -0,0 +1,27 @@
+from datetime import datetime
+from typing import Optional
+
+from pydantic import BaseModel
+
+
+class RepositoryInfo(BaseModel):
+    provider: str
+    encoded_provider: str
+    full_name: str
+    description: Optional[str] = None
+    clone_url_ssh: str
+    clone_url_https: str
+    default_branch: str
+    stargazers_count: Optional[int] = None
+
+
+class GitCommitInfo(BaseModel):
+    hash: str
+    author: str
+    date: datetime
+
+
+class GitCommitState(BaseModel):
+    latest_local: GitCommitInfo
+    latest_online: GitCommitInfo
+    is_up_to_date: bool

api/app/routers/github.py

@@ -1,29 +1,15 @@
-import asyncio
 import logging
 import os
 from urllib.parse import urlencode
 
 import httpx
-from database.pg.token_ops.provider_token_crud import (
-    delete_provider_token,
-    store_github_token_for_user,
-)
+from database.pg.token_ops.provider_token_crud import delete_provider_token, store_provider_token
 from fastapi import APIRouter, Depends, HTTPException, Request, status
-from models.github import GithubCommitState, GitHubStatusResponse, Repo
+from models.github import GitHubStatusResponse, Repo
 from pydantic import BaseModel, ValidationError
 from services.auth import get_current_user_id
 from services.crypto import encrypt_api_key
-from services.github import (
-    CLONED_REPOS_BASE_DIR,
-    build_file_tree_for_branch,
-    clone_repo,
-    get_files_content_for_branch,
-    get_github_access_token,
-    get_latest_local_commit_info,
-    get_latest_online_commit_info,
-    list_branches,
-    pull_repo,
-)
+from services.github import get_github_access_token
 from slowapi import Limiter
 from slowapi.util import get_remote_address
 from starlette.responses import RedirectResponse
@@ -85,7 +71,6 @@ async def github_callback(
             detail="GitHub OAuth is not configured on the server.",
         )
 
-    # Exchange the code for an access token
     async with httpx.AsyncClient() as client:
         token_response = await client.post(
             "https://github.com/login/oauth/access_token",
@@ -128,11 +113,13 @@ async def github_callback(
     github_user = user_response.json()
     github_username = github_user.get("login")
 
-    # Encrypt + store the token
     encrypted_token = await encrypt_api_key(access_token)
-    await delete_provider_token(request.app.state.pg_engine, user_id, "github")
+    if not encrypted_token:
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail="Failed to secure token."
+        )
 
-    await store_github_token_for_user(request.app.state.pg_engine, user_id, encrypted_token or "")
+    await store_provider_token(request.app.state.pg_engine, user_id, "github", encrypted_token)
 
     return {
         "message": "GitHub account connected successfully.",
@@ -180,7 +167,7 @@ async def get_github_connection_status(
 @router.get("/github/repos", response_model=list[Repo])
 async def get_github_repos(request: Request, user_id: str = Depends(get_current_user_id)):
     """
-    Fetches repositories using GitHub App permissions.
+    Fetches repositories using GitHub App permissions via the API.
     """
     access_token = await get_github_access_token(request, user_id)
 
@@ -217,158 +204,3 @@ async def get_github_repos(request: Request, user_id: str = Depends(get_current_
             status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
             detail=f"Failed to parse repository data: {e}",
         )
-
-
-@router.get("/github/repos/{owner}/{repo}/branches")
-async def get_github_repo_branches(
-    owner: str,
-    repo: str,
-    request: Request,
-    user_id: str = Depends(get_current_user_id),
-):
-    """
-    Get a list of branches for a GitHub repository.
-    """
-    access_token = await get_github_access_token(request, user_id)
-    repo_dir = CLONED_REPOS_BASE_DIR / owner / repo
-
-    if not repo_dir.exists():
-        try:
-            await clone_repo(owner, repo, access_token or "", repo_dir)
-        except Exception as e:
-            logger.error(f"Error cloning repo: {e}")
-            raise HTTPException(
-                status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
-                detail=f"Failed to clone repository: {str(e)}",
-            )
-
-    try:
-        branches = await list_branches(repo_dir)
-        return branches
-    except Exception as e:
-        logger.error(f"Error listing branches: {e}")
-        raise HTTPException(
-            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
-            detail=f"Failed to list branches: {str(e)}",
-        )
-
-
-@router.get("/github/repos/{owner}/{repo}/tree")
-async def get_github_repo_tree(
-    owner: str,
-    repo: str,
-    branch: str,
-    request: Request,
-    user_id: str = Depends(get_current_user_id),
-    force_pull: bool = False,
-):
-    """
-    Get file tree structure for a specific branch of a GitHub repository.
-    Clones the repo if not already cloned locally.
-    """
-    access_token = await get_github_access_token(request, user_id)
-    repo_dir = CLONED_REPOS_BASE_DIR / owner / repo
-
-    # Clone repo if it doesn't exist
-    if not repo_dir.exists():
-        try:
-            await clone_repo(owner, repo, access_token or "", repo_dir)
-        except Exception as e:
-            logger.error(f"Error cloning repo: {e}")
-            raise HTTPException(
-                status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
-                detail=f"Failed to clone repository: {str(e)}",
-            )
-
-    if force_pull:
-        try:
-            await pull_repo(repo_dir, branch)
-        except Exception as e:
-            logger.error(f"Error pulling repo: {e}")
-            raise HTTPException(
-                status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
-                detail=f"Failed to pull repository: {str(e)}",
-            )
-
-    # Build file tree structure for the specific branch
-    try:
-        tree = await build_file_tree_for_branch(repo_dir, branch)
-        return tree
-    except FileNotFoundError as e:
-        raise HTTPException(
-            status_code=status.HTTP_404_NOT_FOUND,
-            detail=str(e),
-        )
-    except Exception as e:
-        logger.error(f"Error building file tree for branch {branch}: {e}")
-        raise HTTPException(
-            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
-            detail=f"Failed to build file tree: {str(e)}",
-        )
-
-
-@router.get("/github/repos/{owner}/{repo}/commit/state")
-async def get_github_repo_commit_state(
-    owner: str,
-    repo: str,
-    branch: str,
-    request: Request,
-    user_id: str = Depends(get_current_user_id),
-) -> GithubCommitState:
-    """
-    Get the state of a GitHub repository branch.
-    Return latest commit information and if the cloned version is latest
-    """
-    access_token = await get_github_access_token(request, user_id)
-
-    repo_dir = CLONED_REPOS_BASE_DIR / owner / repo
-    repo_id = f"{owner}/{repo}"
-
-    latest_local_commit_info, latest_online_commit_info = await asyncio.gather(
-        get_latest_local_commit_info(repo_dir, branch),
-        get_latest_online_commit_info(repo_id, access_token, branch),
-    )
-
-    return GithubCommitState(
-        latest_local=latest_local_commit_info,
-        latest_online=latest_online_commit_info,
-        is_up_to_date=latest_local_commit_info.hash == latest_online_commit_info.hash,
-    )
-
-
-@router.get("/github/repos/{owner}/{repo}/contents/{file_path:path}")
-async def get_github_repo_file(
-    owner: str,
-    repo: str,
-    file_path: str,
-    branch: str,
-    request: Request,
-    user_id: str = Depends(get_current_user_id),
-):
-    """
-    Get the content of a file in a specific branch of a GitHub repository.
-    """
-    await get_github_access_token(request, user_id)
-
-    repo_dir = CLONED_REPOS_BASE_DIR / owner / repo
-
-    if not repo_dir.exists():
-        raise HTTPException(
-            status_code=status.HTTP_404_NOT_FOUND,
-            detail="Repository not found locally. Please select it first to clone.",
-        )
-
-    try:
-        content = await get_files_content_for_branch(repo_dir, branch, [file_path])
-        return {"content": content.get(file_path, "")}
-    except FileNotFoundError as e:
-        raise HTTPException(
-            status_code=status.HTTP_404_NOT_FOUND,
-            detail=str(e),
-        )
-    except Exception as e:
-        logger.error(f"Error reading file content: {e}")
-        raise HTTPException(
-            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
-            detail=f"Failed to read file content: {str(e)}",
-        )