diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..ad5eecbf --- /dev/null +++ b/.dockerignore @@ -0,0 +1,11 @@ +venv/ +.pytest_cache/ +.ruff_cache/ +.vscode/ +data_neo4j/ +data_postgres/ +.coverage +.env +config.json +__pycache__/ +settings.toml \ No newline at end of file diff --git a/.gitignore b/.gitignore index f4a119d5..ad5eecbf 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,10 @@ venv/ .pytest_cache/ .ruff_cache/ .vscode/ +data_neo4j/ +data_postgres/ .coverage +.env config.json -__pycache__/ \ No newline at end of file +__pycache__/ +settings.toml \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..0393f00d --- /dev/null +++ b/Dockerfile @@ -0,0 +1,17 @@ +FROM python:3.11-slim + +ENV PYTHONUNBUFFERED=1 + +RUN apt-get update && apt-get install -y git + +WORKDIR /app + +COPY . /app + +RUN pip install --upgrade pip \ + && pip install hatchling \ + && pip install . + +EXPOSE 8000 + +CMD ["uvicorn", "prometheus.app.main:app", "--host", "0.0.0.0", "--port", "8000"] \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..4546ff77 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,61 @@ +version: "3.9" + +services: + neo4j: + image: neo4j:5.20.0 + container_name: neo4j_container + environment: + - NEO4J_AUTH=neo4j/password + - NEO4J_PLUGINS=["apoc"] + ports: + - "7474:7474" + - "7687:7687" + volumes: + - ./data_neo4j:/data + healthcheck: + test: ["CMD-SHELL", "neo4j status || exit 1"] + interval: 15s + timeout: 30s + retries: 3 + + + postgres: + image: postgres + container_name: postgres_container + user: postgres + environment: + - POSTGRES_USER=postgres + - POSTGRES_PASSWORD=password + ports: + - "5432:5432" + volumes: + - ./data_postgres:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready"] + interval: 15s + timeout: 30s + retries: 3 + + app: + build: . + container_name: prometheus + ports: + - "8000:8000" + environment: + - PROMETHEUS_NEO4J_URI="bolt://neo4j:7687" + - PROMETHEUS_NEO4J_USERNAME=neo4j + - PROMETHEUS_NEO4J_PASSWORD=password + - PROMETHEUS_NEO4J_BATCH_SIZE=1000 + - PROMETHEUS_KNOWLEDGE_GRAPH_MAX_AST_DEPTH=5 + - PROMETHEUS_LITELLM_MODEL=${PROMETHEUS_LITELLM_MODEL} + - PROMETHEUS_LITELLM_ANTHROPIC_API_KEY=${PROMETHEUS_LITELLM_ANTHROPIC_API_KEY} + - PROMETHEUS_WORKING_DIRECTORY="/app/working_dir" + - PROMETHEUS_GITHUB_ACCESS_TOKEN=${PROMETHEUS_GITHUB_ACCESS_TOKEN} + - PROMETHEUS_POSTGRES_URI="postgresql://postgres:password@postgres:5432/postgres?sslmode=disable" + volumes: + - .:/app + depends_on: + neo4j: + condition: service_healthy + postgres: + condition: service_healthy \ No newline at end of file diff --git a/prometheus/app/shared_state.py b/prometheus/app/shared_state.py index d82be4bd..2c212c5a 100644 --- a/prometheus/app/shared_state.py +++ b/prometheus/app/shared_state.py @@ -9,7 +9,7 @@ from psycopg import Connection from psycopg.rows import dict_row -from prometheus.configuration import config +from prometheus.configuration.config import settings from prometheus.git.git_repository import GitRepository from prometheus.graph.knowledge_graph import KnowledgeGraph from prometheus.lang_graph.subgraphs import context_provider_subgraph, issue_answer_subgraph @@ -21,16 +21,18 @@ class SharedState: def __init__(self): self.kg = None self.neo4j_driver = GraphDatabase.driver( - config.config["neo4j"]["uri"], - auth=(config.config["neo4j"]["username"], config.config["neo4j"]["password"]), + settings.NEO4J_URI, + auth=(settings.NEO4J_USERNAME, settings.NEO4J_PASSWORD), ) self.kg_handler = knowledge_graph_handler.KnowledgeGraphHandler( self.neo4j_driver, - config.config["neo4j"]["batch_size"], + settings.NEO4J_BATCH_SIZE, + ) + self.model = ChatLiteLLM( + model=settings.LITELLM_MODEL, anthropic_api_key=settings.LITELLM_ANTHROPIC_API_KEY ) - self.model = ChatLiteLLM(**config.config["litellm"]) self.postgres_conn = Connection.connect( - config.config["postgres"]["db_uri"], + settings.POSTGRES_URI, autocommit=True, prepare_threshold=0, row_factory=dict_row, @@ -39,7 +41,7 @@ def __init__(self): self.checkpointer.setup() self.cp_subgraph = None self.ia_subgraph = None - self.git_repo = GitRepository(config.config["github"]["access_token"]) + self.git_repo = GitRepository(settings.GITHUB_ACCESS_TOKEN) self._logger = logging.getLogger("prometheus.app.shared_state") @@ -78,7 +80,7 @@ def get_all_conversation_messages(self, conversation_id: str) -> Sequence[Mappin return postgres_util.get_messages(self.checkpointer, conversation_id) def upload_local_repository(self, path: Path): - kg = KnowledgeGraph(config.config["knowledge_graph"]["max_ast_depth"]) + kg = KnowledgeGraph(settings.KNOWLEDGE_GRAPH_MAX_AST_DEPTH) kg.build_graph(path) self.kg = kg self.kg_handler.write_knowledge_graph(kg) @@ -90,10 +92,10 @@ def upload_local_repository(self, path: Path): ) def upload_github_repository(self, https_url: str): - target_directory = Path(config.config["prometheus"]["working_directory"]) / "repositories" + target_directory = Path(settings.WORKING_DIRECTORY) / "repositories" target_directory.mkdir(parents=True, exist_ok=True) saved_path = self.git_repo.clone_repository(https_url, target_directory) - kg = KnowledgeGraph(config.config["knowledge_graph"]["max_ast_depth"]) + kg = KnowledgeGraph(settings.KNOWLEDGE_GRAPH_MAX_AST_DEPTH) kg.build_graph(saved_path) self.kg = kg self.kg_handler.write_knowledge_graph(kg) diff --git a/prometheus/configuration/config.py b/prometheus/configuration/config.py index b7e26340..6111294a 100644 --- a/prometheus/configuration/config.py +++ b/prometheus/configuration/config.py @@ -1,25 +1,14 @@ -import json -import os from pathlib import Path -CONFIG_FILE = Path(__file__).parent / "config.json" +from dynaconf import Dynaconf +settings_file = Path(__file__).resolve().parent / "settings.toml" -def load_config(): - if not CONFIG_FILE.exists(): - return {} +settings = Dynaconf( + envvar_prefix="PROMETHEUS", + settings_files=[str(settings_file)], + environments=True, +) - with open(CONFIG_FILE, "r") as f: - config_data = json.load(f) - - if "litellm" in config_data: - if "anthropic_api_key" in config_data["litellm"]: - os.environ["ANTHROPIC_API_KEY"] = config_data["litellm"]["anthropic_api_key"] - elif "azure_api_key" in config_data["litellm"]: - os.environ["AZURE_API_KEY"] = config_data["litellm"]["azure_api_key"] - elif "openai_api_key" in config_data["litellm"]: - os.environ["OPENAI_API_KEY"] = config_data["litellm"]["openai_api_key"] - return config_data - - -config = load_config() +# `envvar_prefix` = export envvars with `export DYNACONF_FOO=bar`. +# `settings_files` = Load these files in the order. diff --git a/prometheus/configuration/example_config.json b/prometheus/configuration/example_config.json deleted file mode 100644 index 69f4dbc6..00000000 --- a/prometheus/configuration/example_config.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "neo4j": { - "uri": "CHANGE_ME", - "username": "CHANGE_ME", - "password": "CHANGE_ME", - "batch_size": 1000 - }, - "knowledge_graph": { - "max_ast_depth": 5 - }, - "litellm": { - "model": "CHANGE_ME", - "anthropic_api_key": "CHANGE_ME" - }, - "github": { - "token": "CHANGE_ME" - }, - "postgres": { - "db_uri": "CHANGE_ME" - } -} \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 189063bd..f7af20c9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,12 +12,14 @@ dependencies = [ "neo4j==5.20.0", "fastapi[standard]>=0.115.2", "langchain-anthropic>=0.2.3", + "langchain_community>=0.3.2", "igittigitt>=2.1.5", "litellm>=1.50.1", "GitPython>=3.1.43", "langgraph>=0.2.41", "langgraph-checkpoint-postgres>=2.0.2", "psycopg[binary]>=3.2.3", + "dynaconf>=3.2.6", ] requires-python = ">= 3.11" @@ -27,7 +29,6 @@ test = [ "pytest>=8.3.3", "pytest-cov>=5.0.0", "testcontainers==4.8.2", - "langchain_community>=0.3.2", ] [tool.ruff]