🚧 WIP

Author: Dmytro Parfeniuk

.env.example

@@ -0,0 +1,8 @@
+## Docker configurations
+
+# You can hardcode the platform to build the vLLM locally since it is supported only
+# for the x86 CPU architecture. ARM CPU architecture may cause to some issues without that.
+# BUILDPLATFORM=linux/x86_64
+
+# This environment variable defines which port will be available locally
+DOCKER_VLLM_PORT_EXPOSE=8000

pyproject.toml

@@ -20,6 +20,8 @@ lint.select = ["E", "F", "W"]
 max-line-length = 88
 
 [tool.pytest.ini_options]
+addopts = '-s -vvv --cache-clear'
+asyncio_mode = 'auto'
 python_classes = "DisableTestClasses"
 markers = [
     "smoke: quick tests to check basic functionality",

src/guidellm/__init__.py renamed to src/config/__init__.py

@@ -1,14 +1,12 @@
-import functools
 import uuid
 from abc import ABC, abstractmethod
 from dataclasses import dataclass
 from enum import Enum
-from typing import Generic, Iterator, List, Optional, Type, Union
+from typing import Iterator, List, Optional, Type, Union
 
 from loguru import logger
 
-from guidellm.core.request import TextGenerationRequest
-from guidellm.core.result import TextGenerationResult
+from domain.core import TextGenerationRequest, TextGenerationResult
 
 __all__ = ["Backend", "BackendEngine", "GenerativeResponse"]
 
@@ -46,8 +44,8 @@ class Backend(ABC):
 
     _registry = {}
 
-    @staticmethod
-    def register(backend_type: BackendEngine):
+    @classmethod
+    def register(cls, backend_type: BackendEngine):
         """
         A decorator to register a backend class in the backend registry.
 
@@ -56,13 +54,13 @@ def register(backend_type: BackendEngine):
         """
 
         def inner_wrapper(wrapped_class: Type["Backend"]):
-            Backend._registry[backend_type] = wrapped_class
+            cls._registry[backend_type] = wrapped_class
             return wrapped_class
 
         return inner_wrapper
 
-    @staticmethod
-    def create(backend_type: Union[str, BackendEngine], **kwargs) -> "Backend":
+    @classmethod
+    def create(cls, backend_type: Union[str, BackendEngine], **kwargs) -> "Backend":
         """
         Factory method to create a backend based on the backend type.
 
@@ -76,11 +74,11 @@ def create(backend_type: Union[str, BackendEngine], **kwargs) -> "Backend":
 
         logger.info(f"Creating backend of type {backend_type}")
 
-        if backend_type not in Backend._registry:
+        if backend_type not in cls._registry:
             logger.error(f"Unsupported backend type: {backend_type}")
             raise ValueError(f"Unsupported backend type: {backend_type}")
 
-        return Backend._registry[backend_type](**kwargs)
+        return cls._registry[backend_type](**kwargs)
 
     def submit(self, request: TextGenerationRequest) -> TextGenerationResult:
         """
@@ -91,6 +89,7 @@ def submit(self, request: TextGenerationRequest) -> TextGenerationResult:
         :return: The populated result result.
         :rtype: TextGenerationResult
         """
+
         logger.info(f"Submitting request with prompt: {request.prompt}")
         result_id = str(uuid.uuid4())
         result = TextGenerationResult(result_id)
@@ -108,6 +107,7 @@ def submit(self, request: TextGenerationRequest) -> TextGenerationResult:
                 break
 
         logger.info(f"Request completed with output: {result.output}")
+
         return result
 
     @abstractmethod

src/domain/__init__.py

@@ -3,12 +3,12 @@
 from typing import Any, Dict, Iterable, Iterator, List, Optional
 
 from loguru import logger
-from openai import OpenAI, Stream
+from openai import OpenAI
 from openai.types import Completion
 from transformers import AutoTokenizer
 
-from guidellm.backend import Backend, BackendEngine, GenerativeResponse
-from guidellm.core.request import TextGenerationRequest
+from domain.backend import Backend, BackendEngine, GenerativeResponse
+from domain.core import TextGenerationRequest
 
 __all__ = ["OpenAIBackend"]
 
@@ -49,15 +49,17 @@ def __init__(
 
         if not (_api_key := (openai_api_key or os.getenv("OPENAI_API_KEY", None))):
             raise ValueError(
-                "`OPENAI_API_KEY` environment variable or --openai-api-key CLI parameter "
+                "`OPENAI_API_KEY` environment variable "
+                "or --openai-api-key CLI parameter "
                 "must be specify for the OpenAI backend"
             )
 
         if not (
             _base_url := (internal_callback_url or os.getenv("OPENAI_BASE_URL", None))
         ):
             raise ValueError(
-                "`OPENAI_BASE_URL` environment variable or --openai-base-url CLI parameter "
+                "`OPENAI_BASE_URL` environment variable "
+                "or --openai-base-url CLI parameter "
                 "must be specify for the OpenAI backend"
             )
         self.openai_client = OpenAI(api_key=_api_key, base_url=_base_url)

src/guidellm/backend/__init__.py renamed to src/domain/backend/__init__.py

@@ -1,10 +1,8 @@
-from typing import List, Union
+from typing import List, Union, Optional
 
 import numpy as np
 from loguru import logger
 
-__all__ = ["Distribution"]
-
 
 class Distribution:
     """
@@ -16,7 +14,7 @@ class Distribution:
     :type data: List[Union[int, float]], optional
     """
 
-    def __init__(self, data: List[Union[int, float]] = None):
+    def __init__(self, data: Optional[List[Union[int, float]]] = None):
         """
         Initialize the Distribution with optional data.
 

src/guidellm/backend/base.py renamed to src/domain/backend/base.py

@@ -3,8 +3,6 @@
 
 from loguru import logger
 
-__all__ = ["TextGenerationRequest"]
-
 
 class TextGenerationRequest:
     """