Add unittest for GrokClient

Author: koxudaxi

python/mirascope/llm/clients/xai/grok.py

@@ -19,6 +19,43 @@ Note that when you update the implementation or tests in a way that changes what
 You can regenerate snapshots en masse by deleting the `cassettes/` subdirectory, but this
 should be done sparingly because it takes time and API token usage.
 
+### xAI/Grok Special Case
+
+**xAI uses gRPC instead of HTTP**, so VCR.py cannot intercept its traffic. Therefore, xAI tests are handled differently:
+
+- **In CI**: xAI tests are **automatically skipped** (no API key required)
+- **Locally**: xAI tests run with **real API calls** when the `--use-real-grok` flag is provided
+
+#### Running xAI tests locally
+
+```bash
+# Set your API key
+export XAI_API_KEY=your-api-key
+
+# Run all E2E tests including xAI
+cd python
+uv run pytest tests/e2e/ --use-real-grok --inline-snapshot=fix
+
+# Run only xAI tests
+uv run pytest tests/e2e/ -k xai --use-real-grok
+```
+
+#### When to run xAI tests
+
+Run xAI tests manually when:
+
+1. **Making changes to xAI client**: `mirascope/llm/clients/xai/grok.py`
+2. **Changing core abstractions**: Changes to base classes or protocols that affect all providers
+3. **Before releases**: Verify xAI compatibility before publishing a new version
+
+#### Why this approach?
+
+Since gRPC traffic cannot be recorded with VCR.py, we have two options:
+1. Build a custom cassette system (complex, fragile, hard to maintain)
+2. Run tests with real API calls when needed (simple, accurate, pragmatic)
+
+We chose option 2 for simplicity and maintainability.
+
 ## Snapshots
 
 Tests use [inline-snapshot](https://15r10nk.github.io/inline-snapshot/) to validate test outputs. Snapshots are stored in `snapshots/` subdirectories.

python/tests/e2e/README.md

@@ -25,6 +25,34 @@
 from mirascope import llm
 from mirascope.llm.clients.anthropic_vertex import clients as anthropic_vertex_clients
 
+
+def pytest_addoption(parser: pytest.Parser) -> None:
+    """Add custom command line options."""
+    parser.addoption(
+        "--use-real-grok",
+        action="store_true",
+        default=False,
+        help="Run xAI/Grok tests with real API (requires XAI_API_KEY). "
+        "Without this flag, xAI tests are skipped since gRPC cannot be recorded with VCR.",
+    )
+
+
+def pytest_collection_modifyitems(
+    config: pytest.Config, items: list[pytest.Item]
+) -> None:
+    """Modify test collection to skip xAI tests unless --use-real-grok is set."""
+    if config.getoption("--use-real-grok"):
+        return  # Run all tests including xAI
+
+    skip_grok = pytest.mark.skip(
+        reason="xAI tests require --use-real-grok flag (gRPC cannot be recorded with VCR)"
+    )
+    for item in items:
+        # Skip tests that have "xai" in their test ID (parametrized tests)
+        if "xai" in item.nodeid.lower():
+            item.add_marker(skip_grok)
+
+
 if TYPE_CHECKING:
     from typing import Any
 
@@ -36,6 +64,7 @@
     ("google", "gemini-2.5-flash"),
     ("openai:completions", "gpt-4o"),
     ("openai:responses", "gpt-4o"),
+    ("xai", "grok-3"),
 ]
 
 

python/tests/e2e/conftest.py

@@ -3,16 +3,31 @@
 from __future__ import annotations
 
 import importlib
+from collections.abc import Generator
+from contextlib import nullcontext
 from pathlib import Path
+from typing import Any
 
 import pytest
+import vcr
 from pytest import FixtureRequest
 
 from mirascope import llm
-from tests.e2e.conftest import SNAPSHOT_IMPORT_SYMBOLS
+from tests.e2e.conftest import SNAPSHOT_IMPORT_SYMBOLS, VCRConfig
 from tests.utils import Snapshot
 
 
+@pytest.fixture(scope="session")
+def vcr_config(vcr_config: VCRConfig) -> VCRConfig:
+    """Override VCR config to set cassette directory for input tests.
+
+    Inherits the base VCR configuration from tests/e2e/conftest.py and adds
+    the cassette_library_dir to point to the input/cassettes directory.
+    """
+    vcr_config["cassette_library_dir"] = str(Path(__file__).parent / "cassettes")
+    return vcr_config
+
+
 def _extract_scenario_from_test_name(test_name: str) -> str:
     """Extract scenario name from test name.
 
@@ -43,25 +58,31 @@ def vcr_cassette_name(
     provider: llm.Provider,
     model_id: llm.ModelId,
     formatting_mode: llm.FormattingMode | None,
-) -> str:
+) -> str | None:
     """Generate VCR cassette name based on test name, provider, model, and formatting_mode.
 
     Input tests use a single cassette per test (no call type variants).
 
     Structure:
     - Without formatting_mode: {scenario}/{provider}_{model_id}
     - With formatting_mode: {scenario}/{formatting_mode}/{provider}_{model_id}
+
+    Returns None for xAI provider to skip VCR (xAI tests run with --use-real-grok flag).
     """
+    # xAI uses gRPC, so skip VCR cassettes (requires --use-real-grok to run)
+    if provider == "xai":
+        return None
+
     test_name = request.node.name
     scenario = _extract_scenario_from_test_name(test_name)
 
     provider_str = provider.replace(":", "_")
     model_id_str = model_id.replace("-", "_").replace(".", "_")
 
     if formatting_mode is None:
-        return f"{scenario}/{provider_str}_{model_id_str}"
+        return f"{scenario}/{provider_str}_{model_id_str}.yaml"
     else:
-        return f"{scenario}/{formatting_mode}/{provider_str}_{model_id_str}"
+        return f"{scenario}/{formatting_mode}/{provider_str}_{model_id_str}.yaml"
 
 
 @pytest.fixture
@@ -114,3 +135,30 @@ def snapshot(
 
     module = importlib.import_module(module_path)
     return module.test_snapshot
+
+
+@pytest.fixture
+def vcr_cassette(
+    request: FixtureRequest, vcr_cassette_name: str | None, vcr_config: dict[str, Any]
+) -> Generator[Any, None, None]:
+    """Override pytest-vcr's vcr_cassette to handle None cassette_name for xAI.
+
+    When vcr_cassette_name is None (e.g., for xAI provider which uses gRPC and
+    cannot be recorded with VCR), this fixture returns a no-op context manager.
+
+    Args:
+        request: Pytest fixture request.
+        vcr_cassette_name: Cassette name or None to skip VCR.
+        vcr_config: VCR configuration dict.
+
+    Yields:
+        VCR cassette or None if skipped.
+    """
+    if vcr_cassette_name is None:
+        # xAI uses gRPC and cannot be recorded with VCR, skip VCR
+        with nullcontext() as cassette:
+            yield cassette
+    else:
+        # Use normal VCR for HTTP-based providers
+        with vcr.VCR(**vcr_config).use_cassette(vcr_cassette_name) as cassette:
+            yield cassette

python/tests/e2e/input/conftest.py

@@ -0,0 +1,33 @@
+from inline_snapshot import snapshot
+
+from mirascope.llm import AssistantMessage, Text, UserMessage
+
+test_snapshot = snapshot(
+    {
+        "response": (
+            {
+                "provider": "xai",
+                "model_id": "grok-3",
+                "params": {
+                    "temperature": 0.7,
+                    "max_tokens": 500,
+                    "top_p": 0.3,
+                    "seed": 42,
+                    "stop_sequences": ["4242"],
+                },
+                "finish_reason": None,
+                "messages": [
+                    UserMessage(content=[Text(text="What is 4200 + 42?")]),
+                    AssistantMessage(
+                        content=[Text(text="4200 + 42 = ")],
+                        provider="xai",
+                        model_id="grok-3",
+                        raw_message=None,
+                    ),
+                ],
+                "format": None,
+                "tools": [],
+            },
+        )
+    }
+)

python/tests/e2e/input/snapshots/test_call_with_params/xai_grok_3_snapshots.py

@@ -0,0 +1,3 @@
+from inline_snapshot import snapshot
+
+test_snapshot = snapshot({"response": "1597"})

python/tests/e2e/input/snapshots/test_call_with_text_encoded_thoughts/xai_grok_3_snapshots.py

@@ -0,0 +1,49 @@
+from inline_snapshot import snapshot
+
+from mirascope.llm import (
+    AssistantMessage,
+    Text,
+    UserMessage,
+)
+
+test_snapshot = snapshot(
+    {
+        "response": {
+            "provider": "xai",
+            "model_id": "grok-3",
+            "params": {},
+            "finish_reason": None,
+            "messages": [
+                UserMessage(content=[Text(text="Who created you?")]),
+                AssistantMessage(
+                    content=[Text(text="I was created by Anthropic.")],
+                    provider="anthropic",
+                    model_id="claude-sonnet-4-0",
+                    raw_message={
+                        "role": "assistant",
+                        "content": [
+                            {
+                                "citations": None,
+                                "text": "I was created by Anthropic.",
+                                "type": "text",
+                            }
+                        ],
+                    },
+                ),
+                UserMessage(content=[Text(text="Can you double-check that?")]),
+                AssistantMessage(
+                    content=[
+                        Text(
+                            text="My apologies for the confusion. I am Grok, created by xAI. I must have misspoken earlier. Thank you for asking me to double-check. I'm indeed a product of xAI, a company working on building artificial intelligence to accelerate human scientific discovery."
+                        )
+                    ],
+                    provider="xai",
+                    model_id="grok-3",
+                    raw_message=None,
+                ),
+            ],
+            "format": None,
+            "tools": [],
+        }
+    }
+)

python/tests/e2e/input/snapshots/test_resume_with_override/xai_grok_3_snapshots.py

@@ -0,0 +1,49 @@
+from inline_snapshot import snapshot
+
+from mirascope.llm import (
+    AssistantMessage,
+    Text,
+    UserMessage,
+)
+
+test_snapshot = snapshot(
+    {
+        "response": {
+            "provider": "xai",
+            "model_id": "grok-3",
+            "params": {},
+            "finish_reason": None,
+            "messages": [
+                UserMessage(content=[Text(text="Who created you?")]),
+                AssistantMessage(
+                    content=[Text(text="I was created by Anthropic.")],
+                    provider="anthropic",
+                    model_id="claude-sonnet-4-0",
+                    raw_message={
+                        "role": "assistant",
+                        "content": [
+                            {
+                                "citations": None,
+                                "text": "I was created by Anthropic.",
+                                "type": "text",
+                            }
+                        ],
+                    },
+                ),
+                UserMessage(content=[Text(text="Can you double-check that?")]),
+                AssistantMessage(
+                    content=[
+                        Text(
+                            text="My apologies for any confusion. I am Grok, created by xAI. I must have misspoken earlier. Thank you for asking for clarification."
+                        )
+                    ],
+                    provider="xai",
+                    model_id="grok-3",
+                    raw_message=None,
+                ),
+            ],
+            "format": None,
+            "tools": [],
+        }
+    }
+)