Merge pull request #211 from raga-ai-hub/v2.1.7

v2.1.7

Author: kiranscaria

.github/workflows/ci.yml

@@ -0,0 +1,99 @@
+name: CI Pipeline
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  code-quality:
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+      - name: Install dependencies
+        shell: bash
+        run: |
+          if [ "$RUNNER_OS" == "Windows" ]; then
+            python -m pip install --upgrade pip
+            pip install ruff
+            pip install -e ".[dev]"
+          else
+            curl -LsSf https://astral.sh/uv/install.sh | sh
+            export PATH="$HOME/.cargo/bin:$PATH"
+            uv pip install --system ruff
+            uv pip install --system -e ".[dev]"
+          fi
+      - name: Format and lint with Ruff
+        run: |
+          # First run format to fix formatting issues
+          ruff format .
+          # Then run check with auto-fix for fixable issues
+          ruff check --fix .
+
+  test:
+    needs: code-quality
+    continue-on-error: true
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+        python-version: ['3.10', '3.11', '3.12', '3.13']
+    runs-on: ${{ matrix.os }}
+    outputs:
+      test_summary: ${{ steps.pytest.outputs.test_summary }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        shell: bash
+        run: |
+          if [ "$RUNNER_OS" == "Windows" ]; then
+            python -m pip install --upgrade pip
+            pip install pytest pytest-cov
+            pip install -r tests_requirements.txt
+            pip install -e ".[dev]"
+          else
+            curl -LsSf https://astral.sh/uv/install.sh | sh
+            export PATH="$HOME/.cargo/bin:$PATH"
+            uv pip install --system pytest pytest-cov
+            uv pip install --system -r tests_requirements.txt
+            uv pip install --system -e ".[dev]"
+          fi
+      - name: Test with pytest
+        id: pytest
+        env:
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+          GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }}
+          AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }}
+          AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }}
+          AZURE_OPENAI_API_VERSION: ${{ secrets.AZURE_OPENAI_API_VERSION }}
+          GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
+          GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
+          PROJECT_NAME: ${{ secrets.PROJECT_NAME }}
+          LOCATION: ${{ secrets.LOCATION }}
+          RAGAAI_CATALYST_BASE_URL: ${{ secrets.RAGAAI_CATALYST_BASE_URL }}
+          RAGAAI_CATALYST_ACCESS_KEY: ${{ secrets.RAGAAI_CATALYST_ACCESS_KEY }}
+          RAGAAI_CATALYST_SECRET_KEY: ${{ secrets.RAGAAI_CATALYST_SECRET_KEY }}
+          RAGAAI_PROJECT_NAME: ${{ secrets.RAGAAI_PROJECT_NAME }}
+          RAGAAI_DATASET_NAME: ${{ secrets.RAGAAI_DATASET_NAME }}_$(date +'%Y%m%d%H%M%S')
+          TAVILY_API_KEY: ${{ secrets.TAVILY_API_KEY }}
+          SERPERDEV_API_KEY: ${{ secrets.SERPERDEV_API_KEY }}
+        run: |
+          mkdir -p test-results
+          pytest tests/ -v --junitxml=test-results/junit.xml | tee test-output.txt
+          echo "test_summary<<EOF" >> $GITHUB_OUTPUT
+          echo "### Test Results for ${{ matrix.os }} - Python ${{ matrix.python-version }}" >> $GITHUB_OUTPUT
+          echo '```' >> $GITHUB_OUTPUT
+          cat test-output.txt | grep -E "collected|PASSED|FAILED|ERROR|SKIPPED" >> $GITHUB_OUTPUT
+          echo '```' >> $GITHUB_OUTPUT
+          echo "EOF" >> $GITHUB_OUTPUT

Quickstart.md

@@ -0,0 +1,200 @@
+# Quickstart | RagaAI Catalyst
+
+## **1. Install RagaAI Catalyst**
+
+To install the RagaAI Catalyst package, run the following command in your terminal:
+
+```bash
+pip install ragaai-catalyst
+```
+
+
+
+## **2. Set Up Authentication Keys**
+
+### **How to Get Your API Keys :**
+1. Log in to your account at [RagaAI Catalyst](https://catalyst.raga.ai/).
+2. Navigate to **Profile Settings** → **Authentication**.
+3. Click **Generate New Key** to obtain your **Access Key** and **Secret Key**.
+
+### **Initialize the SDK**
+
+To begin using Catalyst, initialize it as follows:
+
+```python
+from ragaai_catalyst import RagaAICatalyst
+
+catalyst = RagaAICatalyst(
+    access_key="YOUR_ACCESS_KEY",  # Replace with your access key
+    secret_key="YOUR_SECRET_KEY",  # Replace with your secret key
+    base_url="BASE_URL"  
+)
+```
+
+
+## **3. Create Your First Project**
+
+Create a new project and choose a use case from the available options:
+
+```python
+# Create a new project
+project = catalyst.create_project(
+    project_name="Project_Name",
+    usecase="Q/A"  # Options : Chatbot, Q/A, Others, Agentic Application
+)
+
+# List available use cases
+print(catalyst.project_use_cases())
+```
+
+### **Add a Dataset**
+Initialize the dataset manager and create a dataset from a CSV file, DataFrame, or JSONl file.
+
+Define a **schema mapping** for the dataset.
+
+```python
+from ragaai_catalyst import Dataset
+
+# Initialize dataset manager
+dataset_manager = Dataset(project_name="Project_Name")
+
+# Create dataset from a CSV file
+dataset_manager.create_from_csv(
+    csv_path="path/to/your.csv",
+    dataset_name="MyDataset",
+    schema_mapping={
+        'column1': 'schema_element1',
+        'column2': 'schema_element2'
+    }
+)
+
+# View dataset schema
+print(dataset_manager.get_schema_mapping())
+```
+
+
+## **4. Trace Your Application**
+
+
+
+### **Auto-Instrumentation**
+
+Auto-Instrumentation automatically traces your application after initializing the correct tracer.
+
+#### **Implementation**
+
+```python
+from ragaai_catalyst import init_tracing, Tracer
+
+# Initialize the tracer 
+tracer = Tracer(
+    project_name="Project_Name",
+    dataset_name="Dataset_Name",
+    tracer_type="agentic/langgraph"  
+)
+
+# Enable auto-instrumentation
+init_tracing(catalyst=catalyst, tracer=tracer)
+```
+
+#### **Supported Tracer Types**
+
+Choose from the given supported tracer types based on your framework:
+
+- `agentic/langgraph`
+- `agentic/langchain`
+- `agentic/smolagents`
+- `agentic/openai_agents`
+- `agentic/llamaindex`
+- `agentic/haystack`
+
+---
+
+
+
+### Custom Tracing
+
+You can enable custom tracing in two ways:
+
+1. Using the `with tracer()` function.
+2. Manually starting and stopping the tracer with `tracer.start()` and `tracer.stop()`.
+
+```python
+from ragaai_catalyst import Tracer
+
+# Initialize production tracer
+tracer = Tracer(
+    project_name="Project_Name",
+    dataset_name="tracer_dataset_name",
+    tracer_type="tracer_type"
+)
+
+# Start a trace recording (Option 1)
+with tracer():
+    # Your code here
+
+# Start a trace recording (Option 2)
+tracer.start()
+
+# Your code here
+
+# Stop the trace recording
+tracer.stop()
+
+# Verify data capture
+print(tracer.get_upload_status())
+```
+
+
+
+## **5. Evaluation Framework**
+
+
+1. Import `Evaluation` from `ragaai_catalyst`.
+2. Configure evaluation metrics.
+3. Add metrics from the available options.
+4. Check the status and retrieve results after running the evaluation.
+
+```python
+from ragaai_catalyst import Evaluation
+
+# Initialize evaluation engine
+evaluation = Evaluation(
+    project_name="Project_Name",
+    dataset_name="MyDataset"
+)
+
+# Define Schema-mapping
+
+schema_mapping = {
+    'Query': 'prompt',
+    'response': 'response',
+    'Context': 'context',
+    'expectedResponse': 'expected_response'
+}
+
+evaluation.add_metrics(
+    metrics=[
+        {
+            "name": "Faithfulness",
+            "config": {"model": "gpt-4o-mini", "provider": "openai", "threshold": {"gte": 0.232323}},
+            "column_name": "Faithfulness_v1",
+            "schema_mapping": schema_mapping
+        }
+    ]
+)
+
+# Get status and results
+
+print(f"Status: {evaluation.get_status()}")
+print(f"Results: {evaluation.get_results()}")
+```
+
+
+
+## **Next Steps**
+- **Explore the Dashboard:** Visualize metrics and insights in the RagaAI Web UI.
+
+
+
+**Version:** 1.0.0 | **Last Updated:** Mar 2025