From 2d338645e3b2063a4d695ecbf4456290bfafe7a4 Mon Sep 17 00:00:00 2001 From: Ol1ver0413 <790221864@qq.com> Date: Thu, 18 Sep 2025 12:07:34 -0400 Subject: [PATCH 1/6] wk_pipline_0.1 --- camel/societies/workforce/__init__.py | 6 +- camel/societies/workforce/utils.py | 314 ++++++++++++ camel/societies/workforce/workforce.py | 285 ++++++++++- .../workforce/pipeline_workflow_example.py | 448 ++++++++++++++++++ 4 files changed, 1050 insertions(+), 3 deletions(-) create mode 100644 examples/workforce/pipeline_workflow_example.py diff --git a/camel/societies/workforce/__init__.py b/camel/societies/workforce/__init__.py index 8b2f3fe994..3990e8c97a 100644 --- a/camel/societies/workforce/__init__.py +++ b/camel/societies/workforce/__init__.py @@ -14,10 +14,14 @@ from .role_playing_worker import RolePlayingWorker from .single_agent_worker import SingleAgentWorker -from .workforce import Workforce +from .utils import PipelineTaskBuilder +from .workforce import PipelineBuilder, Workforce, WorkforceMode __all__ = [ "Workforce", + "WorkforceMode", + "PipelineBuilder", + "PipelineTaskBuilder", "SingleAgentWorker", "RolePlayingWorker", ] diff --git a/camel/societies/workforce/utils.py b/camel/societies/workforce/utils.py index 388801246e..7796486f30 100644 --- a/camel/societies/workforce/utils.py +++ b/camel/societies/workforce/utils.py @@ -128,6 +128,320 @@ class RecoveryDecision(BaseModel): ) +class PipelineTaskBuilder: + r"""Helper class for building pipeline tasks with dependencies.""" + + def __init__(self): + """Initialize an empty pipeline task builder.""" + from camel.tasks import Task + self._TaskClass = Task + self.task_list = [] + self.task_counter = 0 + self._task_registry = {} # task_id -> Task mapping for fast lookup + self._last_task_id = None # Track the last added task for chain inference + self._last_parallel_tasks = [] # Track the last added parallel tasks for sync + + def add_task( + self, + content: str, + task_id: Optional[str] = None, + dependencies: Optional[List[str]] = None, + additional_info: Optional[dict] = None, + auto_depend: bool = True, + ) -> 'PipelineTaskBuilder': + """Add a task to the pipeline with support for chaining. + + Args: + content (str): The content/description of the task. + task_id (str, optional): Unique identifier for the task. If None, + a unique ID will be generated. (default: :obj:`None`) + dependencies (List[str], optional): List of task IDs that this + task depends on. If None and auto_depend=True, will depend on + the last added task. (default: :obj:`None`) + additional_info (dict, optional): Additional information + for the task. (default: :obj:`None`) + auto_depend (bool, optional): If True and dependencies is None, + automatically depend on the last added task. (default: :obj:`True`) + + Returns: + PipelineTaskBuilder: Self for method chaining. + + Raises: + ValueError: If task_id already exists or if any dependency is not found. + + Example: + >>> builder.add_task("Step 1").add_task("Step 2").add_task("Step 3") + # Step 2 depends on Step 1, Step 3 depends on Step 2 + """ + # Generate or validate task_id + task_id = task_id or f"pipeline_task_{self.task_counter}" + + # Check ID uniqueness + if task_id in self._task_registry: + raise ValueError(f"Task ID '{task_id}' already exists") + + # Auto-infer dependencies if not specified + if dependencies is None and auto_depend and self._last_task_id is not None: + dependencies = [self._last_task_id] + + # Validate dependencies exist + dep_tasks = [] + if dependencies: + missing_deps = [dep for dep in dependencies if dep not in self._task_registry] + if missing_deps: + raise ValueError(f"Dependencies not found: {missing_deps}") + dep_tasks = [self._task_registry[dep] for dep in dependencies] + + # Create task + task = self._TaskClass( + content=content, + id=task_id, + dependencies=dep_tasks, + additional_info=additional_info, + ) + + self.task_list.append(task) + self._task_registry[task_id] = task + self._last_task_id = task_id # Update last task for chaining + self.task_counter += 1 + return self + + def add_parallel_tasks( + self, + task_contents: List[str], + dependencies: Optional[List[str]] = None, + task_id_prefix: str = "parallel", + auto_depend: bool = True, + ) -> 'PipelineTaskBuilder': + """Add multiple parallel tasks that can execute simultaneously. + + Args: + task_contents (List[str]): List of task content strings. + dependencies (List[str], optional): Common dependencies for all + parallel tasks. If None and auto_depend=True, will depend on + the last added task. (default: :obj:`None`) + task_id_prefix (str, optional): Prefix for generated task IDs. + (default: :obj:`"parallel"`) + auto_depend (bool, optional): If True and dependencies is None, + automatically depend on the last added task. (default: :obj:`True`) + + Returns: + PipelineTaskBuilder: Self for method chaining. + + Raises: + ValueError: If any task_id already exists or if any dependency is not found. + + Example: + >>> builder.add_task("Collect Data").add_parallel_tasks([ + ... "Technical Analysis", "Fundamental Analysis" + ... ]).add_sync_task("Generate Report") + """ + if not task_contents: + raise ValueError("task_contents cannot be empty") + + # Auto-infer dependencies if not specified + if dependencies is None and auto_depend and self._last_task_id is not None: + dependencies = [self._last_task_id] + + parallel_task_ids = [] + base_counter = self.task_counter # Save current counter for consistent naming + + for i, content in enumerate(task_contents): + task_id = f"{task_id_prefix}_{i}_{base_counter}" + # Use auto_depend=False since we're manually managing dependencies + self.add_task(content, task_id, dependencies, auto_depend=False) + parallel_task_ids.append(task_id) + + # Set the last task to None since we have multiple parallel endings + # The next task will need to explicitly specify dependencies + self._last_task_id = None + # Store parallel task IDs for potential sync operations + self._last_parallel_tasks = parallel_task_ids + + return self + + def add_sync_task( + self, + content: str, + wait_for: Optional[List[str]] = None, + task_id: Optional[str] = None, + ) -> 'PipelineTaskBuilder': + """Add a synchronization task that waits for multiple tasks. + + Args: + content (str): Content of the synchronization task. + wait_for (List[str], optional): List of task IDs to wait for. + If None, will automatically wait for the last parallel tasks. + (default: :obj:`None`) + task_id (str, optional): ID for the sync task. If None, a unique + ID will be generated. (default: :obj:`None`) + + Returns: + PipelineTaskBuilder: Self for method chaining. + + Raises: + ValueError: If task_id already exists or if any dependency is not found. + + Example: + >>> builder.add_parallel_tasks(["Task A", "Task B"]).add_sync_task("Merge Results") + # Automatically waits for both parallel tasks + """ + # Auto-infer wait_for from last parallel tasks + if wait_for is None: + if hasattr(self, '_last_parallel_tasks') and self._last_parallel_tasks: + wait_for = self._last_parallel_tasks + # Clear the parallel tasks after using them + delattr(self, '_last_parallel_tasks') + else: + raise ValueError("wait_for cannot be empty for sync task and no parallel tasks found") + + if not wait_for: + raise ValueError("wait_for cannot be empty for sync task") + + return self.add_task(content, task_id, dependencies=wait_for, auto_depend=False) + + def add_sync_point( + self, + content: str, + wait_for: List[str], + task_id: Optional[str] = None, + ) -> 'PipelineTaskBuilder': + """Add a synchronization point task that waits for multiple tasks. + + This is an alias for add_sync_task() for backward compatibility. + + Args: + content (str): Content of the synchronization task. + wait_for (List[str]): List of task IDs to wait for. + task_id (str, optional): ID for the sync task. If None, a unique + ID will be generated. (default: :obj:`None`) + + Returns: + PipelineTaskBuilder: Self for method chaining. + """ + return self.add_sync_task(content, wait_for, task_id) + + def build(self) -> List: + """Build and return the complete task list with dependencies. + + Returns: + List[Task]: List of tasks with proper dependency relationships. + + Raises: + ValueError: If there are circular dependencies or other validation errors. + """ + if not self.task_list: + raise ValueError("No tasks defined in pipeline") + + # Validate no circular dependencies + self._validate_dependencies() + + return self.task_list.copy() + + def clear(self) -> None: + """Clear all tasks from the builder.""" + self.task_list.clear() + self._task_registry.clear() + self.task_counter = 0 + self._last_task_id = None + if hasattr(self, '_last_parallel_tasks'): + delattr(self, '_last_parallel_tasks') + + def fork(self, task_contents: List[str]) -> 'PipelineTaskBuilder': + """Create parallel branches from the current task (alias for add_parallel_tasks). + + Args: + task_contents (List[str]): List of task content strings for parallel execution. + + Returns: + PipelineTaskBuilder: Self for method chaining. + + Example: + >>> builder.add_task("Collect Data").fork([ + ... "Technical Analysis", "Fundamental Analysis" + ... ]).join("Generate Report") + """ + return self.add_parallel_tasks(task_contents) + + def join(self, content: str, task_id: Optional[str] = None) -> 'PipelineTaskBuilder': + """Join parallel branches with a synchronization task (alias for add_sync_task). + + Args: + content (str): Content of the join/sync task. + task_id (str, optional): ID for the sync task. + + Returns: + PipelineTaskBuilder: Self for method chaining. + + Example: + >>> builder.fork(["Task A", "Task B"]).join("Merge Results") + """ + return self.add_sync_task(content, task_id=task_id) + + def then(self, content: str, task_id: Optional[str] = None) -> 'PipelineTaskBuilder': + """Add a sequential task (alias for add_task for better readability). + + Args: + content (str): Content of the task. + task_id (str, optional): ID for the task. + + Returns: + PipelineTaskBuilder: Self for method chaining. + + Example: + >>> builder.add_task("Start").then("Middle").then("End") + """ + return self.add_task(content, task_id) + + def _validate_dependencies(self) -> None: + """Validate that there are no circular dependencies. + + Raises: + ValueError: If circular dependencies are detected. + """ + # Use DFS to detect cycles + visited = set() + rec_stack = set() + + def has_cycle(task_id: str) -> bool: + visited.add(task_id) + rec_stack.add(task_id) + + task = self._task_registry[task_id] + for dep in task.dependencies: + if dep.id not in visited: + if has_cycle(dep.id): + return True + elif dep.id in rec_stack: + return True + + rec_stack.remove(task_id) + return False + + for task_id in self._task_registry: + if task_id not in visited: + if has_cycle(task_id): + raise ValueError(f"Circular dependency detected involving task: {task_id}") + + def get_task_info(self) -> dict: + """Get information about all tasks in the pipeline. + + Returns: + dict: Dictionary containing task count and task details. + """ + return { + "task_count": len(self.task_list), + "tasks": [ + { + "id": task.id, + "content": task.content, + "dependencies": [dep.id for dep in task.dependencies] + } + for task in self.task_list + ] + } + + def check_if_running( running: bool, max_retries: int = 3, diff --git a/camel/societies/workforce/workforce.py b/camel/societies/workforce/workforce.py index 8e0c910150..4f07a0c633 100644 --- a/camel/societies/workforce/workforce.py +++ b/camel/societies/workforce/workforce.py @@ -55,6 +55,7 @@ from camel.societies.workforce.task_channel import TaskChannel from camel.societies.workforce.utils import ( FailureContext, + PipelineTaskBuilder, RecoveryDecision, RecoveryStrategy, TaskAssignment, @@ -106,6 +107,14 @@ class WorkforceState(Enum): STOPPED = "stopped" +class WorkforceMode(Enum): + r"""Workforce execution mode for different task processing strategies.""" + + AUTO_DECOMPOSE = "auto_decompose" # Current automatic task decomposition mode + PIPELINE = "pipeline" # Predefined pipeline mode + HYBRID = "hybrid" # Hybrid mode allowing partial auto-decomposition + + class WorkforceSnapshot: r"""Snapshot of workforce state for resuming execution.""" @@ -246,6 +255,7 @@ def __init__( share_memory: bool = False, use_structured_output_handler: bool = True, task_timeout_seconds: Optional[float] = None, + mode: WorkforceMode = WorkforceMode.AUTO_DECOMPOSE, ) -> None: super().__init__(description) self._child_listening_tasks: Deque[ @@ -259,6 +269,7 @@ def __init__( self.task_timeout_seconds = ( task_timeout_seconds or TASK_TIMEOUT_SECONDS ) + self.mode = mode if self.use_structured_output_handler: self.structured_handler = StructuredOutputHandler() self.metrics_logger = WorkforceLogger(workforce_id=self.node_id) @@ -267,6 +278,9 @@ def __init__( self._task_dependencies: Dict[str, List[str]] = {} self._assignees: Dict[str, str] = {} self._in_flight_tasks: int = 0 + + # Pipeline building state + self._pipeline_builder: Optional[PipelineTaskBuilder] = None # Dictionary to track task start times self._task_start_times: Dict[str, float] = {} # Human intervention support @@ -520,9 +534,197 @@ def _ensure_pause_event_in_kwargs(self, kwargs: Optional[Dict]) -> Dict: def __repr__(self): return ( f"Workforce {self.node_id} ({self.description}) - " - f"State: {self._state.value}" + f"State: {self._state.value} - Mode: {self.mode.value}" ) + def _ensure_pipeline_builder(self): + """Ensure pipeline builder is initialized and switch to pipeline mode.""" + if self._pipeline_builder is None: + self._pipeline_builder = PipelineTaskBuilder() + + # Auto-switch to pipeline mode + if self.mode != WorkforceMode.PIPELINE: + self.mode = WorkforceMode.PIPELINE + + def add_pipeline_task( + self, + content: str, + task_id: Optional[str] = None, + dependencies: Optional[List[str]] = None, + additional_info: Optional[Dict[str, Any]] = None, + auto_depend: bool = True, + ) -> 'Workforce': + """Add a task to the pipeline with support for chaining. + + Args: + content (str): The content/description of the task. + task_id (str, optional): Unique identifier for the task. If None, + a unique ID will be generated. (default: :obj:`None`) + dependencies (List[str], optional): List of task IDs that this + task depends on. If None and auto_depend=True, will depend on + the last added task. (default: :obj:`None`) + additional_info (Dict[str, Any], optional): Additional information + for the task. (default: :obj:`None`) + auto_depend (bool, optional): If True and dependencies is None, + automatically depend on the last added task. (default: :obj:`True`) + + Returns: + Workforce: Self for method chaining. + + Example: + >>> workforce.add_pipeline_task("Step 1").add_pipeline_task("Step 2").add_pipeline_task("Step 3") + """ + self._ensure_pipeline_builder() + self._pipeline_builder.add_task(content, task_id, dependencies, additional_info, auto_depend) + return self + + def add_parallel_pipeline_tasks( + self, + task_contents: List[str], + dependencies: Optional[List[str]] = None, + task_id_prefix: str = "parallel", + auto_depend: bool = True, + ) -> 'Workforce': + """Add multiple parallel tasks to the pipeline. + + Args: + task_contents (List[str]): List of task content strings. + dependencies (List[str], optional): Common dependencies for all + parallel tasks. (default: :obj:`None`) + task_id_prefix (str, optional): Prefix for generated task IDs. + (default: :obj:`"parallel"`) + auto_depend (bool, optional): If True and dependencies is None, + automatically depend on the last added task. (default: :obj:`True`) + + Returns: + Workforce: Self for method chaining. + """ + self._ensure_pipeline_builder() + self._pipeline_builder.add_parallel_tasks(task_contents, dependencies, task_id_prefix, auto_depend) + return self + + def add_sync_pipeline_task( + self, + content: str, + wait_for: Optional[List[str]] = None, + task_id: Optional[str] = None, + ) -> 'Workforce': + """Add a synchronization task that waits for multiple tasks. + + Args: + content (str): Content of the synchronization task. + wait_for (List[str], optional): List of task IDs to wait for. + If None, will automatically wait for the last parallel tasks. + task_id (str, optional): ID for the sync task. + + Returns: + Workforce: Self for method chaining. + """ + self._ensure_pipeline_builder() + self._pipeline_builder.add_sync_task(content, wait_for, task_id) + return self + + def fork_pipeline(self, task_contents: List[str]) -> 'Workforce': + """Create parallel branches from the current task (alias for add_parallel_pipeline_tasks). + + Args: + task_contents (List[str]): List of task content strings for parallel execution. + + Returns: + Workforce: Self for method chaining. + """ + self._ensure_pipeline_builder() + self._pipeline_builder.fork(task_contents) + return self + + def join_pipeline(self, content: str, task_id: Optional[str] = None) -> 'Workforce': + """Join parallel branches with a synchronization task (alias for add_sync_pipeline_task). + + Args: + content (str): Content of the join/sync task. + task_id (str, optional): ID for the sync task. + + Returns: + Workforce: Self for method chaining. + """ + self._ensure_pipeline_builder() + self._pipeline_builder.join(content, task_id) + return self + + def then_pipeline(self, content: str, task_id: Optional[str] = None) -> 'Workforce': + """Add a sequential task (alias for add_pipeline_task for better readability). + + Args: + content (str): Content of the task. + task_id (str, optional): ID for the task. + + Returns: + Workforce: Self for method chaining. + """ + self._ensure_pipeline_builder() + self._pipeline_builder.then(content, task_id) + return self + + def finalize_pipeline(self) -> 'Workforce': + """Finalize the pipeline and set up the tasks for execution. + + Returns: + Workforce: Self for method chaining. + """ + if self._pipeline_builder is None: + raise ValueError("No pipeline tasks defined") + + tasks = self._pipeline_builder.build() + self.set_pipeline_tasks(tasks) + + return self + + def get_pipeline_builder(self) -> PipelineTaskBuilder: + """Get the underlying PipelineTaskBuilder for advanced usage. + + Returns: + PipelineTaskBuilder: The pipeline builder instance. + + Example: + >>> builder = workforce.get_pipeline_builder() + >>> builder.add_task("Complex Task").fork(["A", "B"]).join("Merge") + >>> tasks = builder.build() + >>> workforce.set_pipeline_tasks(tasks) + """ + self._ensure_pipeline_builder() + return self._pipeline_builder + + def set_pipeline_tasks(self, tasks: List[Task]) -> None: + """Set predefined pipeline tasks for PIPELINE mode. + + Args: + tasks (List[Task]): List of tasks with dependencies already set. + The dependencies should be Task objects in the Task.dependencies + attribute. + + Raises: + ValueError: If tasks are invalid. + """ + if not tasks: + raise ValueError("Cannot set empty task list for pipeline") + + # Auto-switch to pipeline mode if not already + if self.mode != WorkforceMode.PIPELINE: + self.mode = WorkforceMode.PIPELINE + + # Clear existing tasks and dependencies + self._pending_tasks.clear() + self._task_dependencies.clear() + self._assignees.clear() + + # Add tasks and set up dependencies + for task in tasks: + self._pending_tasks.append(task) + if task.dependencies: + self._task_dependencies[task.id] = [dep.id for dep in task.dependencies] + else: + self._task_dependencies[task.id] = [] + def _collect_shared_memory(self) -> Dict[str, List]: r"""Collect memory from all SingleAgentWorker instances for sharing. @@ -739,10 +941,18 @@ def _decompose_task( r"""Decompose the task into subtasks. This method will also set the relationship between the task and its subtasks. + Args: + task (Task): The task to decompose. + Returns: Union[List[Task], Generator[List[Task], None, None]]: - The subtasks or generator of subtasks. + The subtasks or generator of subtasks. Returns empty list for + PIPELINE mode. """ + # In PIPELINE mode, don't decompose - use predefined tasks + if self.mode == WorkforceMode.PIPELINE: + return [] + decompose_prompt = TASK_DECOMPOSE_PROMPT.format( content=task.content, child_nodes_info=self._get_child_nodes_info(), @@ -1211,6 +1421,16 @@ async def process_task_async( ) return task + # Route to appropriate processing method based on mode + if self.mode == WorkforceMode.PIPELINE: + return await self._process_task_with_pipeline(task) + elif self.mode == WorkforceMode.HYBRID: + return await self._process_task_with_hybrid(task) + else: # AUTO_DECOMPOSE mode (default) + return await self._process_task_with_auto_decompose(task) + + async def _process_task_with_auto_decompose(self, task: Task) -> Task: + """Process task using automatic decomposition (original behavior).""" self.reset() self._task = task if self.metrics_logger: @@ -1273,6 +1493,62 @@ async def process_task_async( return task + async def _process_task_with_pipeline(self, task: Task) -> Task: + """Process task using predefined pipeline tasks.""" + if not self._pending_tasks: + raise ValueError( + "No pipeline tasks defined. Use set_pipeline_tasks() first." + ) + + # Don't reset here - keep the predefined tasks + self._task = task + if self.metrics_logger: + self.metrics_logger.log_task_created( + task_id=task.id, + description=task.content, + task_type=task.type, + metadata=task.additional_info, + ) + + task.state = TaskState.FAILED + self.set_channel(TaskChannel()) + await self.start() + + # Collect results from all pipeline tasks + task.result = self._collect_pipeline_results() + task.state = ( + TaskState.DONE if self._all_pipeline_tasks_successful() + else TaskState.FAILED + ) + + return task + + async def _process_task_with_hybrid(self, task: Task) -> Task: + """Process task using hybrid approach (partial auto-decomposition).""" + # For now, fall back to auto-decompose mode + # This can be extended to support more sophisticated hybrid logic + return await self._process_task_with_auto_decompose(task) + + def _collect_pipeline_results(self) -> str: + """Collect results from all completed pipeline tasks.""" + results = [] + for task in self._completed_tasks: + if task.result: + results.append(f"--- Task {task.id} Result ---\n{task.result}") + return "\n\n".join(results) if results else "Pipeline completed" + + def _all_pipeline_tasks_successful(self) -> bool: + """Check if all pipeline tasks completed successfully.""" + expected_task_ids = {task.id for task in self._pending_tasks} + expected_task_ids.update(task.id for task in self._completed_tasks) + + completed_successful_ids = { + task.id for task in self._completed_tasks + if task.state == TaskState.DONE + } + + return expected_task_ids.issubset(completed_successful_ids) + def process_task(self, task: Task) -> Task: r"""Synchronous wrapper for process_task that handles async operations internally. @@ -1702,6 +1978,10 @@ def reset(self) -> None: self.coordinator_agent.reset() self.task_agent.reset() self._task_start_times.clear() + + # Reset pipeline building state + self._pipeline_builder = None + for child in self._children: child.reset() @@ -3161,6 +3441,7 @@ def clone(self, with_memory: bool = False) -> 'Workforce': share_memory=self.share_memory, use_structured_output_handler=self.use_structured_output_handler, task_timeout_seconds=self.task_timeout_seconds, + mode=self.mode, ) for child in self._children: diff --git a/examples/workforce/pipeline_workflow_example.py b/examples/workforce/pipeline_workflow_example.py new file mode 100644 index 0000000000..28132c1ba1 --- /dev/null +++ b/examples/workforce/pipeline_workflow_example.py @@ -0,0 +1,448 @@ +#!/usr/bin/env python3 +# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. ========= +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. ========= + +""" +Pipeline Workflow Example - Comprehensive demonstration of CAMEL Workforce Pipeline functionality. + +This example demonstrates all the Pipeline features of CAMEL Workforce: +1. Sequential task chains +2. Parallel task execution +3. Task synchronization and joining +4. Different pipeline building approaches +5. Auto-dependency inference +6. Manual dependency specification +7. Complex workflow patterns + +The example simulates a market analysis workflow with data collection, +parallel analysis, and report generation. +""" + +import asyncio +import time +from typing import Dict, Any + +from colorama import Fore, Style, init + +from camel.agents import ChatAgent +from camel.models import ModelFactory +from camel.societies.workforce import ( + PipelineTaskBuilder, + Workforce, + WorkforceMode, + SingleAgentWorker +) +from camel.tasks import Task +from camel.toolkits import SearchToolkit, ThinkingToolkit +from camel.types import ModelPlatformType, ModelType +from camel.messages import BaseMessage + +# Initialize colorama for colored output +init(autoreset=True) + +def print_section(title: str): + """Print a colored section header.""" + print(f"\n{Fore.CYAN}{'='*60}") + print(f"{Fore.CYAN}{title}") + print(f"{Fore.CYAN}{'='*60}{Style.RESET_ALL}") + +def print_result(task_id: str, result: str): + """Print task result with formatting.""" + print(f"{Fore.GREEN}Task {task_id} completed:") + print(f"{Fore.WHITE}Result: {result[:200]}{'...' if len(result) > 200 else ''}") + print() + +async def example_1_simple_sequential_pipeline(): + """Example 1: Simple sequential pipeline using chain methods.""" + print_section("Example 1: Simple Sequential Pipeline") + + # Create workforce with simple workers + workforce = Workforce("Sequential Analysis Team") + + # Add some basic workers + data_collector = ChatAgent( + BaseMessage.make_assistant_message( + role_name="Data Collector", + content="You are a data collector. Collect and organize market data efficiently." + ) + ) + + data_processor = ChatAgent( + BaseMessage.make_assistant_message( + role_name="Data Processor", + content="You are a data processor. Clean and process raw data for analysis." + ) + ) + + report_generator = ChatAgent( + BaseMessage.make_assistant_message( + role_name="Report Generator", + content="You are a report generator. Create comprehensive reports from processed data." + ) + ) + + workforce.add_single_agent_worker("Data Collector", data_collector) + workforce.add_single_agent_worker("Data Processor", data_processor) + workforce.add_single_agent_worker("Report Generator", report_generator) + + # Build pipeline using chain methods + workforce.add_pipeline_task("Collect current market data for technology stocks").then_pipeline("Clean and validate the collected market data").then_pipeline("Generate a summary report of technology stock trends").finalize_pipeline() + + print(f"{Fore.YELLOW}Pipeline built with 3 sequential tasks") + print(f"{Fore.YELLOW}Mode: {workforce.mode}") + + # Execute the pipeline + main_task = Task( + content="Technology Stock Market Analysis Pipeline", + id="sequential_analysis" + ) + + start_time = time.time() + result = await workforce.process_task_async(main_task) + end_time = time.time() + + print_result(result.id, result.result or "Pipeline completed successfully") + print(f"{Fore.BLUE}[TIME] Execution time: {end_time - start_time:.2f} seconds") + + return result + +async def example_2_parallel_and_sync(): + """Example 2: Parallel tasks with synchronization.""" + print_section("Example 2: Parallel Tasks with Synchronization") + + # Create workforce with specialized workers + workforce = Workforce("Parallel Analysis Team") + + # Add specialized workers for different analysis types + workers = [ + ("Data Collector", "You collect and organize financial market data."), + ("Technical Analyst", "You perform technical analysis on stock charts and indicators."), + ("Fundamental Analyst", "You analyze company fundamentals and financial statements."), + ("Sentiment Analyst", "You analyze market sentiment and news impact."), + ("Report Synthesizer", "You synthesize multiple analyses into comprehensive reports.") + ] + + for role, description in workers: + agent = ChatAgent( + BaseMessage.make_assistant_message(role_name=role, content=description), + tools=[*ThinkingToolkit().get_tools()] + ) + workforce.add_single_agent_worker(role, agent) + + # Build pipeline with parallel analysis + workforce.add_pipeline_task("Collect comprehensive market data for AAPL, GOOGL, and MSFT").fork_pipeline([ + "Perform technical analysis on collected stock data using indicators", + "Conduct fundamental analysis of company financial health", + "Analyze market sentiment and recent news impact" + ]).join_pipeline("Synthesize all analyses into a comprehensive investment report").finalize_pipeline() + + print(f"{Fore.YELLOW}Pipeline built with 1 initial task, 3 parallel tasks, and 1 sync task") + print(f"{Fore.YELLOW}Mode: {workforce.mode}") + + # Execute the pipeline + main_task = Task( + content="Comprehensive Stock Analysis Pipeline", + id="parallel_analysis" + ) + + start_time = time.time() + result = await workforce.process_task_async(main_task) + end_time = time.time() + + print_result(result.id, result.result or "Parallel pipeline completed successfully") + print(f"{Fore.BLUE}[TIME] Execution time: {end_time - start_time:.2f} seconds") + + return result + +async def example_3_direct_pipeline_builder(): + """Example 3: Using PipelineTaskBuilder directly.""" + print_section("Example 3: Direct PipelineTaskBuilder Usage") + + # Create workforce + workforce = Workforce("Direct Builder Team") + + # Add workers + for i, role in enumerate(["Researcher", "Analyst", "Writer", "Reviewer"]): + agent = ChatAgent( + BaseMessage.make_assistant_message( + role_name=role, + content=f"You are a {role.lower()} specializing in market research." + ) + ) + workforce.add_single_agent_worker(f"{role} {i+1}", agent) + + # Use PipelineTaskBuilder directly for more control + builder = PipelineTaskBuilder() + + # Build a complex pipeline with explicit dependencies + research_id = builder.add_task( + "Research emerging technology trends and market opportunities", + "research_task" + ) + + analysis_ids = builder.add_parallel_tasks( + [ + "Analyze market size and growth potential", + "Assess competitive landscape and key players", + "Evaluate technology adoption trends" + ], + dependencies=[research_id], + task_id_prefix="analysis" + ) + + synthesis_id = builder.add_sync_point( + "Synthesize research and analysis into key insights", + wait_for=analysis_ids, + task_id="synthesis" + ) + + builder.add_task( + "Write a comprehensive market opportunity report", + "final_report", + dependencies=[synthesis_id] + ) + + # Build and set tasks + tasks = builder.build() + workforce.set_pipeline_tasks(tasks) + + print(f"{Fore.YELLOW}Pipeline built with PipelineTaskBuilder") + print(f"{Fore.YELLOW}Tasks: {len(tasks)}") + print(f"{Fore.YELLOW}Mode: {workforce.mode}") + + # Show task information + task_info = builder.get_task_info() + print(f"{Fore.CYAN}Pipeline structure:") + for task in task_info["tasks"]: + deps = ", ".join(task["dependencies"]) if task["dependencies"] else "None" + print(f" {Fore.WHITE}{task['id']}: {task['content'][:50]}... (deps: {deps})") + + # Execute the pipeline + main_task = Task( + content="Technology Market Opportunity Analysis", + id="direct_builder_analysis" + ) + + start_time = time.time() + result = await workforce.process_task_async(main_task) + end_time = time.time() + + print_result(result.id, result.result or "Direct builder pipeline completed successfully") + print(f"{Fore.BLUE}[TIME] Execution time: {end_time - start_time:.2f} seconds") + + return result + +async def example_4_complex_workflow(): + """Example 4: Complex workflow with multiple branches and joins.""" + print_section("Example 4: Complex Multi-Branch Workflow") + + # Create a larger workforce for complex analysis + workforce = Workforce("Complex Analysis Workforce", mode=WorkforceMode.PIPELINE) + + # Add various specialized workers + worker_configs = [ + ("Market Researcher", "You research market trends and collect data."), + ("Risk Analyst", "You analyze financial and market risks."), + ("Growth Analyst", "You analyze growth potential and opportunities."), + ("Tech Analyst", "You analyze technology trends and innovations."), + ("Competitive Analyst", "You analyze competitive landscapes."), + ("Financial Modeler", "You create financial models and projections."), + ("Strategy Consultant", "You develop strategic recommendations."), + ("Report Writer", "You write comprehensive business reports.") + ] + + for role, description in worker_configs: + agent = ChatAgent( + BaseMessage.make_assistant_message(role_name=role, content=description) + ) + workforce.add_single_agent_worker(role, agent) + + # Build complex pipeline + workforce.add_pipeline_task("Gather comprehensive market and industry data").fork_pipeline([ + "Analyze market risks and potential challenges", + "Analyze growth opportunities and market potential", + "Analyze technology trends and innovations" + ]).join_pipeline("Synthesize initial market analysis findings").add_parallel_pipeline_tasks([ + "Conduct competitive landscape analysis", + "Create financial projections and models" + ]).add_sync_pipeline_task("Integrate competitive and financial analysis").then_pipeline("Develop strategic recommendations and action plan").then_pipeline("Write comprehensive strategic analysis report").finalize_pipeline() + + print(f"{Fore.YELLOW}Complex pipeline built with multiple parallel branches") + print(f"{Fore.YELLOW}Mode: {workforce.mode}") + + # Execute the complex pipeline + main_task = Task( + content="Comprehensive Strategic Market Analysis", + id="complex_workflow" + ) + + start_time = time.time() + result = await workforce.process_task_async(main_task) + end_time = time.time() + + print_result(result.id, result.result or "Complex workflow completed successfully") + print(f"{Fore.BLUE}[TIME] Execution time: {end_time - start_time:.2f} seconds") + + return result + +async def example_5_mode_comparison(): + """Example 5: Compare AUTO_DECOMPOSE vs PIPELINE modes.""" + print_section("Example 5: Mode Comparison") + + # Test task + test_task = Task( + content="Analyze the cryptocurrency market trends and provide investment recommendations", + id="crypto_analysis" + ) + + # Create two identical workforces + auto_workforce = Workforce("Auto Decompose Team", mode=WorkforceMode.AUTO_DECOMPOSE) + pipeline_workforce = Workforce("Pipeline Team", mode=WorkforceMode.PIPELINE) + + # Add same workers to both + for workforce_instance in [auto_workforce, pipeline_workforce]: + for role in ["Crypto Analyst", "Market Researcher", "Investment Advisor"]: + agent = ChatAgent( + BaseMessage.make_assistant_message( + role_name=role, + content=f"You are a {role.lower()} specializing in cryptocurrency markets." + ) + ) + workforce_instance.add_single_agent_worker(role, agent) + + # Set up pipeline for pipeline workforce + pipeline_workforce.add_pipeline_task("Research current cryptocurrency market conditions").add_pipeline_task("Analyze major cryptocurrency trends and patterns").add_pipeline_task("Generate investment recommendations based on analysis").finalize_pipeline() + + print(f"{Fore.YELLOW}Testing AUTO_DECOMPOSE mode...") + start_time = time.time() + auto_result = await auto_workforce.process_task_async(test_task) + auto_time = time.time() - start_time + + print(f"{Fore.YELLOW}Testing PIPELINE mode...") + start_time = time.time() + pipeline_result = await pipeline_workforce.process_task_async(test_task) + pipeline_time = time.time() - start_time + + # Compare results + print(f"\n{Fore.CYAN}Comparison Results:") + print(f"{Fore.WHITE}AUTO_DECOMPOSE mode:") + print(f" Time: {auto_time:.2f}s") + print(f" Result length: {len(auto_result.result or '')}") + print(f" Status: {auto_result.state}") + + print(f"\n{Fore.WHITE}PIPELINE mode:") + print(f" Time: {pipeline_time:.2f}s") + print(f" Result length: {len(pipeline_result.result or '')}") + print(f" Status: {pipeline_result.state}") + + return auto_result, pipeline_result + +async def example_6_error_handling(): + """Example 6: Pipeline with error handling and recovery.""" + print_section("Example 6: Error Handling and Recovery") + + workforce = Workforce("Error Handling Team") + + # Add workers + agent = ChatAgent( + BaseMessage.make_assistant_message( + role_name="Resilient Analyst", + content="You are an analyst who handles errors gracefully and provides alternative solutions." + ) + ) + workforce.add_single_agent_worker("Resilient Analyst", agent) + + try: + # Build pipeline with potential error scenarios + workforce.add_pipeline_task("Attempt to access restricted financial data").then_pipeline("Process data even if some sources fail").then_pipeline("Generate report with available information and note limitations").finalize_pipeline() + + main_task = Task( + content="Resilient Data Analysis Pipeline", + id="error_handling_test" + ) + + result = await workforce.process_task_async(main_task) + print_result(result.id, result.result or "Error handling pipeline completed") + + except Exception as e: + print(f"{Fore.RED}[ERROR] Pipeline failed with error: {e}") + print(f"{Fore.YELLOW}This demonstrates error handling in pipeline execution") + + return result + +def print_summary(): + """Print example summary.""" + print_section("Pipeline Workflow Examples Summary") + + examples = [ + "1. Simple Sequential Pipeline - Basic chain of dependent tasks", + "2. Parallel Tasks with Sync - Fork/join pattern for parallel processing", + "3. Direct PipelineTaskBuilder - Advanced control with explicit dependencies", + "4. Complex Multi-Branch - Multiple parallel branches with synchronization", + "5. Mode Comparison - AUTO_DECOMPOSE vs PIPELINE performance", + "6. Error Handling - Graceful failure handling in pipelines" + ] + + print(f"{Fore.CYAN}Examples demonstrated:") + for example in examples: + print(f"{Fore.WHITE} [CHECK] {example}") + + print(f"\n{Fore.GREEN}Key Pipeline Features:") + features = [ + "[CHAIN] Chain methods: .add_pipeline_task().then_pipeline().fork_pipeline()", + "[PARALLEL] Parallel execution: .fork_pipeline() and .add_parallel_pipeline_tasks()", + "[SYNC] Synchronization: .join_pipeline() and .add_sync_pipeline_task()", + "[AUTO] Auto-dependency: Automatic inference of task dependencies", + "[MANUAL] Manual control: Explicit dependency specification when needed", + "[BUILDER] Builder access: Direct PipelineTaskBuilder usage for advanced scenarios", + "[MODE] Mode switching: Automatic PIPELINE mode activation", + "[ERROR] Error handling: Graceful failure recovery and reporting" + ] + + for feature in features: + print(f"{Fore.WHITE} {feature}") + +async def main(): + """Main function to run all pipeline examples.""" + print_section("CAMEL Workforce Pipeline Examples") + print(f"{Fore.YELLOW}This script demonstrates comprehensive Pipeline functionality in CAMEL Workforce.") + print(f"{Fore.YELLOW}Each example shows different patterns and capabilities.") + + results = {} + + try: + # Run all examples + print(f"\n{Fore.BLUE}Running all pipeline examples...") + + results['sequential'] = await example_1_simple_sequential_pipeline() + results['parallel'] = await example_2_parallel_and_sync() + results['direct_builder'] = await example_3_direct_pipeline_builder() + results['complex'] = await example_4_complex_workflow() + results['comparison'] = await example_5_mode_comparison() + results['error_handling'] = await example_6_error_handling() + + print_summary() + + print(f"\n{Fore.GREEN}[SUCCESS] All pipeline examples completed successfully!") + print(f"{Fore.CYAN}Total examples run: {len(results)}") + + except Exception as e: + print(f"\n{Fore.RED}[ERROR] Error running examples: {e}") + raise + + return results + +if __name__ == "__main__": + # Run the examples + asyncio.run(main()) \ No newline at end of file From 9ce04ee383ce1ed0e904876f0bb2d622adda08e2 Mon Sep 17 00:00:00 2001 From: Ol1ver0413 <790221864@qq.com> Date: Sun, 21 Sep 2025 10:54:29 -0400 Subject: [PATCH 2/6] wk_pipeline_0.2 --- camel/societies/workforce/__init__.py | 3 +- camel/societies/workforce/utils.py | 34 - camel/societies/workforce/workforce.py | 48 +- .../workforce/pipeline_workflow_example.py | 1289 +++++++++++++---- 4 files changed, 1027 insertions(+), 347 deletions(-) diff --git a/camel/societies/workforce/__init__.py b/camel/societies/workforce/__init__.py index 3990e8c97a..af7bbeca5a 100644 --- a/camel/societies/workforce/__init__.py +++ b/camel/societies/workforce/__init__.py @@ -15,12 +15,11 @@ from .role_playing_worker import RolePlayingWorker from .single_agent_worker import SingleAgentWorker from .utils import PipelineTaskBuilder -from .workforce import PipelineBuilder, Workforce, WorkforceMode +from .workforce import Workforce, WorkforceMode __all__ = [ "Workforce", "WorkforceMode", - "PipelineBuilder", "PipelineTaskBuilder", "SingleAgentWorker", "RolePlayingWorker", diff --git a/camel/societies/workforce/utils.py b/camel/societies/workforce/utils.py index 7796486f30..6b8cb325bd 100644 --- a/camel/societies/workforce/utils.py +++ b/camel/societies/workforce/utils.py @@ -300,26 +300,6 @@ def add_sync_task( return self.add_task(content, task_id, dependencies=wait_for, auto_depend=False) - def add_sync_point( - self, - content: str, - wait_for: List[str], - task_id: Optional[str] = None, - ) -> 'PipelineTaskBuilder': - """Add a synchronization point task that waits for multiple tasks. - - This is an alias for add_sync_task() for backward compatibility. - - Args: - content (str): Content of the synchronization task. - wait_for (List[str]): List of task IDs to wait for. - task_id (str, optional): ID for the sync task. If None, a unique - ID will be generated. (default: :obj:`None`) - - Returns: - PipelineTaskBuilder: Self for method chaining. - """ - return self.add_sync_task(content, wait_for, task_id) def build(self) -> List: """Build and return the complete task list with dependencies. @@ -378,20 +358,6 @@ def join(self, content: str, task_id: Optional[str] = None) -> 'PipelineTaskBuil """ return self.add_sync_task(content, task_id=task_id) - def then(self, content: str, task_id: Optional[str] = None) -> 'PipelineTaskBuilder': - """Add a sequential task (alias for add_task for better readability). - - Args: - content (str): Content of the task. - task_id (str, optional): ID for the task. - - Returns: - PipelineTaskBuilder: Self for method chaining. - - Example: - >>> builder.add_task("Start").then("Middle").then("End") - """ - return self.add_task(content, task_id) def _validate_dependencies(self) -> None: """Validate that there are no circular dependencies. diff --git a/camel/societies/workforce/workforce.py b/camel/societies/workforce/workforce.py index 4f07a0c633..8f8fcd269f 100644 --- a/camel/societies/workforce/workforce.py +++ b/camel/societies/workforce/workforce.py @@ -281,6 +281,7 @@ def __init__( # Pipeline building state self._pipeline_builder: Optional[PipelineTaskBuilder] = None + self._pipeline_tasks_need_assignment: bool = False # Dictionary to track task start times self._task_start_times: Dict[str, float] = {} # Human intervention support @@ -651,19 +652,6 @@ def join_pipeline(self, content: str, task_id: Optional[str] = None) -> 'Workfor self._pipeline_builder.join(content, task_id) return self - def then_pipeline(self, content: str, task_id: Optional[str] = None) -> 'Workforce': - """Add a sequential task (alias for add_pipeline_task for better readability). - - Args: - content (str): Content of the task. - task_id (str, optional): ID for the task. - - Returns: - Workforce: Self for method chaining. - """ - self._ensure_pipeline_builder() - self._pipeline_builder.then(content, task_id) - return self def finalize_pipeline(self) -> 'Workforce': """Finalize the pipeline and set up the tasks for execution. @@ -724,6 +712,9 @@ def set_pipeline_tasks(self, tasks: List[Task]) -> None: self._task_dependencies[task.id] = [dep.id for dep in task.dependencies] else: self._task_dependencies[task.id] = [] + + # Mark that pipeline tasks need assignment + self._pipeline_tasks_need_assignment = True def _collect_shared_memory(self) -> Dict[str, List]: r"""Collect memory from all SingleAgentWorker instances for sharing. @@ -1975,6 +1966,7 @@ def reset(self) -> None: self._completed_tasks = [] self._assignees.clear() self._in_flight_tasks = 0 + self._pipeline_tasks_need_assignment = False self.coordinator_agent.reset() self.task_agent.reset() self._task_start_times.clear() @@ -2687,11 +2679,22 @@ async def _post_ready_tasks(self) -> None: tasks whose dependencies have been met.""" # Step 1: Identify and assign any new tasks in the pending queue - tasks_to_assign = [ - task - for task in self._pending_tasks - if task.id not in self._task_dependencies - ] + # For pipeline mode, assign all tasks that don't have assignees yet + # For other modes, assign tasks that don't have dependencies tracked + if self.mode == WorkforceMode.PIPELINE and self._pipeline_tasks_need_assignment: + tasks_to_assign = [ + task + for task in self._pending_tasks + if task.id not in self._assignees + ] + # Reset the flag after assignment + self._pipeline_tasks_need_assignment = False + else: + tasks_to_assign = [ + task + for task in self._pending_tasks + if task.id not in self._task_dependencies + ] if tasks_to_assign: logger.debug( f"Found {len(tasks_to_assign)} new tasks. " @@ -2703,9 +2706,12 @@ async def _post_ready_tasks(self) -> None: f"{json.dumps(batch_result.dict(), indent=2)}" ) for assignment in batch_result.assignments: - self._task_dependencies[assignment.task_id] = ( - assignment.dependencies - ) + # For pipeline mode, dependencies are already set, only update assignees + # For other modes, update both dependencies and assignees + if self.mode != WorkforceMode.PIPELINE: + self._task_dependencies[assignment.task_id] = ( + assignment.dependencies + ) self._assignees[assignment.task_id] = assignment.assignee_id if self.metrics_logger: # queue_time_seconds can be derived by logger if task diff --git a/examples/workforce/pipeline_workflow_example.py b/examples/workforce/pipeline_workflow_example.py index 28132c1ba1..39b5f5ca1c 100644 --- a/examples/workforce/pipeline_workflow_example.py +++ b/examples/workforce/pipeline_workflow_example.py @@ -16,25 +16,25 @@ """ Pipeline Workflow Example - Comprehensive demonstration of CAMEL Workforce Pipeline functionality. -This example demonstrates all the Pipeline features of CAMEL Workforce: -1. Sequential task chains -2. Parallel task execution -3. Task synchronization and joining -4. Different pipeline building approaches -5. Auto-dependency inference -6. Manual dependency specification -7. Complex workflow patterns - -The example simulates a market analysis workflow with data collection, -parallel analysis, and report generation. +This example demonstrates practical Pipeline features of CAMEL Workforce: +1. Literature analysis with parallel summarization +2. API documentation processing +3. Data analysis with fork-join patterns +4. Complex multi-stage workflows +5. Real-world use cases + +The examples show how to handle branching tasks where multiple agents +work on different parts of the same data source. """ import asyncio +import json import time from typing import Dict, Any from colorama import Fore, Style, init +from camel.configs import ChatGPTConfig, GeminiConfig from camel.agents import ChatAgent from camel.models import ModelFactory from camel.societies.workforce import ( @@ -44,13 +44,25 @@ SingleAgentWorker ) from camel.tasks import Task -from camel.toolkits import SearchToolkit, ThinkingToolkit +from camel.toolkits import ThinkingToolkit from camel.types import ModelPlatformType, ModelType from camel.messages import BaseMessage # Initialize colorama for colored output init(autoreset=True) +# Create model once for all agents +# model = ModelFactory.create( +# model_platform=ModelPlatformType.DEFAULT, +# model_type=ModelType.DEFAULT, +# ) + +model = ModelFactory.create( + model_platform=ModelPlatformType.OPENAI, + model_type=ModelType.GPT_4_1, + model_config_dict=ChatGPTConfig().as_dict(), +) + def print_section(title: str): """Print a colored section header.""" print(f"\n{Fore.CYAN}{'='*60}") @@ -63,386 +75,1083 @@ def print_result(task_id: str, result: str): print(f"{Fore.WHITE}Result: {result[:200]}{'...' if len(result) > 200 else ''}") print() -async def example_1_simple_sequential_pipeline(): - """Example 1: Simple sequential pipeline using chain methods.""" - print_section("Example 1: Simple Sequential Pipeline") - - # Create workforce with simple workers - workforce = Workforce("Sequential Analysis Team") + +async def example_1_literature_analysis_pipeline(): + """Example 1: Literature analysis with parallel summarization.""" + print_section("Example 1: Literature Analysis with Parallel Processing") - # Add some basic workers - data_collector = ChatAgent( + # Create coordinator agent + coordinator_agent = ChatAgent( BaseMessage.make_assistant_message( - role_name="Data Collector", - content="You are a data collector. Collect and organize market data efficiently." - ) + role_name="Literature Analysis Coordinator", + content="You are a coordinator responsible for managing literature analysis tasks and ensuring quality output." + ), + model=model ) - data_processor = ChatAgent( + # Create task agent using the same model + task_agent = ChatAgent( BaseMessage.make_assistant_message( - role_name="Data Processor", - content="You are a data processor. Clean and process raw data for analysis." - ) + role_name="Task Planning Agent", + content="You are a task planning agent responsible for analyzing and coordinating complex tasks." + ), + model=model ) - report_generator = ChatAgent( - BaseMessage.make_assistant_message( - role_name="Report Generator", - content="You are a report generator. Create comprehensive reports from processed data." + # Create workforce for literature analysis + workforce = Workforce("Literature Analysis Team", mode=WorkforceMode.PIPELINE, coordinator_agent=coordinator_agent, task_agent=task_agent) + + # Add search agent with search tools (using mock data for demonstration) + search_system_message = BaseMessage.make_assistant_message( + role_name="Literature Researcher", + content="You are a literature researcher. Provide 5 recent AI/ML papers with titles, authors, and brief descriptions. Format them as [Paper 1], [Paper 2], etc. Use representative examples from recent AI/ML research." + ) + search_agent = ChatAgent( + system_message=search_system_message, + model=model, + tools=[] + ) + + # Add multiple summary agents for parallel processing + for i in range(5): + summary_system_message = BaseMessage.make_assistant_message( + role_name=f"Summary Specialist {i+1}", + content="You are a literature summary specialist. Focus on extracting key insights, methodologies, and contributions from research papers." + ) + summary_agent = ChatAgent( + system_message=summary_system_message, + model=model, + tools=[] ) + workforce.add_single_agent_worker(f"Summary Specialist {i+1}", summary_agent) + + # Add synthesis agent + synthesis_system_message = BaseMessage.make_assistant_message( + role_name="Research Synthesizer", + content="You are a research synthesizer. Combine multiple literature summaries into comprehensive analysis and identify research trends." + ) + synthesis_agent = ChatAgent( + system_message=synthesis_system_message, + model=model, + tools=[] ) - workforce.add_single_agent_worker("Data Collector", data_collector) - workforce.add_single_agent_worker("Data Processor", data_processor) - workforce.add_single_agent_worker("Report Generator", report_generator) + workforce.add_single_agent_worker("Literature Researcher", search_agent) + workforce.add_single_agent_worker("Research Synthesizer", synthesis_agent) - # Build pipeline using chain methods - workforce.add_pipeline_task("Collect current market data for technology stocks").then_pipeline("Clean and validate the collected market data").then_pipeline("Generate a summary report of technology stock trends").finalize_pipeline() + # Build literature analysis pipeline + workforce.add_pipeline_task("Generate 5 representative recent AI/ML papers with titles, authors, and brief descriptions. Format as [Paper 1] to [Paper 5]") \ + .fork_pipeline([ + "Summarize [Paper 1] core insights, methodology and contributions", + "Summarize [Paper 2] core insights, methodology and contributions", + "Summarize [Paper 3] core insights, methodology and contributions", + "Summarize [Paper 4] core insights, methodology and contributions", + "Summarize [Paper 5] core insights, methodology and contributions" + ]) \ + .join_pipeline("Analyze AI/ML research trends based on the 5 paper summaries") \ + .finalize_pipeline() - print(f"{Fore.YELLOW}Pipeline built with 3 sequential tasks") + print(f"{Fore.YELLOW}Literature analysis pipeline built: 1 search β†’ 5 parallel summaries β†’ 1 synthesis") print(f"{Fore.YELLOW}Mode: {workforce.mode}") # Execute the pipeline main_task = Task( - content="Technology Stock Market Analysis Pipeline", - id="sequential_analysis" + content="AI/ML Literature Review and Trend Analysis", + id="literature_analysis" ) start_time = time.time() result = await workforce.process_task_async(main_task) end_time = time.time() - print_result(result.id, result.result or "Pipeline completed successfully") + print_result(result.id, result.result or "Literature analysis completed successfully") print(f"{Fore.BLUE}[TIME] Execution time: {end_time - start_time:.2f} seconds") return result -async def example_2_parallel_and_sync(): - """Example 2: Parallel tasks with synchronization.""" - print_section("Example 2: Parallel Tasks with Synchronization") +async def example_2_api_documentation_pipeline(): + """Example 2: API documentation processing with parallel analysis.""" + print_section("Example 2: API Documentation Processing Pipeline") - # Create workforce with specialized workers - workforce = Workforce("Parallel Analysis Team") + # Create coordinator agent + coordinator_agent = ChatAgent( + BaseMessage.make_assistant_message( + role_name="API Documentation Coordinator", + content="You are a coordinator responsible for managing API documentation analysis and ensuring comprehensive coverage." + ), + model=model + ) - # Add specialized workers for different analysis types - workers = [ - ("Data Collector", "You collect and organize financial market data."), - ("Technical Analyst", "You perform technical analysis on stock charts and indicators."), - ("Fundamental Analyst", "You analyze company fundamentals and financial statements."), - ("Sentiment Analyst", "You analyze market sentiment and news impact."), - ("Report Synthesizer", "You synthesize multiple analyses into comprehensive reports.") - ] + # Create task agent using the same model + task_agent = ChatAgent( + BaseMessage.make_assistant_message( + role_name="Task Planning Agent", + content="You are a task planning agent responsible for analyzing and coordinating complex tasks." + ), + model=model + ) - for role, description in workers: - agent = ChatAgent( - BaseMessage.make_assistant_message(role_name=role, content=description), - tools=[*ThinkingToolkit().get_tools()] + # Create workforce for API documentation analysis + workforce = Workforce("API Documentation Team", mode=WorkforceMode.PIPELINE, coordinator_agent=coordinator_agent, task_agent=task_agent) + + # Add API discovery agent (using mock data for demonstration) + api_researcher_system_message = BaseMessage.make_assistant_message( + role_name="API Researcher", + content="You are an API researcher. Generate 4 core RESTful API endpoint types with descriptions. Format them as [API 1], [API 2], etc. Use common REST patterns like GET, POST, PUT, DELETE." + ) + api_researcher = ChatAgent( + system_message=api_researcher_system_message, + model=model, + tools=[] + ) + + # Add multiple API testing/analysis agents + for i in range(4): + api_analyst_system_message = BaseMessage.make_assistant_message( + role_name=f"API Analyst {i+1}", + content="You are an API analyst. Test API endpoints, analyze functionality, and document usage patterns and best practices." ) - workforce.add_single_agent_worker(role, agent) + api_analyst = ChatAgent( + system_message=api_analyst_system_message, + model=model, + tools=[] + ) + workforce.add_single_agent_worker(f"API Analyst {i+1}", api_analyst) + + # Add documentation writer + doc_writer_system_message = BaseMessage.make_assistant_message( + role_name="Documentation Writer", + content="You are a technical documentation writer. Create comprehensive API documentation from analysis results." + ) + doc_writer = ChatAgent( + system_message=doc_writer_system_message, + model=model, + tools=[] + ) + + workforce.add_single_agent_worker("API Researcher", api_researcher) + workforce.add_single_agent_worker("Documentation Writer", doc_writer) - # Build pipeline with parallel analysis - workforce.add_pipeline_task("Collect comprehensive market data for AAPL, GOOGL, and MSFT").fork_pipeline([ - "Perform technical analysis on collected stock data using indicators", - "Conduct fundamental analysis of company financial health", - "Analyze market sentiment and recent news impact" - ]).join_pipeline("Synthesize all analyses into a comprehensive investment report").finalize_pipeline() + # Build API documentation pipeline + workforce.add_pipeline_task("Generate 4 core RESTful API endpoint types (GET, POST, PUT, DELETE) with descriptions and format as [API 1] to [API 4]") \ + .fork_pipeline([ + "Analyze [API 1] functionality, parameters, responses and use cases", + "Analyze [API 2] functionality, parameters, responses and use cases", + "Analyze [API 3] functionality, parameters, responses and use cases", + "Analyze [API 4] functionality, parameters, responses and use cases" + ]) \ + .join_pipeline("Generate complete API usage guide based on 4 endpoint analyses") \ + .finalize_pipeline() - print(f"{Fore.YELLOW}Pipeline built with 1 initial task, 3 parallel tasks, and 1 sync task") + print(f"{Fore.YELLOW}API documentation pipeline built: 1 discovery β†’ 4 parallel analyses β†’ 1 synthesis") print(f"{Fore.YELLOW}Mode: {workforce.mode}") # Execute the pipeline main_task = Task( - content="Comprehensive Stock Analysis Pipeline", - id="parallel_analysis" + content="RESTful API Documentation and Best Practices Guide", + id="api_documentation" ) start_time = time.time() result = await workforce.process_task_async(main_task) end_time = time.time() - print_result(result.id, result.result or "Parallel pipeline completed successfully") + print_result(result.id, result.result or "API documentation completed successfully") print(f"{Fore.BLUE}[TIME] Execution time: {end_time - start_time:.2f} seconds") return result -async def example_3_direct_pipeline_builder(): - """Example 3: Using PipelineTaskBuilder directly.""" - print_section("Example 3: Direct PipelineTaskBuilder Usage") +async def example_3_code_review_pipeline(): + """Example 3: Code review with multiple file analysis.""" + print_section("Example 3: Code Review Pipeline") - # Create workforce - workforce = Workforce("Direct Builder Team") - - # Add workers - for i, role in enumerate(["Researcher", "Analyst", "Writer", "Reviewer"]): - agent = ChatAgent( - BaseMessage.make_assistant_message( - role_name=role, - content=f"You are a {role.lower()} specializing in market research." - ) - ) - workforce.add_single_agent_worker(f"{role} {i+1}", agent) - - # Use PipelineTaskBuilder directly for more control - builder = PipelineTaskBuilder() - - # Build a complex pipeline with explicit dependencies - research_id = builder.add_task( - "Research emerging technology trends and market opportunities", - "research_task" + # Create coordinator agent + coordinator_agent = ChatAgent( + BaseMessage.make_assistant_message( + role_name="Code Review Coordinator", + content="You are a coordinator responsible for managing code review processes and ensuring thorough analysis." + ), + model=model ) - analysis_ids = builder.add_parallel_tasks( - [ - "Analyze market size and growth potential", - "Assess competitive landscape and key players", - "Evaluate technology adoption trends" - ], - dependencies=[research_id], - task_id_prefix="analysis" + # Create task agent using the same model + task_agent = ChatAgent( + BaseMessage.make_assistant_message( + role_name="Task Planning Agent", + content="You are a task planning agent responsible for analyzing and coordinating complex tasks." + ), + model=model ) - synthesis_id = builder.add_sync_point( - "Synthesize research and analysis into key insights", - wait_for=analysis_ids, - task_id="synthesis" - ) + # Create workforce for code review + workforce = Workforce("Code Review Team", mode=WorkforceMode.PIPELINE, coordinator_agent=coordinator_agent, task_agent=task_agent) - builder.add_task( - "Write a comprehensive market opportunity report", - "final_report", - dependencies=[synthesis_id] - ) + # Sample code files for review + code_files = { + "file1.py": """ +def calculate_total(items): + total = 0 + for item in items: + total += item.price * item.quantity + return total + +class ShoppingCart: + def __init__(self): + self.items = [] - # Build and set tasks - tasks = builder.build() - workforce.set_pipeline_tasks(tasks) + def add_item(self, item): + self.items.append(item) - print(f"{Fore.YELLOW}Pipeline built with PipelineTaskBuilder") - print(f"{Fore.YELLOW}Tasks: {len(tasks)}") - print(f"{Fore.YELLOW}Mode: {workforce.mode}") + def get_total(self): + return calculate_total(self.items) +""", + "file2.py": """ +import requests +import json + +def fetch_user_data(user_id): + url = f"https://api.example.com/users/{user_id}" + response = requests.get(url) + if response.status_code == 200: + return json.loads(response.text) + return None + +def process_users(user_ids): + results = [] + for uid in user_ids: + data = fetch_user_data(uid) + if data: + results.append(data) + return results +""", + "file3.py": """ +class DatabaseConnection: + def __init__(self, host, port, database): + self.host = host + self.port = port + self.database = database + self.connection = None - # Show task information - task_info = builder.get_task_info() - print(f"{Fore.CYAN}Pipeline structure:") - for task in task_info["tasks"]: - deps = ", ".join(task["dependencies"]) if task["dependencies"] else "None" - print(f" {Fore.WHITE}{task['id']}: {task['content'][:50]}... (deps: {deps})") + def connect(self): + # Simulate database connection + self.connection = f"connected to {self.host}:{self.port}/{self.database}" + return True - # Execute the pipeline - main_task = Task( - content="Technology Market Opportunity Analysis", - id="direct_builder_analysis" + def execute_query(self, query): + if not self.connection: + raise Exception("Not connected to database") + return f"Executed: {query}" +""" + } + + # Add code scanner agent + scanner_system_message = BaseMessage.make_assistant_message( + role_name="Code Scanner", + content="You are a code scanner. Analyze code files and format findings as [File 1], [File 2], [File 3] with code snippets." + ) + scanner_agent = ChatAgent( + system_message=scanner_system_message, + model=model, + tools=[] ) - start_time = time.time() - result = await workforce.process_task_async(main_task) - end_time = time.time() + # Add multiple code reviewers for parallel processing + for i in range(3): + reviewer_system_message = BaseMessage.make_assistant_message( + role_name=f"Code Reviewer {i+1}", + content="You are a code reviewer. Review code for bugs, performance issues, and best practices." + ) + reviewer_agent = ChatAgent( + system_message=reviewer_system_message, + model=model, + tools=[] + ) + workforce.add_single_agent_worker(f"Code Reviewer {i+1}", reviewer_agent) - print_result(result.id, result.result or "Direct builder pipeline completed successfully") - print(f"{Fore.BLUE}[TIME] Execution time: {end_time - start_time:.2f} seconds") + # Add report generator + report_system_message = BaseMessage.make_assistant_message( + role_name="Review Summarizer", + content="You are a review summarizer. Compile code review findings into comprehensive reports." + ) + report_agent = ChatAgent( + system_message=report_system_message, + model=model, + tools=[] + ) - return result - -async def example_4_complex_workflow(): - """Example 4: Complex workflow with multiple branches and joins.""" - print_section("Example 4: Complex Multi-Branch Workflow") - - # Create a larger workforce for complex analysis - workforce = Workforce("Complex Analysis Workforce", mode=WorkforceMode.PIPELINE) - - # Add various specialized workers - worker_configs = [ - ("Market Researcher", "You research market trends and collect data."), - ("Risk Analyst", "You analyze financial and market risks."), - ("Growth Analyst", "You analyze growth potential and opportunities."), - ("Tech Analyst", "You analyze technology trends and innovations."), - ("Competitive Analyst", "You analyze competitive landscapes."), - ("Financial Modeler", "You create financial models and projections."), - ("Strategy Consultant", "You develop strategic recommendations."), - ("Report Writer", "You write comprehensive business reports.") - ] + workforce.add_single_agent_worker("Code Scanner", scanner_agent) + workforce.add_single_agent_worker("Review Summarizer", report_agent) - for role, description in worker_configs: - agent = ChatAgent( - BaseMessage.make_assistant_message(role_name=role, content=description) - ) - workforce.add_single_agent_worker(role, agent) - - # Build complex pipeline - workforce.add_pipeline_task("Gather comprehensive market and industry data").fork_pipeline([ - "Analyze market risks and potential challenges", - "Analyze growth opportunities and market potential", - "Analyze technology trends and innovations" - ]).join_pipeline("Synthesize initial market analysis findings").add_parallel_pipeline_tasks([ - "Conduct competitive landscape analysis", - "Create financial projections and models" - ]).add_sync_pipeline_task("Integrate competitive and financial analysis").then_pipeline("Develop strategic recommendations and action plan").then_pipeline("Write comprehensive strategic analysis report").finalize_pipeline() - - print(f"{Fore.YELLOW}Complex pipeline built with multiple parallel branches") + # Build code review pipeline + workforce.add_pipeline_task(f"Scan these code files and format as [File 1], [File 2], [File 3]:\n\n{json.dumps(code_files, indent=2)}") \ + .fork_pipeline([ + "Review [File 1] for code quality, bugs, and improvements", + "Review [File 2] for code quality, bugs, and improvements", + "Review [File 3] for code quality, bugs, and improvements" + ]) \ + .join_pipeline("Generate comprehensive code review report with all findings") \ + .finalize_pipeline() + + print(f"{Fore.YELLOW}Code review pipeline built: Scan β†’ 3 parallel reviews β†’ Summary report") print(f"{Fore.YELLOW}Mode: {workforce.mode}") - # Execute the complex pipeline + # Execute the pipeline main_task = Task( - content="Comprehensive Strategic Market Analysis", - id="complex_workflow" + content="Code Review Analysis Pipeline", + id="code_review" ) start_time = time.time() result = await workforce.process_task_async(main_task) end_time = time.time() - print_result(result.id, result.result or "Complex workflow completed successfully") + print_result(result.id, result.result or "Code review pipeline completed successfully") print(f"{Fore.BLUE}[TIME] Execution time: {end_time - start_time:.2f} seconds") return result -async def example_5_mode_comparison(): - """Example 5: Compare AUTO_DECOMPOSE vs PIPELINE modes.""" - print_section("Example 5: Mode Comparison") - - # Test task - test_task = Task( - content="Analyze the cryptocurrency market trends and provide investment recommendations", - id="crypto_analysis" - ) - - # Create two identical workforces - auto_workforce = Workforce("Auto Decompose Team", mode=WorkforceMode.AUTO_DECOMPOSE) - pipeline_workforce = Workforce("Pipeline Team", mode=WorkforceMode.PIPELINE) - - # Add same workers to both - for workforce_instance in [auto_workforce, pipeline_workforce]: - for role in ["Crypto Analyst", "Market Researcher", "Investment Advisor"]: - agent = ChatAgent( - BaseMessage.make_assistant_message( - role_name=role, - content=f"You are a {role.lower()} specializing in cryptocurrency markets." - ) - ) - workforce_instance.add_single_agent_worker(role, agent) - - # Set up pipeline for pipeline workforce - pipeline_workforce.add_pipeline_task("Research current cryptocurrency market conditions").add_pipeline_task("Analyze major cryptocurrency trends and patterns").add_pipeline_task("Generate investment recommendations based on analysis").finalize_pipeline() - - print(f"{Fore.YELLOW}Testing AUTO_DECOMPOSE mode...") - start_time = time.time() - auto_result = await auto_workforce.process_task_async(test_task) - auto_time = time.time() - start_time - - print(f"{Fore.YELLOW}Testing PIPELINE mode...") - start_time = time.time() - pipeline_result = await pipeline_workforce.process_task_async(test_task) - pipeline_time = time.time() - start_time - - # Compare results - print(f"\n{Fore.CYAN}Comparison Results:") - print(f"{Fore.WHITE}AUTO_DECOMPOSE mode:") - print(f" Time: {auto_time:.2f}s") - print(f" Result length: {len(auto_result.result or '')}") - print(f" Status: {auto_result.state}") - - print(f"\n{Fore.WHITE}PIPELINE mode:") - print(f" Time: {pipeline_time:.2f}s") - print(f" Result length: {len(pipeline_result.result or '')}") - print(f" Status: {pipeline_result.state}") - - return auto_result, pipeline_result - -async def example_6_error_handling(): - """Example 6: Pipeline with error handling and recovery.""" - print_section("Example 6: Error Handling and Recovery") - - workforce = Workforce("Error Handling Team") - - # Add workers - agent = ChatAgent( - BaseMessage.make_assistant_message( - role_name="Resilient Analyst", - content="You are an analyst who handles errors gracefully and provides alternative solutions." - ) - ) - workforce.add_single_agent_worker("Resilient Analyst", agent) - - try: - # Build pipeline with potential error scenarios - workforce.add_pipeline_task("Attempt to access restricted financial data").then_pipeline("Process data even if some sources fail").then_pipeline("Generate report with available information and note limitations").finalize_pipeline() - - main_task = Task( - content="Resilient Data Analysis Pipeline", - id="error_handling_test" - ) - - result = await workforce.process_task_async(main_task) - print_result(result.id, result.result or "Error handling pipeline completed") - - except Exception as e: - print(f"{Fore.RED}[ERROR] Pipeline failed with error: {e}") - print(f"{Fore.YELLOW}This demonstrates error handling in pipeline execution") - - return result def print_summary(): """Print example summary.""" - print_section("Pipeline Workflow Examples Summary") + print_section("Pipeline Examples Summary") examples = [ - "1. Simple Sequential Pipeline - Basic chain of dependent tasks", - "2. Parallel Tasks with Sync - Fork/join pattern for parallel processing", - "3. Direct PipelineTaskBuilder - Advanced control with explicit dependencies", - "4. Complex Multi-Branch - Multiple parallel branches with synchronization", - "5. Mode Comparison - AUTO_DECOMPOSE vs PIPELINE performance", - "6. Error Handling - Graceful failure handling in pipelines" + "1. Literature Analysis - Fork/join pattern with 5 parallel agents", + "2. API Documentation - 4 parallel analysts with structured parsing", + "3. Data Analysis - Multi-stage workflow with dataset branching" ] print(f"{Fore.CYAN}Examples demonstrated:") for example in examples: - print(f"{Fore.WHITE} [CHECK] {example}") - - print(f"\n{Fore.GREEN}Key Pipeline Features:") - features = [ - "[CHAIN] Chain methods: .add_pipeline_task().then_pipeline().fork_pipeline()", - "[PARALLEL] Parallel execution: .fork_pipeline() and .add_parallel_pipeline_tasks()", - "[SYNC] Synchronization: .join_pipeline() and .add_sync_pipeline_task()", - "[AUTO] Auto-dependency: Automatic inference of task dependencies", - "[MANUAL] Manual control: Explicit dependency specification when needed", - "[BUILDER] Builder access: Direct PipelineTaskBuilder usage for advanced scenarios", - "[MODE] Mode switching: Automatic PIPELINE mode activation", - "[ERROR] Error handling: Graceful failure recovery and reporting" + print(f"{Fore.WHITE} {example}") + + print(f"\n{Fore.GREEN}Pipeline patterns tested:") + patterns = [ + "SEARCH BRANCH: Single search distributed to multiple specialists", + "FORK-JOIN: Parallel processing with automatic synchronization", + "STRUCTURED OUTPUT: Using [markers] for smart task distribution", + "SMART ROUTING: Agents identify their target data sections", + "PARALLEL SCALING: Easy adjustment of parallel worker count", + "DEPENDENCY CHAINS: Multi-stage workflows with proper sequencing" ] - for feature in features: + for pattern in patterns: + print(f"{Fore.WHITE} {pattern}") + + print(f"\n{Fore.CYAN}Technical features:") + tech_features = [ + "ModelFactory.create() for agent initialization", + "SearchToolkit integration", + "Pipeline mode with fork_pipeline() and join_pipeline()", + "Automatic dependency resolution", + "Structured output parsing", + "Multi-agent coordination" + ] + + for feature in tech_features: print(f"{Fore.WHITE} {feature}") async def main(): - """Main function to run all pipeline examples.""" + """Main function to run pipeline examples.""" print_section("CAMEL Workforce Pipeline Examples") - print(f"{Fore.YELLOW}This script demonstrates comprehensive Pipeline functionality in CAMEL Workforce.") - print(f"{Fore.YELLOW}Each example shows different patterns and capabilities.") + print(f"{Fore.YELLOW}Testing Pipeline mode with branching and parallel processing.") results = {} try: - # Run all examples - print(f"\n{Fore.BLUE}Running all pipeline examples...") + print(f"\n{Fore.BLUE}Running pipeline examples...") - results['sequential'] = await example_1_simple_sequential_pipeline() - results['parallel'] = await example_2_parallel_and_sync() - results['direct_builder'] = await example_3_direct_pipeline_builder() - results['complex'] = await example_4_complex_workflow() - results['comparison'] = await example_5_mode_comparison() - results['error_handling'] = await example_6_error_handling() + results['literature'] = await example_1_literature_analysis_pipeline() + results['api_docs'] = await example_2_api_documentation_pipeline() + results['code_review'] = await example_3_code_review_pipeline() print_summary() - print(f"\n{Fore.GREEN}[SUCCESS] All pipeline examples completed successfully!") - print(f"{Fore.CYAN}Total examples run: {len(results)}") + print(f"\n{Fore.GREEN}Pipeline examples completed successfully!") + print(f"{Fore.CYAN}Total examples: {len(results)}") except Exception as e: - print(f"\n{Fore.RED}[ERROR] Error running examples: {e}") + print(f"\n{Fore.RED}Error: {e}") + import traceback + traceback.print_exc() raise return results if __name__ == "__main__": # Run the examples - asyncio.run(main()) \ No newline at end of file + asyncio.run(main()) + +""" + +============================================================ +CAMEL Workforce Pipeline Examples +============================================================ +Testing Pipeline mode with branching and parallel processing. + +Running pipeline examples... + +============================================================ +Example 1: Literature Analysis with Parallel Processing +============================================================ +Literature analysis pipeline built: 1 search β†’ 5 parallel summaries β†’ 1 synthesis +Mode: WorkforceMode.PIPELINE +Worker node 515f890a-c013-4876-9d29-1dc3e65c9f69 (Literature Researcher) get task pipeline_task_0: Generate 5 representative recent AI/ML papers with titles, authors, and brief descriptions. Format as [Paper 1] to [Paper 5] +====== +Response from Worker node 515f890a-c013-4876-9d29-1dc3e65c9f69 (Literature Researcher): + +[Paper 1] +"Gemini: A Family of Highly Capable Multimodal Models" +Authors: F. Zhai, Z. Ma, Y. Du, and others (Google DeepMind, 2023) +Description: This paper introduces Gemini, a new family of large multimodal models capable of processing text, images, and audio. It demonstrates state-of-the-art performance on a range of benchmarks, including language understanding and video reasoning tasks. + +[Paper 2] +"Llama 2: Open Foundation and Fine-Tuned Chat Models" +Authors: Hugo Touvron, Louis Martin, Kevin Stone, and others (Meta AI, 2023) +Description: Llama 2 presents a suite of open-access large language models trained on textual data. These models show competitive results in various natural language processing tasks and offer strong baselines for further research in ethical and transparent AI development. + +[Paper 3] +"EfficientViT: Memory Efficient Vision Transformers with Cascaded Group Attention" +Authors: Xiangtai Li, Shangchen Han, Shiji Song, and others (Tsinghua University, 2024) +Description: EfficientViT addresses the high memory and compute costs of existing vision transformers. It introduces a cascaded group attention mechanism that achieves similar or better accuracy than previous approaches while drastically reducing resource requirements. + +[Paper 4] +"Self-Supervised Learning with SwAV: Unsupervised Learning of Visual Features by Contrasting Cluster Assignments" +Authors: Mathilde Caron, Priya Goyal, Piotr Bojanowski, and others (Facebook AI Research, 2023) +Description: SwAV proposes a clustering-based unsupervised learning algorithm for vision models. By contrasting multiple cluster assignments, it enables models to learn robust visual features without requiring labeled data, accelerating progress in self-supervised learning. + +[Paper 5] +"RetNet: Delving into Efficient and Scalable Transformers for Long Sequences" +Authors: Xunyu Lin, Xiang Kong, Renzhi Mao, and others (Microsoft Research, 2024) +Description: RetNet introduces mechanisms for transformers to more efficiently process very long sequences, reducing computation and memory bottlenecks. It achieves competitive results on NLP tasks with sequences thousands of tokens long. +====== +🎯 Task pipeline_task_0 completed successfully. +Worker node 16a98984-7d43-4fec-8abc-7127353d7ef7 (Summary Specialist 1) get task parallel_0_1: Summarize [Paper 1] core insights, methodology and contributions +Worker node 12050652-606c-4706-843b-460318d1ff1e (Summary Specialist 2) get task parallel_1_1: Summarize [Paper 2] core insights, methodology and contributions +Worker node 4ccbbd32-0d8f-4d8a-9892-69f9082fc2eb (Summary Specialist 3) get task parallel_2_1: Summarize [Paper 3] core insights, methodology and contributions +Worker node 556a1d0d-e1c0-4d7e-ad2f-4fdc8bc8993f (Summary Specialist 4) get task parallel_3_1: Summarize [Paper 4] core insights, methodology and contributions +Worker node 5cb31fc3-611c-423b-b45a-e67e87b53c67 (Summary Specialist 5) get task parallel_4_1: Summarize [Paper 5] core insights, methodology and contributions +====== +Response from Worker node 16a98984-7d43-4fec-8abc-7127353d7ef7 (Summary Specialist 1): + +Paper 1, "Gemini: A Family of Highly Capable Multimodal Models" by F. Zhai, Z. Ma, Y. Du, and others (Google DeepMind, 2023), presents the Gemini model family, a new suite of large multimodal models designed to process and understand inputs across text, images, and audio. + +Core Insights: Gemini demonstrates state-of-the-art performance on a variety of benchmarks spanning language understanding, video reasoning, and multimodal comprehension tasks. The central insight is that a unified architecture can achieve robust, high-performing multimodal understanding by leveraging large-scale pretraining and architectural innovations. + +Methodology: The Gemini models are trained on vast, diverse datasets that include both unimodal and multimodal content. The architecture integrates advanced components for processing different data types, allowing seamless fusion of textual, visual, and audio information. Comprehensive quantitative evaluations are performed on established benchmarks, comparing Gemini to prior state-of-the-art models. + +Contributions: Gemini establishes new benchmarks in multimodal AI, outperforming prior models on a range of tasks. It advances the field by offering a generalizable framework for multimodal learning, supporting complex tasks such as language-image-text reasoning and video-based question answering. Additionally, the paper provides detailed empirical results and analysis, setting a new standard for future multimodal model development. +====== +🎯 Task parallel_0_1 completed successfully. +====== +Response from Worker node 5cb31fc3-611c-423b-b45a-e67e87b53c67 (Summary Specialist 5): + +Paper 5, "RetNet: Delving into Efficient and Scalable Transformers for Long Sequences" (Xunyu Lin, Xiang Kong, Renzhi Mao, et al., Microsoft Research, 2024), addresses the computational and memory inefficiencies of standard Transformer architectures when processing very long sequences. + +Core Insights: The paper observes that existing Transformers struggle to efficiently handle sequences on the scale of thousands of tokens due to their quadratic memory and computation requirements. RetNet is proposed as a solution, introducing novel mechanisms designed to make transformers more scalable and resource-efficient for such cases. + +Methodology: RetNet rethinks the architecture of self-attention to specifically focus on improving efficiency for long-context natural language processing tasks. Although the detailed mechanisms are not specified in the brief, the paper claims to have introduced approaches to reduce both computation and memory bottlenecks typically seen in large-sequence processing. These enhancements are empirically validated on benchmarks involving lengthy input sequences. + +Contributions: (1) A new transformer-based architecture (RetNet) tailored for long-sequence efficiency, (2) Demonstration of competitive results on NLP tasks involving sequences with thousands of tokens, and (3) Advancements in reducing memory and computational overhead, enabling wider application of transformers on tasks that require handling long inputs. + +Overall, RetNet represents a key step forward in scalable deep learning for sequence modeling, making transformer models viable for previously prohibitive long-sequence applications. +====== +🎯 Task parallel_4_1 completed successfully. +====== +Response from Worker node 12050652-606c-4706-843b-460318d1ff1e (Summary Specialist 2): + +Paper 2, "Llama 2: Open Foundation and Fine-Tuned Chat Models" by Hugo Touvron et al. (Meta AI, 2023), presents core advances in open-access large language models (LLMs). + +Core Insights: Llama 2 models, trained on extensive textual datasets, demonstrate competitive results on multiple natural language processing (NLP) benchmarks. The work emphasizes ethical considerations, transparency, and provides models suitable for research and downstream fine-tuning. + +Methodology: The paper details the architectural design and training of Llama 2, including the use of high-quality, filtered text corpora. Various model sizes are trained with standard transformer-based architectures. Fine-tuning techniques tailored for chat and alignment with human feedback are described, ensuring the models perform well in dialog and instruction-following tasks while mitigating biases and unsafe outputs. + +Contributions: (1) Introduction of an openly available suite of LLMs with both base and fine-tuned "chat" versions, (2) rigorous evaluation and benchmarking across a wide set of NLP tasks, showing state-of-the-art or near SOTA performance for open models, (3) a strong baseline for future research in responsible, transparent, and accessible AI development, and (4) detailed discussions on responsible model use and deployment. +====== +🎯 Task parallel_1_1 completed successfully. +====== +Response from Worker node 556a1d0d-e1c0-4d7e-ad2f-4fdc8bc8993f (Summary Specialist 4): + +Core Insights: 'Self-Supervised Learning with SwAV: Unsupervised Learning of Visual Features by Contrasting Cluster Assignments' introduces SwAV, a novel self-supervised learning method for training vision models without labeled data. The core insight is to exploit clustering for representation learning by comparing different image augmentations not directly, but via their assignments to clusters, enabling efficient learning of useful visual features. + +Methodology: SwAV stands for 'Swapping Assignments between Views'. The approach relies on applying multiple augmentations to each image and assigning each augmented view to a cluster (prototype) online. The model is trained to predict the cluster assignment of one view from the representation of another, effectively creating a self-supervised signal without the need for explicit pairing or negative samples (as in contrastive learning). SwAV employs online clustering and a 'multi-crop' augmentation strategy, increasing the diversity and number of image views per training step, which boosts performance and efficiency. + +Contributions: The paper's key contributions are: (1) a novel clustering-based approach to self-supervised visual feature learning that does not require pairwise comparisons across batch samples, (2) the introduction of the 'swapped prediction' mechanism as a training objective, and (3) demonstration of strong performance on standard image classification benchmarks (e.g., ImageNet), matching or surpassing existing self-supervised learning methods while improving training efficiency. SwAV's approach has influenced subsequent research in self-supervised learning and clustering-based representation learning. +====== +🎯 Task parallel_3_1 completed successfully. +====== +Response from Worker node 4ccbbd32-0d8f-4d8a-9892-69f9082fc2eb (Summary Specialist 3): + +Paper 3, titled "EfficientViT: Memory Efficient Vision Transformers with Cascaded Group Attention" (Xiangtai Li, Shangchen Han, Shiji Song, et al., Tsinghua University, 2024), addresses the high memory and computational costs associated with standard vision transformer (ViT) architectures. + +Core Insights: The paper identifies that existing vision transformers are resource-intensive, which limits their deployment in environments with constrained memory and compute capabilities. The authors propose a novel solution through the cascaded group attention mechanism, which selectively focuses attention computation within groups of tokens, thereby reducing overhead without sacrificing model performance. + +Methodology: EfficientViT introduces cascaded group attention, wherein the model organizes input image tokens into groups and applies attention mechanisms hierarchically and sparsely within and across these groups. This design reduces the quadratic memory and computation scaling of traditional ViTs to a more manageable level. The paper details formal architectural changes and provides empirical benchmarks to compare its efficiency and accuracy against baseline transformers. + +Contributions: The main contributions of the paper are (1) the novel cascaded group attention mechanism enabling memory- and compute-efficient vision transformers; (2) empirical demonstration that EfficientViT achieves comparable or superior accuracy on key vision tasks relative to standard ViTs while using significantly less memory and computation; and (3) the potential to broaden the practical deployment of transformer-based models in resource-limited environments. +====== +🎯 Task parallel_2_1 completed successfully. +Worker node de3d7797-579a-4002-b64b-8ef6d5abf5ed (Research Synthesizer) get task pipeline_task_6: Analyze AI/ML research trends based on the 5 paper summaries +====== +Response from Worker node de3d7797-579a-4002-b64b-8ef6d5abf5ed (Research Synthesizer): + +Analysis of the five AI/ML paper summaries reveals several prominent research trends shaping the current landscape: + +1. Multimodal Intelligence and Unified Architectures: Papers like Gemini (Paper 1) highlight the drive to create highly capable models that process and integrate various modalities (text, images, audio) within a unified framework. This trend enables models to solve complex tasks requiring cross-modal understanding, advancing generalist AI systems. + +2. Open and Responsible Large Language Models: Llama 2 (Paper 2) demonstrates a commitment to openness, transparency, and responsible deployment in LLMs. The emphasis on public availability, ethical considerations, and alignment with human preferences underlines the shift toward accessible AI research and safer model release practices. + +3. Efficient and Scalable Model Design: Both EfficientViT (Paper 3) and RetNet (Paper 5) focus on architectural innovation to address the growing computational and memory demands of state-of-the-art models. EfficientViT introduces cascaded group attention for resource-constrained vision transformers, while RetNet targets scalable transformers for processing long sequences in NLP, collectively marking a broader trend toward improving model efficiency and hardware accessibility. + +4. Advances in Self-Supervised and Unsupervised Learning: SwAV (Paper 4) showcases the impact of self-supervised learning and clustering for visual representations. Moving beyond labeled data and traditional contrastive methods, SwAV's cluster-based self-supervision enables sample efficiency and strong benchmark performance, reflecting a larger pivot to less supervised paradigm in representation learning. + +5. Benchmarking and Empirical Evaluation: All papers underscore rigorous empirical evaluation across diverse benchmarks as essential for validating new methods and architectures. This trend solidifies the importance of reproducible research and standardized comparison in propelling the field forward. + +Overall, the AI/ML research landscape is marked by a convergence of open, multimodal, and efficient model architectures; responsible AI deployment; and a continual shift towards less supervised, self-sufficient learning paradigms. Architectural and algorithmic innovations are tightly coupled with accessibility, scalability, and ethical considerations, driving the next generation of AI systems. +====== +🎯 Task pipeline_task_6 completed successfully. +Task literature_analysis completed: +Result: --- Task pipeline_task_0 Result --- +[Paper 1] +"Gemini: A Family of Highly Capable Multimodal Models" +Authors: F. Zhai, Z. Ma, Y. Du, and others (Google DeepMind, 2023) +Description: This paper introduc... + +[TIME] Execution time: 39.61 seconds + +============================================================ +Example 2: API Documentation Processing Pipeline +============================================================ +API documentation pipeline built: 1 discovery β†’ 4 parallel analyses β†’ 1 synthesis +Mode: WorkforceMode.PIPELINE +Worker node c79877ae-537b-440b-adb1-e9ccce756540 (API Researcher) get task pipeline_task_0: Generate 4 core RESTful API endpoint types (GET, POST, PUT, DELETE) with descriptions and format as [API 1] to [API 4] +====== +Response from Worker node c79877ae-537b-440b-adb1-e9ccce756540 (API Researcher): + +[API 1] GET /resources - Retrieves a list of resources or a single resource by ID. Typically used to read or fetch data from the server. + +[API 2] POST /resources - Creates a new resource. Sends data to the server and returns the newly created resource or its unique identifier. + +[API 3] PUT /resources/{id} - Updates an existing resource identified by its ID. Used to fully replace the contents of the resource with the provided data. + +[API 4] DELETE /resources/{id} - Deletes the specified resource by its ID. Removes the resource from the server. +====== +🎯 Task pipeline_task_0 completed successfully. +Worker node a1506367-5bb1-47ba-8532-4e0603830e4f (API Analyst 1) get task parallel_0_1: Analyze [API 1] functionality, parameters, responses and use cases +Worker node cef71fcc-1304-461b-8267-9217cb3e9cac (API Analyst 2) get task parallel_1_1: Analyze [API 2] functionality, parameters, responses and use cases +Worker node a786426e-ff39-458e-95b3-14453f37028c (API Analyst 3) get task parallel_2_1: Analyze [API 3] functionality, parameters, responses and use cases +Worker node bb1cf632-8df3-438d-afe3-f9499e6a3aea (API Analyst 4) get task parallel_3_1: Analyze [API 4] functionality, parameters, responses and use cases +====== +Response from Worker node a1506367-5bb1-47ba-8532-4e0603830e4f (API Analyst 1): + +[API 1] GET /resources - This endpoint is used to retrieve information about resources. + +Functionality: +- Allows clients to fetch a list of all resources. +- Can also be used to fetch a single resource by specifying an ID (often via query parameter or path, e.g., /resources/{id}). + +Parameters: +- Optional query parameters such as filters (e.g., ?type=active), pagination (e.g., ?page=2&limit=10), sorting (e.g., ?sort=name), and specific resource ID if retrieving a single resource. +- Authorization headers may be required if resources are protected. + +Responses: +- 200 OK: Success, returns an array of resource objects for a list, or a resource object for a specific ID. +- 404 Not Found: Returned when a specific resource ID does not exist. +- 401 Unauthorized/403 Forbidden: If access is denied or the client is not authenticated. +- Typical response body for list: [{"id":1, "name":"Resource 1", ...}, ...] +- Typical response body for single: {"id":1, "name":"Resource 1", ...} + +Use Cases: +- Displaying a list of available resources in an application dashboard. +- Fetching detailed information of a specific resource for viewing or editing. +- Supporting features like searching, filtering, and pagination in front-end apps consuming the resource list. + +In summary, [API 1] implements standard RESTful 'read' behavior for resources, supporting querying and retrieval for various client needs. +====== +🎯 Task parallel_0_1 completed successfully. +====== +Response from Worker node a786426e-ff39-458e-95b3-14453f37028c (API Analyst 3): + +[API 3] PUT /resources/{id} is an endpoint designed to update an existing resource on the server, where the resource is identified by its unique 'id'. + +Functionality: +- This endpoint replaces the current contents of the resource with the data provided in the request body. +- It expects the full representation of the resource in the request payload. + +Parameters: +- Path Parameter: 'id' (required) – the unique identifier of the resource to be updated. +- Request Body: A JSON (or other accepted format) object that represents the complete, updated state of the resource. The specific fields depend on the resource model (not specified here). +- Headers: Typically expects 'Content-Type: application/json' (can vary). + +Responses: +- 200 OK: Resource was successfully fully updated; the response typically returns the updated resource or a success message. +- 204 No Content: Resource updated successfully with no content returned (varies by implementation). +- 400 Bad Request: Invalid input data or malformed request. +- 404 Not Found: A resource with the specified 'id' does not exist. +- 401 Unauthorized/403 Forbidden: The client is not authorized to perform the action (if authentication is required). + +Use Cases: +- Updating all properties of a user's profile information by sending a complete updated profile to PUT /resources/123. +- Replacing an entire configuration object for a device or application. +- Correcting all fields of a previously submitted data record by resubmitting the entire object. + +Best Practices: +- Clients should send the complete representation of the resource, not just the fields that are changing. +- For partial updates, PATCH requests are more appropriate; PUT is intended for full replacement. +- Validate input data to avoid overwriting existing fields with null or default values inadvertently. + +Overall, [API 3] PUT /resources/{id} is used where full replacement of a resource's data is required, ensuring that the resource's state on the server exactly matches the client's submitted data. +====== +🎯 Task parallel_2_1 completed successfully. +====== +Response from Worker node bb1cf632-8df3-438d-afe3-f9499e6a3aea (API Analyst 4): + +Analysis of [API 4]: + +Functionality: +[API 4] is a RESTful endpoint defined as DELETE /resources/{id}. It removes a specific resource identified by its unique ID from the server. This operation is destructive and irreversible, making it essential for data management and cleanup workflows. + +Parameters: +- Path Parameter: id (required) – the unique identifier of the resource to be deleted. + +Typical endpoint format: DELETE /resources/12345 + +Responses: +- Success (typically 200 OK or 204 No Content): Indicates the resource was successfully deleted. Some implementations may return the deleted resource data or just a confirmation message. +- 404 Not Found: Returned if the resource with the provided ID does not exist. +- 400 Bad Request: Returned if the ID is improperly formatted or invalid. +- 401 Unauthorized / 403 Forbidden: Indicate lack of access permissions for the operation. + +Use Cases: +- Removing obsolete or irrelevant resources (e.g., deleting a user, file, or record). +- Supporting administrative functions (e.g., batch deletions). +- Implementing resource lifecycle management. +- Enabling users to retract or remove their own content or data. + +Best Practice: +- Ensure the operation is idempotent: multiple DELETE requests for the same non-existent resource should result in the same response (typically 404 Not Found). +- Consider implementing role-based access control to prevent unauthorized deletions. + +Summary: [API 4] enables clients to permanently remove resources by ID. Appropriate checks, confirmations, and security measures should be implemented to avoid accidental or unauthorized deletions. +====== +🎯 Task parallel_3_1 completed successfully. +====== +Response from Worker node cef71fcc-1304-461b-8267-9217cb3e9cac (API Analyst 2): + +[API 2] Functionality, Parameters, Responses, and Use Cases Analysis: + +Functionality: +[API 2] is a POST endpoint at /resources. Its primary function is to create a new resource on the server. The client sends the details of the resource to be created in the request body, and the server processes this request and returns either the created resource object or a unique identifier for the new resource. + +Parameters: +- Request Body: The specific properties or fields required for resource creation will depend on the type of resource the API manages, but typically this would be a JSON object containing required and optional fields (e.g., name, description, attributes of the resource). No path or query parameters are typically used for a create endpoint. +- Headers: Usually includes 'Content-Type: application/json' to indicate JSON payload; may also require authentication/authorization headers (e.g., 'Authorization: Bearer '). + +Responses: +- Success (201 Created): Returns the full details of the created resource (as a JSON object), or a JSON object with at least the unique identifier assigned (e.g., { "id": 123 }). +- Failure (4xx/5xx): + - 400 Bad Request: Returned if the input data is invalid or required fields are missing. + - 401 Unauthorized/403 Forbidden: If the user is not authenticated/authorized. + - 409 Conflict: If a resource with the same unique value already exists. + - 500 Internal Server Error: For unexpected server errors. + +Use Cases: +- Creating a new data entry (e.g., a new user profile, an article, a product in inventory). +- Submitting a form by an end user to add a new entity. +- Backend tools or automation scripts populating the system with new resources. + +Best Practices: +- Client should validate input before making the request. +- Server should return appropriate status codes and error messages for all failure scenarios. +- Use idempotency keys if creating resources where duplicate submissions should not create duplicate records. + +In summary, [API 2] POST /resources is the standard endpoint for resource creation, expects a well-formed JSON body with all mandatory fields, and returns the result of the operation with relevant HTTP status codes and a response object. +====== +🎯 Task parallel_1_1 completed successfully. +Worker node 743a178b-9d2a-4e42-901d-fdba755319fc (Documentation Writer) get task pipeline_task_5: Generate complete API usage guide based on 4 endpoint analyses +2025-09-21 10:50:54,339 - camel.models.model_manager - ERROR - Error processing with model: +2025-09-21 10:50:54,339 - camel.camel.agents.chat_agent - WARNING - Rate limit hit (attempt 1/3). Retrying in 0.1s +2025-09-21 10:51:11,829 - camel.models.model_manager - ERROR - Error processing with model: +2025-09-21 10:51:11,829 - camel.camel.agents.chat_agent - WARNING - Rate limit hit (attempt 2/3). Retrying in 0.3s +2025-09-21 10:51:44,868 - camel.camel.societies.workforce.structured_output_handler - WARNING - Failed to parse TaskResult from response. Using fallback values. +====== +Response from Worker node 743a178b-9d2a-4e42-901d-fdba755319fc (Documentation Writer): + +Task processing failed +====== +2025-09-21 10:51:44,868 - camel.camel.societies.workforce.workforce - ERROR - Task pipeline_task_5 failed (attempt 1/3): Task processing failed (assigned to worker: 743a178b-9d2a-4e42-901d-fdba755319fc) +Task api_documentation completed: +Result: --- Task pipeline_task_0 Result --- +[API 1] GET /resources - Retrieves a list of resources or a single resource by ID. Typically used to read or fetch data from the server. + +[API 2] POST /resources - ... + +[TIME] Execution time: 80.34 seconds + +============================================================ +Example 3: Code Review Pipeline +============================================================ +Code review pipeline built: Scan β†’ 3 parallel reviews β†’ Summary report +Mode: WorkforceMode.PIPELINE +Worker node d7079fe4-be5b-49b9-b3c6-2712633de9db (Code Scanner) get task pipeline_task_0: Scan these code files and format as [File 1], [File 2], [File 3]: + +{ + "file1.py": "\ndef calculate_total(items):\n total = 0\n for item in items:\n total += item.price * item.quantity\n return total\n\nclass ShoppingCart:\n def __init__(self):\n self.items = []\n \n def add_item(self, item):\n self.items.append(item)\n \n def get_total(self):\n return calculate_total(self.items)\n", + "file2.py": "\nimport requests\nimport json\n\ndef fetch_user_data(user_id):\n url = f\"https://api.example.com/users/{user_id}\"\n response = requests.get(url)\n if response.status_code == 200:\n return json.loads(response.text)\n return None\n\ndef process_users(user_ids):\n results = []\n for uid in user_ids:\n data = fetch_user_data(uid)\n if data:\n results.append(data)\n return results\n", + "file3.py": "\nclass DatabaseConnection:\n def __init__(self, host, port, database):\n self.host = host\n self.port = port \n self.database = database\n self.connection = None\n \n def connect(self):\n # Simulate database connection\n self.connection = f\"connected to {self.host}:{self.port}/{self.database}\"\n return True\n \n def execute_query(self, query):\n if not self.connection:\n raise Exception(\"Not connected to database\")\n return f\"Executed: {query}\"\n" +} +====== +Response from Worker node d7079fe4-be5b-49b9-b3c6-2712633de9db (Code Scanner): + +[File 1] +file1.py: + +``` +def calculate_total(items): + total = 0 + for item in items: + total += item.price * item.quantity + return total + +class ShoppingCart: + def __init__(self): + self.items = [] + + def add_item(self, item): + self.items.append(item) + + def get_total(self): + return calculate_total(self.items) +``` + +[File 2] +file2.py: + +``` +import requests +import json + +def fetch_user_data(user_id): + url = f"https://api.example.com/users/{user_id}" + response = requests.get(url) + if response.status_code == 200: + return json.loads(response.text) + return None + +def process_users(user_ids): + results = [] + for uid in user_ids: + data = fetch_user_data(uid) + if data: + results.append(data) + return results +``` + +[File 3] +file3.py: + +``` +class DatabaseConnection: + def __init__(self, host, port, database): + self.host = host + self.port = port + self.database = database + self.connection = None + + def connect(self): + # Simulate database connection + self.connection = f"connected to {self.host}:{self.port}/{self.database}" + return True + + def execute_query(self, query): + if not self.connection: + raise Exception("Not connected to database") + return f"Executed: {query}" +``` +====== +🎯 Task pipeline_task_0 completed successfully. +Worker node f24b0d6e-22ea-4029-a16d-ba88a271e316 (Code Reviewer 1) get task parallel_0_1: Review [File 1] for code quality, bugs, and improvements +Worker node c54b9e75-b1f6-449d-bc57-80c5c32a1fed (Code Reviewer 2) get task parallel_1_1: Review [File 2] for code quality, bugs, and improvements +Worker node 3993ab6f-8fa5-4176-8eee-1c3109c250ab (Code Reviewer 3) get task parallel_2_1: Review [File 3] for code quality, bugs, and improvements +====== +Response from Worker node f24b0d6e-22ea-4029-a16d-ba88a271e316 (Code Reviewer 1): + +Review of [File 1]: + +Code: +``` +def calculate_total(items): + total = 0 + for item in items: + total += item.price * item.quantity + return total + +class ShoppingCart: + def __init__(self): + self.items = [] + + def add_item(self, item): + self.items.append(item) + + def get_total(self): + return calculate_total(self.items) +``` + +Code Quality: +- The code is simple, readable, and follows standard Python coding conventions. +- Class and function names are descriptive. +- Method and function signatures are clear. + +Potential Bugs: +- There is an implicit assumption that `item` objects passed to `calculate_total` have `price` and `quantity` attributes. No type-checking or error handling is present for invalid items. +- There is no check for non-numeric or negative values in `price` or `quantity`. + +Performance: +- For reasonably sized lists, the current implementation is efficient. There are no obvious performance concerns. + +Improvements: +1. Add type and value checking in `calculate_total` for `price` and `quantity` fields. +2. Optionally, consider supporting property checking or try/except block in case items are malformed. +3. Optionally, provide typing annotations to improve code clarity and assist with static analysis. +4. Consider defining an `Item` class or namedtuple for items, to ensure structure and type safety in the cart. +5. Consider handling situations where an item might be added more than once (aggregate quantities). +6. Docstrings could be added for better documentation. + +Example of an improved calculate_total function: +```python +def calculate_total(items): + total = 0 + for item in items: + try: + price = float(item.price) + quantity = int(item.quantity) + if price < 0 or quantity < 0: + continue # or raise ValueError + total += price * quantity + except (AttributeError, ValueError, TypeError): + continue # or handle/log the error appropriately + return total +``` + +Overall, the current implementation is functional but can be made more robust with the suggested improvements. +====== +🎯 Task parallel_0_1 completed successfully. +====== +Response from Worker node c54b9e75-b1f6-449d-bc57-80c5c32a1fed (Code Reviewer 2): + +Review of [File 2] (file2.py): + +**Code Quality:** +- The code is readable and functionally straightforward. +- Usage of requests and json modules is appropriate. +- Function and variable naming are clear and concise. + +**Bugs:** +- No outright bugs are present in the code as written. Both functions perform their jobs as described. +- However, there is no exception handling for the HTTP request. If the GET request fails (e.g., due to network issues), raises an exception, or non-2xx responses other than 200 are received, the code does not handle these cases, which may cause the program to crash. + +**Performance Issues:** +- The process_users function runs fetch_user_data serially for each user_id. If there are many user_ids, this could lead to slow performance due to sequential network calls. Using concurrency (e.g., via threading, asyncio, or concurrent.futures) could considerably speed up processing large user_id lists. + +**Best Practices & Improvements:** +1. **Exception Handling:** Add exception handling to catch requests.exceptions.RequestException and possibly json.JSONDecodeError inside fetch_user_data to prevent crashes. +2. **Timeouts:** The requests.get() call lacks a timeout. This may potentially cause the call to hang indefinitely if the remote server does not respond. It is best practice to specify a timeout (e.g., requests.get(url, timeout=5)). +3. **Direct Usage of response.json():** Instead of manually decoding JSON with json.loads(response.text), you can use response.json(), which is cleaner and more efficient, and already parses JSON (though it can still throw exceptions if JSON is invalid). + +**Suggested Improved Version:** +```python +import requests + +def fetch_user_data(user_id): + url = f"https://api.example.com/users/{user_id}" + try: + response = requests.get(url, timeout=5) + response.raise_for_status() + return response.json() + except (requests.RequestException, ValueError): # ValueError for JSON decoding issues + return None + +def process_users(user_ids): + results = [] + for uid in user_ids: + data = fetch_user_data(uid) + if data: + results.append(data) + return results +``` +For improved performance, consider parallelizing requests for large lists. + +**Summary:** +- The code is functionally correct but lacks exception handling and timeout safety. +- Consider using response.json(), add try/except blocks, and set a timeout on network requests. +- For significant performance increases with large lists, parallel processing should be considered. + +====== +🎯 Task parallel_1_1 completed successfully. +====== +Response from Worker node 3993ab6f-8fa5-4176-8eee-1c3109c250ab (Code Reviewer 3): + +Review for [File 3] (file3.py): + +Code Quality: +- The class `DatabaseConnection` is written in a readable way and adheres to typical Python conventions. +- Method and variable naming is clear and descriptive. +- Docstrings and comments are minimal. Only one comment is present in `connect()`; adding docstrings to each method and the class itself would improve understandability and maintainability. + +Bugs: +- There are no functional bugs in this mock implementation, as it's simply simulating connections and query execution with string manipulations. +- However, the constructor does not validate host, port, or database arguments; passing invalid values could cause unexpected behaviors in a real-world scenario. + +Performance: +- Since this is a mock/simulated connection and not a real database connection, there are no significant performance concerns. In a real implementation, connecting to a database inside the object constructor or failing to close connections would be a problem, but those do not apply here. + +Best Practices & Improvements: +1. **Docstrings**: Add method and class docstrings for clarity. +2. **Connection State**: Consider adding a `disconnect()` method to fully simulate a connection lifecycle. +3. **Exception Specificity**: Raise a more specific exception (e.g., a custom `DatabaseConnectionError`) rather than a generic `Exception` when not connected. +4. **Validate Parameters**: Optionally, validate constructor parameters (`host`, `port`, `database`) to ensure they meet expected types/values. +5. **PEP8**: The code is PEP8-compliant except for an extra space after `self.port = port `; remove trailing white space. + +Example incorporating improvements: +```python +class DatabaseConnection: + """Simulates a database connection.""" + + def __init__(self, host, port, database): + self.host = host + self.port = port + self.database = database + self.connection = None + + def connect(self): + """Simulate establishing a database connection.""" + self.connection = f"connected to {self.host}:{self.port}/{self.database}" + return True + + def disconnect(self): + """Simulate closing a database connection.""" + self.connection = None + + def execute_query(self, query): + """Simulate executing a database query.""" + if not self.connection: + raise ConnectionError("Not connected to database") + return f"Executed: {query}" +``` + +Summary: The code is clean and correct for a fake/mock connection class. The suggestions above are for further robustness, readability, and code quality in cases where a more realistic simulation or extension is needed. +====== +🎯 Task parallel_2_1 completed successfully. +Worker node 24b9cc1f-a802-4789-a422-1b09fff96e07 (Review Summarizer) get task pipeline_task_4: Generate comprehensive code review report with all findings +====== +Response from Worker node 24b9cc1f-a802-4789-a422-1b09fff96e07 (Review Summarizer): + +Comprehensive Code Review Report + +1. [File 1] Shopping Cart Logic +--- +Code Quality: +- The code is readable and uses descriptive naming for classes and methods. +- The implementation is straightforward, but lacks documentation (docstrings) for functions and classes. + +Bugs & Reliability: +- Assumes items in the cart possess `price` and `quantity` attributes, but performs no type or attribute validation. This may lead to attribute errors if invalid objects are added. +- Negative or non-numeric values for `price` and `quantity` are not checked. + +Improvements Suggested: +- Add explicit attribute and type checking in `calculate_total`. +- Consider using try/except to catch malformed item entries. +- Implement type annotations for better code clarity. +- Provide an `Item` class or structure (e.g., namedtuple) for stronger type safety. +- Optionally, aggregate identical items rather than allowing duplicates. +- Add docstrings for all methods and classes for maintainability. + +2. [File 2] User Data Retrieval Functions +--- +Code Quality: +- Code is readable, function and variable names are clear. +- Makes appropriate use of the `requests` and `json` modules. + +Bugs & Reliability: +- Lacks exception handling for relevant failure scenarios, such as network errors or malformed JSON, leading to potential program crashes. + +Performance: +- The current implementation fetches user data serially for each ID, which is inefficient for large lists. Consider concurrency or parallelization for improved performance. + +Best Practices & Improvements: +- Add try/except blocks for network and JSON decoding errors (e.g., `requests.RequestException`, `ValueError`). +- Set a timeout in HTTP requests to prevent hanging. +- Use `response.json()` instead of manual JSON parsing for safety and simplicity. +- For large user lists, investigate parallel execution (e.g., threading or async calls). + +3. [File 3] Mock DatabaseConnection Class +--- +Code Quality: +- Code is well organized, readable, and mostly PEP8 compliant (except one minor whitespace issue). + +Bugs & Reliability: +- Class is a mock and not functionally problematic, but lacks validation of constructor parameters (host, port, database). +- Only one comment is present and there are no docstrings; code documentation should be added. +- The `execute_query` method raises a generic Exception for not being connected. + +Improvements Suggested: +- Add class-level and method-level docstrings for maintainability. +- Validate constructor arguments for expected types and values. +- Add a `disconnect` method to fully represent the connection lifecycle. +- Raise more specific exceptions (e.g., custom `ConnectionError`) for better error handling. +- Remove extra whitespace to maintain PEP8 compliance. + +--- + +General Recommendations: +- Across all files, improve input validation, type safety, and add comprehensive docstrings. +- Consistent and specific exception handling will improve robustness and maintainability. +- Consider performance implications and use concurrency where relevant when interacting with external systems (e.g., web APIs). + +All code reviewed is functional within scope, but can be improved in terms of error handling, input validation, and code documentation for production use. +====== +🎯 Task pipeline_task_4 completed successfully. +Task code_review completed: +Result: --- Task pipeline_task_0 Result --- +[File 1] +file1.py: + +``` +def calculate_total(items): + total = 0 + for item in items: + total += item.price * item.quantity + return total + +class Shoppin... + +[TIME] Execution time: 33.88 seconds + +============================================================ +Pipeline Examples Summary +============================================================ +Examples demonstrated: + 1. Literature Analysis - Fork/join pattern with 5 parallel agents + 2. API Documentation - 4 parallel analysts with structured parsing + 3. Data Analysis - Multi-stage workflow with dataset branching + +Pipeline patterns tested: + SEARCH BRANCH: Single search distributed to multiple specialists + FORK-JOIN: Parallel processing with automatic synchronization + STRUCTURED OUTPUT: Using [markers] for smart task distribution + SMART ROUTING: Agents identify their target data sections + PARALLEL SCALING: Easy adjustment of parallel worker count + DEPENDENCY CHAINS: Multi-stage workflows with proper sequencing + +Technical features: + ModelFactory.create() for agent initialization + SearchToolkit integration + Pipeline mode with fork_pipeline() and join_pipeline() + Automatic dependency resolution + Structured output parsing + Multi-agent coordination + +Pipeline examples completed successfully! +Total examples: 3 + +""" \ No newline at end of file From 9648550a3d662614c43bff18fb455c1e84224b82 Mon Sep 17 00:00:00 2001 From: Ol1ver0413 <790221864@qq.com> Date: Wed, 22 Oct 2025 00:14:42 +0800 Subject: [PATCH 3/6] wk_pipeline_0.3 --- camel/societies/workforce/utils.py | 12 +- camel/societies/workforce/workforce.py | 245 ++-- .../workforce/pipeline_workflow_example.py | 1049 +++-------------- 3 files changed, 318 insertions(+), 988 deletions(-) diff --git a/camel/societies/workforce/utils.py b/camel/societies/workforce/utils.py index a146658c1b..570a46e6cb 100644 --- a/camel/societies/workforce/utils.py +++ b/camel/societies/workforce/utils.py @@ -233,7 +233,7 @@ def __init__(self): self._last_task_id = None # Track the last added task for chain inference self._last_parallel_tasks = [] # Track the last added parallel tasks for sync - def add_task( + def add( self, content: str, task_id: Optional[str] = None, @@ -262,7 +262,7 @@ def add_task( ValueError: If task_id already exists or if any dependency is not found. Example: - >>> builder.add_task("Step 1").add_task("Step 2").add_task("Step 3") + >>> builder.add("Step 1").add("Step 2").add("Step 3") # Step 2 depends on Step 1, Step 3 depends on Step 2 """ # Generate or validate task_id @@ -324,7 +324,7 @@ def add_parallel_tasks( ValueError: If any task_id already exists or if any dependency is not found. Example: - >>> builder.add_task("Collect Data").add_parallel_tasks([ + >>> builder.add("Collect Data").add_parallel_tasks([ ... "Technical Analysis", "Fundamental Analysis" ... ]).add_sync_task("Generate Report") """ @@ -341,7 +341,7 @@ def add_parallel_tasks( for i, content in enumerate(task_contents): task_id = f"{task_id_prefix}_{i}_{base_counter}" # Use auto_depend=False since we're manually managing dependencies - self.add_task(content, task_id, dependencies, auto_depend=False) + self.add(content, task_id, dependencies, auto_depend=False) parallel_task_ids.append(task_id) # Set the last task to None since we have multiple parallel endings @@ -390,7 +390,7 @@ def add_sync_task( if not wait_for: raise ValueError("wait_for cannot be empty for sync task") - return self.add_task(content, task_id, dependencies=wait_for, auto_depend=False) + return self.add(content, task_id, dependencies=wait_for, auto_depend=False) def build(self) -> List: @@ -429,7 +429,7 @@ def fork(self, task_contents: List[str]) -> 'PipelineTaskBuilder': PipelineTaskBuilder: Self for method chaining. Example: - >>> builder.add_task("Collect Data").fork([ + >>> builder.add("Collect Data").fork([ ... "Technical Analysis", "Fundamental Analysis" ... ]).join("Generate Report") """ diff --git a/camel/societies/workforce/workforce.py b/camel/societies/workforce/workforce.py index ea2df64ccc..cf18122842 100644 --- a/camel/societies/workforce/workforce.py +++ b/camel/societies/workforce/workforce.py @@ -69,7 +69,7 @@ from camel.societies.workforce.task_channel import TaskChannel from camel.societies.workforce.utils import ( FailureContext, - RecoveryDecision, + PipelineTaskBuilder, RecoveryStrategy, TaskAnalysisResult, TaskAssignment, @@ -77,6 +77,8 @@ WorkerConf, check_if_running, ) +from camel.societies.workforce.events import WorkerCreatedEvent +from camel.societies.workforce.workforce_callback import WorkforceCallback from camel.societies.workforce.worker import Worker from camel.tasks.task import ( Task, @@ -135,9 +137,8 @@ class WorkforceState(Enum): class WorkforceMode(Enum): r"""Workforce execution mode for different task processing strategies.""" - AUTO_DECOMPOSE = "auto_decompose" # Current automatic task decomposition mode + AUTO_DECOMPOSE = "auto_decompose" # Automatic task decomposition mode PIPELINE = "pipeline" # Predefined pipeline mode - HYBRID = "hybrid" # Hybrid mode allowing partial auto-decomposition class WorkforceSnapshot: @@ -228,17 +229,12 @@ class Workforce(BaseNode): support native structured output. When disabled, the workforce uses the native response_format parameter. (default: :obj:`True`) - callbacks (Optional[List[WorkforceCallback]], optional): A list of - callback handlers to observe and record workforce lifecycle events - and metrics (e.g., task creation/assignment/start/completion/ - failure, worker creation/deletion, all-tasks-completed). All - items must be instances of :class:`WorkforceCallback`, otherwise - a :class:`ValueError` is raised. If none of the provided - callbacks implement :class:`WorkforceMetrics`, a built-in - :class:`WorkforceLogger` (implements both callback and metrics) - is added automatically. If at least one provided callback - implements :class:`WorkforceMetrics`, no default logger is added. - (default: :obj:`None`) + mode (WorkforceMode, optional): The execution mode for task + processing. AUTO_DECOMPOSE mode uses intelligent recovery + strategies (decompose, replan, etc.) when tasks fail. + PIPELINE mode uses simple retry logic and allows failed + tasks to continue the workflow, passing error information + to dependent tasks. (default: :obj:`WorkforceMode.AUTO_DECOMPOSE`) Example: >>> import asyncio @@ -291,6 +287,8 @@ def __init__( share_memory: bool = False, use_structured_output_handler: bool = True, task_timeout_seconds: Optional[float] = None, + mode: WorkforceMode = WorkforceMode.AUTO_DECOMPOSE, + callbacks: Optional[List[WorkforceCallback]] = None, ) -> None: super().__init__(description) self._child_listening_tasks: Deque[ @@ -498,24 +496,22 @@ def __init__( def _initialize_callbacks( self, callbacks: Optional[List[WorkforceCallback]] ) -> None: - r"""Validate, register, and prime workforce callbacks.""" - self._callbacks: List[WorkforceCallback] = [] - - if callbacks: - for cb in callbacks: - if isinstance(cb, WorkforceCallback): - self._callbacks.append(cb) - else: - raise ValueError( - "All callbacks must be instances of WorkforceCallback" - ) - + r"""Initialize workforce callbacks.""" + self._callbacks: List[WorkforceCallback] = callbacks or [] + + # Check if any metrics callback is provided has_metrics_callback = any( - isinstance(cb, WorkforceMetrics) for cb in self._callbacks + hasattr(cb, 'get_workforce_kpis') for cb in self._callbacks ) - + if not has_metrics_callback: - self._callbacks.append(WorkforceLogger(workforce_id=self.node_id)) + # Add default WorkforceLogger if no metrics callback provided + try: + from camel.societies.workforce.workforce_logger import WorkforceLogger + self._callbacks.append(WorkforceLogger(workforce_id=self.node_id)) + except ImportError: + # If WorkforceLogger is not available, continue without it + pass else: logger.info( "WorkforceMetrics implementation detected. Skipping default " @@ -645,13 +641,14 @@ def __repr__(self): def _ensure_pipeline_builder(self): """Ensure pipeline builder is initialized and switch to pipeline mode.""" if self._pipeline_builder is None: + from camel.societies.workforce.utils import PipelineTaskBuilder self._pipeline_builder = PipelineTaskBuilder() # Auto-switch to pipeline mode if self.mode != WorkforceMode.PIPELINE: self.mode = WorkforceMode.PIPELINE - def add_pipeline_task( + def pipeline_add( self, content: str, task_id: Optional[str] = None, @@ -677,10 +674,10 @@ def add_pipeline_task( Workforce: Self for method chaining. Example: - >>> workforce.add_pipeline_task("Step 1").add_pipeline_task("Step 2").add_pipeline_task("Step 3") + >>> workforce.pipeline_add("Step 1").pipeline_add("Step 2").pipeline_add("Step 3") """ self._ensure_pipeline_builder() - self._pipeline_builder.add_task(content, task_id, dependencies, additional_info, auto_depend) + self._pipeline_builder.add(content, task_id, dependencies, additional_info, auto_depend) return self def add_parallel_pipeline_tasks( @@ -729,21 +726,26 @@ def add_sync_pipeline_task( self._pipeline_builder.add_sync_task(content, wait_for, task_id) return self - def fork_pipeline(self, task_contents: List[str]) -> 'Workforce': - """Create parallel branches from the current task (alias for add_parallel_pipeline_tasks). + def pipeline_fork(self, task_contents: List[str]) -> 'Workforce': + """Create parallel branches from the current task. Args: task_contents (List[str]): List of task content strings for parallel execution. Returns: Workforce: Self for method chaining. + + Example: + >>> workforce.pipeline_add("Collect Data").pipeline_fork([ + ... "Technical Analysis", "Fundamental Analysis" + ... ]).pipeline_join("Generate Report") """ self._ensure_pipeline_builder() self._pipeline_builder.fork(task_contents) return self - def join_pipeline(self, content: str, task_id: Optional[str] = None) -> 'Workforce': - """Join parallel branches with a synchronization task (alias for add_sync_pipeline_task). + def pipeline_join(self, content: str, task_id: Optional[str] = None) -> 'Workforce': + """Join parallel branches with a synchronization task. Args: content (str): Content of the join/sync task. @@ -751,17 +753,25 @@ def join_pipeline(self, content: str, task_id: Optional[str] = None) -> 'Workfor Returns: Workforce: Self for method chaining. + + Example: + >>> workforce.pipeline_fork(["Task A", "Task B"]).pipeline_join("Merge Results") """ self._ensure_pipeline_builder() self._pipeline_builder.join(content, task_id) return self - def finalize_pipeline(self) -> 'Workforce': - """Finalize the pipeline and set up the tasks for execution. + def pipeline_build(self) -> 'Workforce': + """Build the pipeline and set up the tasks for execution. Returns: Workforce: Self for method chaining. + + Example: + >>> workforce.pipeline_add("Step 1").pipeline_fork([ + ... "Task A", "Task B" + ... ]).pipeline_join("Merge").pipeline_build() """ if self._pipeline_builder is None: raise ValueError("No pipeline tasks defined") @@ -779,7 +789,7 @@ def get_pipeline_builder(self) -> PipelineTaskBuilder: Example: >>> builder = workforce.get_pipeline_builder() - >>> builder.add_task("Complex Task").fork(["A", "B"]).join("Merge") + >>> builder.add("Complex Task").fork(["A", "B"]).join("Merge") >>> tasks = builder.build() >>> workforce.set_pipeline_tasks(tasks) """ @@ -1975,26 +1985,31 @@ async def process_task_async( if interactive: return await self._process_task_with_snapshot(task) - subtasks = await self.handle_decompose_append_task(task) + # Handle different execution modes + if self.mode == WorkforceMode.PIPELINE: + return await self._process_task_with_pipeline(task) + else: + # AUTO_DECOMPOSE mode (default) + subtasks = await self.handle_decompose_append_task(task) - self.set_channel(TaskChannel()) + self.set_channel(TaskChannel()) - await self.start() + await self.start() - if subtasks: - task.result = "\n\n".join( - f"--- Subtask {sub.id} Result ---\n{sub.result}" - for sub in task.subtasks - if sub.result - ) - if task.subtasks and all( - sub.state == TaskState.DONE for sub in task.subtasks - ): - task.state = TaskState.DONE - else: - task.state = TaskState.FAILED + if subtasks: + task.result = "\n\n".join( + f"--- Subtask {sub.id} Result ---\n{sub.result}" + for sub in task.subtasks + if sub.result + ) + if task.subtasks and all( + sub.state == TaskState.DONE for sub in task.subtasks + ): + task.state = TaskState.DONE + else: + task.state = TaskState.FAILED - return task + return task async def _process_task_with_pipeline(self, task: Task) -> Task: """Process task using predefined pipeline tasks.""" @@ -2005,13 +2020,17 @@ async def _process_task_with_pipeline(self, task: Task) -> Task: # Don't reset here - keep the predefined tasks self._task = task - if self.metrics_logger: - self.metrics_logger.log_task_created( - task_id=task.id, - description=task.content, - task_type=task.type, - metadata=task.additional_info, - ) + + # Log main task creation event through callbacks (following source code pattern) + task_created_event = TaskCreatedEvent( + task_id=task.id, + description=task.content, + parent_task_id=None, + task_type=task.type, + metadata=task.additional_info, + ) + for cb in self._callbacks: + cb.log_task_created(task_created_event) task.state = TaskState.FAILED self.set_channel(TaskChannel()) @@ -3544,11 +3563,20 @@ async def _post_ready_tasks(self) -> None: tasks whose dependencies have been met.""" # Step 1: Identify and assign any new tasks in the pending queue - tasks_to_assign = [ - task - for task in self._pending_tasks - if task.id not in self._task_dependencies - ] + # In PIPELINE mode, tasks already have dependencies set but need worker assignment + # In other modes, tasks without dependencies entry are new and need both + if self.mode == WorkforceMode.PIPELINE: + tasks_to_assign = [ + task + for task in self._pending_tasks + if task.id not in self._assignees + ] + else: + tasks_to_assign = [ + task + for task in self._pending_tasks + if task.id not in self._task_dependencies + ] if tasks_to_assign: logger.debug( f"Found {len(tasks_to_assign)} new tasks. " @@ -3618,21 +3646,26 @@ async def _post_ready_tasks(self) -> None: # Only proceed with dependency checks if all deps are completed if all_deps_completed: - # Check if all dependencies succeeded (state is DONE) - all_deps_done = all( - completed_tasks_info[dep_id] == TaskState.DONE - for dep_id in dependencies - ) - - # Check if any dependency failed - any_dep_failed = any( - completed_tasks_info[dep_id] == TaskState.FAILED - for dep_id in dependencies - ) + # Determine if task should be posted based on mode + should_post_task = False + + if self.mode == WorkforceMode.PIPELINE: + # PIPELINE mode: Dependencies completed (success or failure) + should_post_task = True + logger.debug( + f"Task {task.id} ready in PIPELINE mode. " + f"All dependencies completed." + ) + else: + # AUTO_DECOMPOSE mode: All dependencies must succeed + all_deps_done = all( + completed_tasks_info[dep_id] == TaskState.DONE + for dep_id in dependencies + ) + should_post_task = all_deps_done - if all_deps_done: - # All dependencies completed successfully - post the - # task + if should_post_task: + # Post the task assignee_id = self._assignees[task.id] logger.debug( f"Posting task {task.id} to " @@ -3641,13 +3674,19 @@ async def _post_ready_tasks(self) -> None: ) await self._post_task(task, assignee_id) posted_tasks.append(task) - elif any_dep_failed: - # Check if any failed dependencies can still be retried - failed_deps = [ - dep_id + elif self.mode == WorkforceMode.AUTO_DECOMPOSE: + # AUTO_DECOMPOSE mode: Handle dependency failures + any_dep_failed = any( + completed_tasks_info[dep_id] == TaskState.FAILED for dep_id in dependencies - if completed_tasks_info[dep_id] == TaskState.FAILED - ] + ) + if any_dep_failed: + # Check if any failed dependencies can still be retried + failed_deps = [ + dep_id + for dep_id in dependencies + if completed_tasks_info[dep_id] == TaskState.FAILED + ] # Check if any failed dependency is still retryable failed_tasks_with_retry_potential = [] @@ -3773,6 +3812,24 @@ async def _handle_failed_task(self, task: Task) -> bool: # Check for immediate halt conditions if task.failure_count >= MAX_TASK_RETRIES: + # PIPELINE mode: Allow workflow to continue with failed task + if self.mode == WorkforceMode.PIPELINE: + logger.warning( + f"Task {task.id} failed after {MAX_TASK_RETRIES} " + f"retries in PIPELINE mode. Marking as failed and " + f"allowing workflow to continue. Error: {failure_reason}" + ) + task.state = TaskState.FAILED + self._cleanup_task_tracking(task.id) + self._completed_tasks.append(task) + if task.id in self._assignees: + await self._channel.archive_task(task.id) + + # Check if any pending tasks are now ready + await self._post_ready_tasks() + return False # Don't halt workforce + + # AUTO_DECOMPOSE mode: Halt on max retries logger.error( f"Task {task.id} has exceeded maximum retry attempts " f"({MAX_TASK_RETRIES}). Final failure reason: " @@ -3797,7 +3854,19 @@ async def _handle_failed_task(self, task: Task) -> bool: await self._channel.archive_task(task.id) return True - # Use intelligent failure analysis to decide recovery strategy + # PIPELINE mode: Simple retry without intelligent recovery + if self.mode == WorkforceMode.PIPELINE: + logger.info( + f"Task {task.id} failed in PIPELINE mode. " + f"Will retry (attempt {task.failure_count}/{MAX_TASK_RETRIES})" + ) + # Simply reset to pending for retry + task.state = TaskState.PENDING + self._pending_tasks.append(task) + await self._post_ready_tasks() + return False + + # AUTO_DECOMPOSE mode: Use intelligent failure analysis recovery_decision = self._analyze_task( task, for_failure=True, error_message=detailed_error ) diff --git a/examples/workforce/pipeline_workflow_example.py b/examples/workforce/pipeline_workflow_example.py index 39b5f5ca1c..28dd82595d 100644 --- a/examples/workforce/pipeline_workflow_example.py +++ b/examples/workforce/pipeline_workflow_example.py @@ -14,37 +14,27 @@ # ========= Copyright 2023-2024 @ CAMEL-AI.org. All Rights Reserved. ========= """ -Pipeline Workflow Example - Comprehensive demonstration of CAMEL Workforce Pipeline functionality. +Pipeline Workflow Example - Demonstration of CAMEL Workforce Pipeline functionality. -This example demonstrates practical Pipeline features of CAMEL Workforce: -1. Literature analysis with parallel summarization -2. API documentation processing -3. Data analysis with fork-join patterns -4. Complex multi-stage workflows -5. Real-world use cases +This example demonstrates the fork-join pattern in CAMEL Workforce Pipeline mode: +- Single source task (literature search) +- Fork to 5 parallel summarization tasks +- Join results into a comprehensive synthesis -The examples show how to handle branching tasks where multiple agents -work on different parts of the same data source. +The example shows how to handle parallel task processing where multiple agents +work on different parts of the same data source with automatic synchronization. """ import asyncio -import json import time -from typing import Dict, Any from colorama import Fore, Style, init -from camel.configs import ChatGPTConfig, GeminiConfig +from camel.configs import ChatGPTConfig from camel.agents import ChatAgent from camel.models import ModelFactory -from camel.societies.workforce import ( - PipelineTaskBuilder, - Workforce, - WorkforceMode, - SingleAgentWorker -) +from camel.societies.workforce import Workforce, WorkforceMode from camel.tasks import Task -from camel.toolkits import ThinkingToolkit from camel.types import ModelPlatformType, ModelType from camel.messages import BaseMessage @@ -99,7 +89,12 @@ async def example_1_literature_analysis_pipeline(): ) # Create workforce for literature analysis - workforce = Workforce("Literature Analysis Team", mode=WorkforceMode.PIPELINE, coordinator_agent=coordinator_agent, task_agent=task_agent) + workforce = Workforce( + "Literature Analysis Team", + coordinator_agent=coordinator_agent, + task_agent=task_agent, + mode=WorkforceMode.PIPELINE + ) # Add search agent with search tools (using mock data for demonstration) search_system_message = BaseMessage.make_assistant_message( @@ -140,16 +135,16 @@ async def example_1_literature_analysis_pipeline(): workforce.add_single_agent_worker("Research Synthesizer", synthesis_agent) # Build literature analysis pipeline - workforce.add_pipeline_task("Generate 5 representative recent AI/ML papers with titles, authors, and brief descriptions. Format as [Paper 1] to [Paper 5]") \ - .fork_pipeline([ + workforce.pipeline_add("Generate 5 representative recent AI/ML papers with titles, authors, and brief descriptions. Format as [Paper 1] to [Paper 5]") \ + .pipeline_fork([ "Summarize [Paper 1] core insights, methodology and contributions", "Summarize [Paper 2] core insights, methodology and contributions", "Summarize [Paper 3] core insights, methodology and contributions", "Summarize [Paper 4] core insights, methodology and contributions", "Summarize [Paper 5] core insights, methodology and contributions" ]) \ - .join_pipeline("Analyze AI/ML research trends based on the 5 paper summaries") \ - .finalize_pipeline() + .pipeline_join("Analyze AI/ML research trends based on the 5 paper summaries") \ + .pipeline_build() print(f"{Fore.YELLOW}Literature analysis pipeline built: 1 search β†’ 5 parallel summaries β†’ 1 synthesis") print(f"{Fore.YELLOW}Mode: {workforce.mode}") @@ -169,270 +164,16 @@ async def example_1_literature_analysis_pipeline(): return result -async def example_2_api_documentation_pipeline(): - """Example 2: API documentation processing with parallel analysis.""" - print_section("Example 2: API Documentation Processing Pipeline") - - # Create coordinator agent - coordinator_agent = ChatAgent( - BaseMessage.make_assistant_message( - role_name="API Documentation Coordinator", - content="You are a coordinator responsible for managing API documentation analysis and ensuring comprehensive coverage." - ), - model=model - ) - - # Create task agent using the same model - task_agent = ChatAgent( - BaseMessage.make_assistant_message( - role_name="Task Planning Agent", - content="You are a task planning agent responsible for analyzing and coordinating complex tasks." - ), - model=model - ) - - # Create workforce for API documentation analysis - workforce = Workforce("API Documentation Team", mode=WorkforceMode.PIPELINE, coordinator_agent=coordinator_agent, task_agent=task_agent) - - # Add API discovery agent (using mock data for demonstration) - api_researcher_system_message = BaseMessage.make_assistant_message( - role_name="API Researcher", - content="You are an API researcher. Generate 4 core RESTful API endpoint types with descriptions. Format them as [API 1], [API 2], etc. Use common REST patterns like GET, POST, PUT, DELETE." - ) - api_researcher = ChatAgent( - system_message=api_researcher_system_message, - model=model, - tools=[] - ) - - # Add multiple API testing/analysis agents - for i in range(4): - api_analyst_system_message = BaseMessage.make_assistant_message( - role_name=f"API Analyst {i+1}", - content="You are an API analyst. Test API endpoints, analyze functionality, and document usage patterns and best practices." - ) - api_analyst = ChatAgent( - system_message=api_analyst_system_message, - model=model, - tools=[] - ) - workforce.add_single_agent_worker(f"API Analyst {i+1}", api_analyst) - - # Add documentation writer - doc_writer_system_message = BaseMessage.make_assistant_message( - role_name="Documentation Writer", - content="You are a technical documentation writer. Create comprehensive API documentation from analysis results." - ) - doc_writer = ChatAgent( - system_message=doc_writer_system_message, - model=model, - tools=[] - ) - - workforce.add_single_agent_worker("API Researcher", api_researcher) - workforce.add_single_agent_worker("Documentation Writer", doc_writer) - - # Build API documentation pipeline - workforce.add_pipeline_task("Generate 4 core RESTful API endpoint types (GET, POST, PUT, DELETE) with descriptions and format as [API 1] to [API 4]") \ - .fork_pipeline([ - "Analyze [API 1] functionality, parameters, responses and use cases", - "Analyze [API 2] functionality, parameters, responses and use cases", - "Analyze [API 3] functionality, parameters, responses and use cases", - "Analyze [API 4] functionality, parameters, responses and use cases" - ]) \ - .join_pipeline("Generate complete API usage guide based on 4 endpoint analyses") \ - .finalize_pipeline() - - print(f"{Fore.YELLOW}API documentation pipeline built: 1 discovery β†’ 4 parallel analyses β†’ 1 synthesis") - print(f"{Fore.YELLOW}Mode: {workforce.mode}") - - # Execute the pipeline - main_task = Task( - content="RESTful API Documentation and Best Practices Guide", - id="api_documentation" - ) - - start_time = time.time() - result = await workforce.process_task_async(main_task) - end_time = time.time() - - print_result(result.id, result.result or "API documentation completed successfully") - print(f"{Fore.BLUE}[TIME] Execution time: {end_time - start_time:.2f} seconds") - - return result - -async def example_3_code_review_pipeline(): - """Example 3: Code review with multiple file analysis.""" - print_section("Example 3: Code Review Pipeline") - - # Create coordinator agent - coordinator_agent = ChatAgent( - BaseMessage.make_assistant_message( - role_name="Code Review Coordinator", - content="You are a coordinator responsible for managing code review processes and ensuring thorough analysis." - ), - model=model - ) - - # Create task agent using the same model - task_agent = ChatAgent( - BaseMessage.make_assistant_message( - role_name="Task Planning Agent", - content="You are a task planning agent responsible for analyzing and coordinating complex tasks." - ), - model=model - ) - - # Create workforce for code review - workforce = Workforce("Code Review Team", mode=WorkforceMode.PIPELINE, coordinator_agent=coordinator_agent, task_agent=task_agent) - - # Sample code files for review - code_files = { - "file1.py": """ -def calculate_total(items): - total = 0 - for item in items: - total += item.price * item.quantity - return total - -class ShoppingCart: - def __init__(self): - self.items = [] - - def add_item(self, item): - self.items.append(item) - - def get_total(self): - return calculate_total(self.items) -""", - "file2.py": """ -import requests -import json - -def fetch_user_data(user_id): - url = f"https://api.example.com/users/{user_id}" - response = requests.get(url) - if response.status_code == 200: - return json.loads(response.text) - return None - -def process_users(user_ids): - results = [] - for uid in user_ids: - data = fetch_user_data(uid) - if data: - results.append(data) - return results -""", - "file3.py": """ -class DatabaseConnection: - def __init__(self, host, port, database): - self.host = host - self.port = port - self.database = database - self.connection = None - - def connect(self): - # Simulate database connection - self.connection = f"connected to {self.host}:{self.port}/{self.database}" - return True - - def execute_query(self, query): - if not self.connection: - raise Exception("Not connected to database") - return f"Executed: {query}" -""" - } - - # Add code scanner agent - scanner_system_message = BaseMessage.make_assistant_message( - role_name="Code Scanner", - content="You are a code scanner. Analyze code files and format findings as [File 1], [File 2], [File 3] with code snippets." - ) - scanner_agent = ChatAgent( - system_message=scanner_system_message, - model=model, - tools=[] - ) - - # Add multiple code reviewers for parallel processing - for i in range(3): - reviewer_system_message = BaseMessage.make_assistant_message( - role_name=f"Code Reviewer {i+1}", - content="You are a code reviewer. Review code for bugs, performance issues, and best practices." - ) - reviewer_agent = ChatAgent( - system_message=reviewer_system_message, - model=model, - tools=[] - ) - workforce.add_single_agent_worker(f"Code Reviewer {i+1}", reviewer_agent) - - # Add report generator - report_system_message = BaseMessage.make_assistant_message( - role_name="Review Summarizer", - content="You are a review summarizer. Compile code review findings into comprehensive reports." - ) - report_agent = ChatAgent( - system_message=report_system_message, - model=model, - tools=[] - ) - - workforce.add_single_agent_worker("Code Scanner", scanner_agent) - workforce.add_single_agent_worker("Review Summarizer", report_agent) - - # Build code review pipeline - workforce.add_pipeline_task(f"Scan these code files and format as [File 1], [File 2], [File 3]:\n\n{json.dumps(code_files, indent=2)}") \ - .fork_pipeline([ - "Review [File 1] for code quality, bugs, and improvements", - "Review [File 2] for code quality, bugs, and improvements", - "Review [File 3] for code quality, bugs, and improvements" - ]) \ - .join_pipeline("Generate comprehensive code review report with all findings") \ - .finalize_pipeline() - - print(f"{Fore.YELLOW}Code review pipeline built: Scan β†’ 3 parallel reviews β†’ Summary report") - print(f"{Fore.YELLOW}Mode: {workforce.mode}") - - # Execute the pipeline - main_task = Task( - content="Code Review Analysis Pipeline", - id="code_review" - ) - - start_time = time.time() - result = await workforce.process_task_async(main_task) - end_time = time.time() - - print_result(result.id, result.result or "Code review pipeline completed successfully") - print(f"{Fore.BLUE}[TIME] Execution time: {end_time - start_time:.2f} seconds") - - return result - - def print_summary(): """Print example summary.""" - print_section("Pipeline Examples Summary") + print_section("Pipeline Example Summary") - examples = [ - "1. Literature Analysis - Fork/join pattern with 5 parallel agents", - "2. API Documentation - 4 parallel analysts with structured parsing", - "3. Data Analysis - Multi-stage workflow with dataset branching" - ] - - print(f"{Fore.CYAN}Examples demonstrated:") - for example in examples: - print(f"{Fore.WHITE} {example}") - - print(f"\n{Fore.GREEN}Pipeline patterns tested:") + print(f"{Fore.GREEN}Pipeline pattern demonstrated:") patterns = [ - "SEARCH BRANCH: Single search distributed to multiple specialists", - "FORK-JOIN: Parallel processing with automatic synchronization", - "STRUCTURED OUTPUT: Using [markers] for smart task distribution", - "SMART ROUTING: Agents identify their target data sections", - "PARALLEL SCALING: Easy adjustment of parallel worker count", - "DEPENDENCY CHAINS: Multi-stage workflows with proper sequencing" + "βœ“ FORK-JOIN: Single task β†’ 5 parallel processing β†’ 1 synthesis", + "βœ“ PARALLEL SCALING: Easy adjustment of parallel worker count", + "βœ“ DEPENDENCY CHAINS: Automatic synchronization and dependency resolution", + "βœ“ STRUCTURED OUTPUT: Using [markers] for smart task distribution" ] for pattern in patterns: @@ -440,35 +181,30 @@ def print_summary(): print(f"\n{Fore.CYAN}Technical features:") tech_features = [ - "ModelFactory.create() for agent initialization", - "SearchToolkit integration", - "Pipeline mode with fork_pipeline() and join_pipeline()", - "Automatic dependency resolution", - "Structured output parsing", - "Multi-agent coordination" + "β€’ Pipeline mode with pipeline_fork() and pipeline_join()", + "β€’ Automatic worker assignment and task routing", + "β€’ Multi-agent parallel coordination", + "β€’ Structured task dependencies" ] for feature in tech_features: print(f"{Fore.WHITE} {feature}") async def main(): - """Main function to run pipeline examples.""" - print_section("CAMEL Workforce Pipeline Examples") - print(f"{Fore.YELLOW}Testing Pipeline mode with branching and parallel processing.") + """Main function to run pipeline example.""" + print_section("CAMEL Workforce Pipeline Example") + print(f"{Fore.YELLOW}Testing Pipeline mode with fork-join pattern and parallel processing.") - results = {} + result = None try: - print(f"\n{Fore.BLUE}Running pipeline examples...") + print(f"\n{Fore.BLUE}Running literature analysis pipeline example...") - results['literature'] = await example_1_literature_analysis_pipeline() - results['api_docs'] = await example_2_api_documentation_pipeline() - results['code_review'] = await example_3_code_review_pipeline() + result = await example_1_literature_analysis_pipeline() print_summary() - print(f"\n{Fore.GREEN}Pipeline examples completed successfully!") - print(f"{Fore.CYAN}Total examples: {len(results)}") + print(f"\n{Fore.GREEN}Pipeline example completed successfully!") except Exception as e: print(f"\n{Fore.RED}Error: {e}") @@ -476,682 +212,207 @@ async def main(): traceback.print_exc() raise - return results + return result if __name__ == "__main__": # Run the examples asyncio.run(main()) """ - ============================================================ -CAMEL Workforce Pipeline Examples +CAMEL Workforce Pipeline Example ============================================================ -Testing Pipeline mode with branching and parallel processing. +Testing Pipeline mode with fork-join pattern and parallel processing. -Running pipeline examples... +Running literature analysis pipeline example... ============================================================ Example 1: Literature Analysis with Parallel Processing ============================================================ -Literature analysis pipeline built: 1 search β†’ 5 parallel summaries β†’ 1 synthesis +Literature analysis pipeline built: 1 search β†’ 5 parallel summaries β†’ 1 synthesis Mode: WorkforceMode.PIPELINE -Worker node 515f890a-c013-4876-9d29-1dc3e65c9f69 (Literature Researcher) get task pipeline_task_0: Generate 5 representative recent AI/ML papers with titles, authors, and brief descriptions. Format as [Paper 1] to [Paper 5] +Worker node f6cd7177-2d02-4398-bd78-3116b0807db4 (Literature Researcher) get task pipeline_task_0: Generate 5 representative recent AI/ML papers with titles, authors, and brief descriptions. Format as [Paper 1] to [Paper 5] ====== -Response from Worker node 515f890a-c013-4876-9d29-1dc3e65c9f69 (Literature Researcher): +Response from Worker node f6cd7177-2d02-4398-bd78-3116b0807db4 (Literature Researcher): [Paper 1] "Gemini: A Family of Highly Capable Multimodal Models" -Authors: F. Zhai, Z. Ma, Y. Du, and others (Google DeepMind, 2023) -Description: This paper introduces Gemini, a new family of large multimodal models capable of processing text, images, and audio. It demonstrates state-of-the-art performance on a range of benchmarks, including language understanding and video reasoning tasks. +Authors: Google DeepMind (Peter Barham, Jeff Dean, et al.) +Description: Introduces the Gemini family of large language models designed for multimodal reasoning across text, images, audio, and video. Highlights performance surpassing prior state-of-the-art on multiple multimodal and language understanding benchmarks. [Paper 2] -"Llama 2: Open Foundation and Fine-Tuned Chat Models" -Authors: Hugo Touvron, Louis Martin, Kevin Stone, and others (Meta AI, 2023) -Description: Llama 2 presents a suite of open-access large language models trained on textual data. These models show competitive results in various natural language processing tasks and offer strong baselines for further research in ethical and transparent AI development. +"LLM-Augmented Learning: Enhancing Data Quality via Iterative Refinement" +Authors: Kaixuan Zhang, Luyu Wang, Wenjuan Han, et al. (Microsoft Research) +Description: Explores the use of large language models (LLMs) for data cleaning and augmentation via iterative refinement, showing improved downstream model performance on NLP tasks with noisy data. [Paper 3] -"EfficientViT: Memory Efficient Vision Transformers with Cascaded Group Attention" -Authors: Xiangtai Li, Shangchen Han, Shiji Song, and others (Tsinghua University, 2024) -Description: EfficientViT addresses the high memory and compute costs of existing vision transformers. It introduces a cascaded group attention mechanism that achieves similar or better accuracy than previous approaches while drastically reducing resource requirements. +"MEGAVERSE: Benchmarking Foundation Models for Autonomous Agents" +Authors: Jun Shern Chan, Kevin F. Le, Yilun Du, et al. (MIT, Stanford, Google DeepMind) +Description: Presents MEGAVERSE, a new benchmark suite for evaluating how well foundation models can power general autonomous agents, across virtual environments and tasks. [Paper 4] -"Self-Supervised Learning with SwAV: Unsupervised Learning of Visual Features by Contrasting Cluster Assignments" -Authors: Mathilde Caron, Priya Goyal, Piotr Bojanowski, and others (Facebook AI Research, 2023) -Description: SwAV proposes a clustering-based unsupervised learning algorithm for vision models. By contrasting multiple cluster assignments, it enables models to learn robust visual features without requiring labeled data, accelerating progress in self-supervised learning. +"Qwen-VL: A Versatile Vision-Language Model based on Large Language Models" +Authors: Alibaba DAMO Academy (Jianfei Chen, Yuheng Xie, et al.) +Description: Introduces Qwen-VL, a powerful open-source multimodal model capable of advanced image and document understanding, outperforming previous vision-language models of similar scale. [Paper 5] -"RetNet: Delving into Efficient and Scalable Transformers for Long Sequences" -Authors: Xunyu Lin, Xiang Kong, Renzhi Mao, and others (Microsoft Research, 2024) -Description: RetNet introduces mechanisms for transformers to more efficiently process very long sequences, reducing computation and memory bottlenecks. It achieves competitive results on NLP tasks with sequences thousands of tokens long. +"Eagle: Layer-wise Fine-tuning of Foundation Models" +Authors: Xinyi Xu, Jindong Wang, Wenxin Hou, et al. (Microsoft Research) +Description: Proposes a new fine-tuning approach for large foundation models, showing that targeted layer-wise updates can achieve high performance with lower computational cost, and improving knowledge transfer across tasks. ====== -🎯 Task pipeline_task_0 completed successfully. -Worker node 16a98984-7d43-4fec-8abc-7127353d7ef7 (Summary Specialist 1) get task parallel_0_1: Summarize [Paper 1] core insights, methodology and contributions -Worker node 12050652-606c-4706-843b-460318d1ff1e (Summary Specialist 2) get task parallel_1_1: Summarize [Paper 2] core insights, methodology and contributions -Worker node 4ccbbd32-0d8f-4d8a-9892-69f9082fc2eb (Summary Specialist 3) get task parallel_2_1: Summarize [Paper 3] core insights, methodology and contributions -Worker node 556a1d0d-e1c0-4d7e-ad2f-4fdc8bc8993f (Summary Specialist 4) get task parallel_3_1: Summarize [Paper 4] core insights, methodology and contributions -Worker node 5cb31fc3-611c-423b-b45a-e67e87b53c67 (Summary Specialist 5) get task parallel_4_1: Summarize [Paper 5] core insights, methodology and contributions +Task pipeline_task_0 completed successfully (quality score: 100). +Worker node 5d4ea69d-eb97-4091-8ebb-c073ca555048 (Summary Specialist 2) get task parallel_1_1: Summarize [Paper 2] core insights, methodology and contributions +Worker node cbf6bda1-c9c2-42f2-a9e3-115010772117 (Summary Specialist 4) get task parallel_3_1: Summarize [Paper 4] core insights, methodology and contributions +Worker node 5299ade7-751b-4037-9b64-cfc134a3347c (Summary Specialist 1) get task parallel_0_1: Summarize [Paper 1] core insights, methodology and contributions +Worker node e28830f2-3ee7-43ca-b2b6-30b2d287f329 (Summary Specialist 5) get task parallel_4_1: Summarize [Paper 5] core insights, methodology and contributions +Worker node d71aeebb-93bb-4c87-9990-826b5ccf97e3 (Summary Specialist 3) get task parallel_2_1: Summarize [Paper 3] core insights, methodology and contributions ====== -Response from Worker node 16a98984-7d43-4fec-8abc-7127353d7ef7 (Summary Specialist 1): - -Paper 1, "Gemini: A Family of Highly Capable Multimodal Models" by F. Zhai, Z. Ma, Y. Du, and others (Google DeepMind, 2023), presents the Gemini model family, a new suite of large multimodal models designed to process and understand inputs across text, images, and audio. +Response from Worker node cbf6bda1-c9c2-42f2-a9e3-115010772117 (Summary Specialist 4): -Core Insights: Gemini demonstrates state-of-the-art performance on a variety of benchmarks spanning language understanding, video reasoning, and multimodal comprehension tasks. The central insight is that a unified architecture can achieve robust, high-performing multimodal understanding by leveraging large-scale pretraining and architectural innovations. +Core Insights: Qwen-VL introduces a highly capable open-source vision-language model that demonstrates advanced performance in both image and document understanding tasks. It outperforms previous vision-language models of similar scale, showcasing strong generalization and multimodal reasoning abilities. -Methodology: The Gemini models are trained on vast, diverse datasets that include both unimodal and multimodal content. The architecture integrates advanced components for processing different data types, allowing seamless fusion of textual, visual, and audio information. Comprehensive quantitative evaluations are performed on established benchmarks, comparing Gemini to prior state-of-the-art models. +Methodology: The authors design Qwen-VL by building on large language model (LLM) architectures and augmenting them to handle multimodal inputs (images and text). The model is trained on large-scale, carefully curated vision-language datasets, utilizing both supervised and self-supervised objectives. Key methodological advances include the alignment of visual and textual representations, multimodal attention mechanisms, and strategies for robust cross-modal understanding such as unified embedding spaces and task-specific fine-tuning. -Contributions: Gemini establishes new benchmarks in multimodal AI, outperforming prior models on a range of tasks. It advances the field by offering a generalizable framework for multimodal learning, supporting complex tasks such as language-image-text reasoning and video-based question answering. Additionally, the paper provides detailed empirical results and analysis, setting a new standard for future multimodal model development. +Contributions: (1) The introduction of a versatile and powerful open-source multimodal model (Qwen-VL) tailored for vision-language understanding; (2) Comprehensive evaluation showing the model outperforms previous state-of-the-art vision-language models of comparable scale on a variety of image and document understanding benchmarks; (3) Release of model weights and code, contributing valuable resources for the research community and enabling further advancements in multimodal AI. ====== -🎯 Task parallel_0_1 completed successfully. +Task parallel_3_1 completed successfully (quality score: 98). ====== -Response from Worker node 5cb31fc3-611c-423b-b45a-e67e87b53c67 (Summary Specialist 5): - -Paper 5, "RetNet: Delving into Efficient and Scalable Transformers for Long Sequences" (Xunyu Lin, Xiang Kong, Renzhi Mao, et al., Microsoft Research, 2024), addresses the computational and memory inefficiencies of standard Transformer architectures when processing very long sequences. - -Core Insights: The paper observes that existing Transformers struggle to efficiently handle sequences on the scale of thousands of tokens due to their quadratic memory and computation requirements. RetNet is proposed as a solution, introducing novel mechanisms designed to make transformers more scalable and resource-efficient for such cases. +Response from Worker node e28830f2-3ee7-43ca-b2b6-30b2d287f329 (Summary Specialist 5): -Methodology: RetNet rethinks the architecture of self-attention to specifically focus on improving efficiency for long-context natural language processing tasks. Although the detailed mechanisms are not specified in the brief, the paper claims to have introduced approaches to reduce both computation and memory bottlenecks typically seen in large-sequence processing. These enhancements are empirically validated on benchmarks involving lengthy input sequences. +Core Insights: [Paper 5] "Eagle: Layer-wise Fine-tuning of Foundation Models" introduces a novel approach to fine-tuning large foundation models by updating selected layers rather than the whole model. The core insight is that targeted, layer-wise updates can retain or surpass the performance of full-model fine-tuning while significantly reducing computational cost and memory requirements. This strategy also improves knowledge transfer across multiple tasks, making adaptation more efficient. -Contributions: (1) A new transformer-based architecture (RetNet) tailored for long-sequence efficiency, (2) Demonstration of competitive results on NLP tasks involving sequences with thousands of tokens, and (3) Advancements in reducing memory and computational overhead, enabling wider application of transformers on tasks that require handling long inputs. +Methodology: The authors design an algorithm that identifies and fine-tunes specific layers of a foundation model, based on task requirements. Through empirical studies, they compare their method against full-model and other parameter-efficient fine-tuning approaches across various NLP tasks. Metrics such as task accuracy, training time, and resource consumption are thoroughly assessed. -Overall, RetNet represents a key step forward in scalable deep learning for sequence modeling, making transformer models viable for previously prohibitive long-sequence applications. +Contributions: 1) Proposes a systematic, layer-wise fine-tuning method for large models. 2) Demonstrates that this approach substantially reduces computational overhead. 3) Shows improved or matched performance in multi-task scenarios compared to prior methods. 4) Provides comprehensive benchmarks and analysis, establishing a new practical avenue for efficient and effective model adaptation. ====== -🎯 Task parallel_4_1 completed successfully. ====== -Response from Worker node 12050652-606c-4706-843b-460318d1ff1e (Summary Specialist 2): +Response from Worker node 5d4ea69d-eb97-4091-8ebb-c073ca555048 (Summary Specialist 2): -Paper 2, "Llama 2: Open Foundation and Fine-Tuned Chat Models" by Hugo Touvron et al. (Meta AI, 2023), presents core advances in open-access large language models (LLMs). +Paper 2, titled "LLM-Augmented Learning: Enhancing Data Quality via Iterative Refinement" by Kaixuan Zhang, Luyu Wang, Wenjuan Han, et al. (Microsoft Research), explores the use of large language models (LLMs) to improve data quality for NLP tasks, with a particular emphasis on handling noisy datasets. -Core Insights: Llama 2 models, trained on extensive textual datasets, demonstrate competitive results on multiple natural language processing (NLP) benchmarks. The work emphasizes ethical considerations, transparency, and provides models suitable for research and downstream fine-tuning. +Core Insights: The key insight of the paper is that LLMs can be leveraged not just for downstream inference tasks, but as intelligent agents for data cleaning and augmentation. By iteratively refining the training data using LLM-generated suggestions, the quality of the dataset improves, which in turn leads to better model performance on downstream tasks, especially in the presence of noise or imperfect data. -Methodology: The paper details the architectural design and training of Llama 2, including the use of high-quality, filtered text corpora. Various model sizes are trained with standard transformer-based architectures. Fine-tuning techniques tailored for chat and alignment with human feedback are described, ensuring the models perform well in dialog and instruction-following tasks while mitigating biases and unsafe outputs. +Methodology: The authors introduce a framework where an LLM is employed to identify errors, inconsistencies, or ambiguities in training datasets. The LLM proposes corrections or improvements in an iterative loop, with human oversight or additional automated validation ensuring changes maintain or increase data integrity. This process is repeated, resulting in progressively cleaner datasets. The refined data is then used to train standard NLP models, and performance is evaluated on benchmark tasks to assess gains from the approach. -Contributions: (1) Introduction of an openly available suite of LLMs with both base and fine-tuned "chat" versions, (2) rigorous evaluation and benchmarking across a wide set of NLP tasks, showing state-of-the-art or near SOTA performance for open models, (3) a strong baseline for future research in responsible, transparent, and accessible AI development, and (4) detailed discussions on responsible model use and deployment. +Contributions: The main contributions of this work are: (1) Proposing a novel use-case for LLMs as data quality improvers, not just as end-task predictors; (2) Demonstrating through experiments that iterative LLM-driven data refinement can lead to measurable improvements in downstream NLP model accuracy and robustness, particularly with initially noisy data; (3) Providing a framework adaptable to various NLP scenarios where data quality is a bottleneck. The paper offers empirical results that support these claims and discusses implications for scalable automated data curation. ====== -🎯 Task parallel_1_1 completed successfully. +Task parallel_4_1 completed successfully (quality score: 97). +Task parallel_1_1 completed successfully (quality score: 97). ====== -Response from Worker node 556a1d0d-e1c0-4d7e-ad2f-4fdc8bc8993f (Summary Specialist 4): +Response from Worker node d71aeebb-93bb-4c87-9990-826b5ccf97e3 (Summary Specialist 3): -Core Insights: 'Self-Supervised Learning with SwAV: Unsupervised Learning of Visual Features by Contrasting Cluster Assignments' introduces SwAV, a novel self-supervised learning method for training vision models without labeled data. The core insight is to exploit clustering for representation learning by comparing different image augmentations not directly, but via their assignments to clusters, enabling efficient learning of useful visual features. +Core Insights: 'MEGAVERSE: Benchmarking Foundation Models for Autonomous Agents' introduces MEGAVERSEβ€”a comprehensive benchmark suite aimed at systematically evaluating how foundation models (such as large language or multimodal models) serve as the backbone for general autonomous agents. The core insight is the need for a standardized, rigorous evaluation environment to measure the capabilities and limitations of foundation models when deployed in agentic, multi-domain scenarios. -Methodology: SwAV stands for 'Swapping Assignments between Views'. The approach relies on applying multiple augmentations to each image and assigning each augmented view to a cluster (prototype) online. The model is trained to predict the cluster assignment of one view from the representation of another, effectively creating a self-supervised signal without the need for explicit pairing or negative samples (as in contrastive learning). SwAV employs online clustering and a 'multi-crop' augmentation strategy, increasing the diversity and number of image views per training step, which boosts performance and efficiency. +Methodology: The paper constructs a varied suite of simulated virtual environments and tasks designed to reflect real-world agent challenges, encompassing navigation, manipulation, multi-step reasoning, and adaptation. It defines standardized tasks with clear, reproducible metrics for evaluation. Foundation models are integrated as the central reasoning or decision-making module for agents, whose behaviors are then assessed across diverse environments and task types. The benchmark measures factors such as generalization, instruction following, task completion, transfer learning, and emergent behaviors. Extensive experiments are conducted with multiple state-of-the-art foundation models (including LLMs and vision-language models) to ensure comprehensive coverage. -Contributions: The paper's key contributions are: (1) a novel clustering-based approach to self-supervised visual feature learning that does not require pairwise comparisons across batch samples, (2) the introduction of the 'swapped prediction' mechanism as a training objective, and (3) demonstration of strong performance on standard image classification benchmarks (e.g., ImageNet), matching or surpassing existing self-supervised learning methods while improving training efficiency. SwAV's approach has influenced subsequent research in self-supervised learning and clustering-based representation learning. +Contributions: +1. Proposes MEGAVERSE as the first broad benchmark tailored to test the agentic capabilities of foundation models. +2. Provides an open, extensible set of tasks and virtual worlds enabling systematic, reproducible evaluation and comparison. +3. Reveals critical strengths and limitations of existing foundation models when deployed as autonomous agents, informing future research directions on improving real-world agent generality. +4. Delivers thorough experimental results, establishing initial baselines and highlighting emergent patterns and performance gaps. ====== -🎯 Task parallel_3_1 completed successfully. ====== -Response from Worker node 4ccbbd32-0d8f-4d8a-9892-69f9082fc2eb (Summary Specialist 3): +Response from Worker node 5299ade7-751b-4037-9b64-cfc134a3347c (Summary Specialist 1): -Paper 3, titled "EfficientViT: Memory Efficient Vision Transformers with Cascaded Group Attention" (Xiangtai Li, Shangchen Han, Shiji Song, et al., Tsinghua University, 2024), addresses the high memory and computational costs associated with standard vision transformer (ViT) architectures. +Paper: "Gemini: A Family of Highly Capable Multimodal Models" -Core Insights: The paper identifies that existing vision transformers are resource-intensive, which limits their deployment in environments with constrained memory and compute capabilities. The authors propose a novel solution through the cascaded group attention mechanism, which selectively focuses attention computation within groups of tokens, thereby reducing overhead without sacrificing model performance. +Core Insights: +- The Gemini models represent a new generation of large-scale multimodal AI systems developed by Google DeepMind, designed to process and reason across multiple input modalities including text, images, audio, and video. These models achieve strong performance, surpassing previous state-of-the-art results on various multimodal and language understanding benchmarks. -Methodology: EfficientViT introduces cascaded group attention, wherein the model organizes input image tokens into groups and applies attention mechanisms hierarchically and sparsely within and across these groups. This design reduces the quadratic memory and computation scaling of traditional ViTs to a more manageable level. The paper details formal architectural changes and provides empirical benchmarks to compare its efficiency and accuracy against baseline transformers. +Methodology: +- Gemini comprises a family of models trained using large-scale datasets spanning diverse modalities, enabling integrated vision, language, and audio understanding. Training leverages cross-modal objectives and alignment techniques, allowing the models to interpret, synthesize, and generate outputs across different types of inputs. +- Rigorous benchmarking is conducted against prior state-of-the-art models, using standardized multimodal datasets and tasks to demonstrate capabilities in reasoning, comprehension, and generation. -Contributions: The main contributions of the paper are (1) the novel cascaded group attention mechanism enabling memory- and compute-efficient vision transformers; (2) empirical demonstration that EfficientViT achieves comparable or superior accuracy on key vision tasks relative to standard ViTs while using significantly less memory and computation; and (3) the potential to broaden the practical deployment of transformer-based models in resource-limited environments. +Key Contributions: +1. Introduction of a scalable family of multimodal models (Gemini) capable of unified reasoning across text, image, audio, and video domains. +2. Empirical evidence showing Gemini’s superior performance on a wide range of multimodal and language benchmarks, establishing new performance records in several categories. +3. Advancements in multimodal training and alignment techniques, demonstrating how large language model principles can be generalized beyond text to rich, multi-sensory data. +4. An extensive evaluation framework and analysis that highlights both strengths and limitations, setting a new standard for multimodal model assessment. ====== -🎯 Task parallel_2_1 completed successfully. -Worker node de3d7797-579a-4002-b64b-8ef6d5abf5ed (Research Synthesizer) get task pipeline_task_6: Analyze AI/ML research trends based on the 5 paper summaries +Task parallel_2_1 completed successfully (quality score: 97). +Task parallel_0_1 completed successfully (quality score: 96). +Worker node c7d22a97-4127-4248-a818-a625c0f50862 (Research Synthesizer) get task pipeline_task_6: Analyze AI/ML research trends based on the 5 paper summaries ====== -Response from Worker node de3d7797-579a-4002-b64b-8ef6d5abf5ed (Research Synthesizer): +Response from Worker node c7d22a97-4127-4248-a818-a625c0f50862 (Research Synthesizer): -Analysis of the five AI/ML paper summaries reveals several prominent research trends shaping the current landscape: +Analysis of recent AI/ML research trends based on the five paper summaries reveals several converging themes and emerging directions: -1. Multimodal Intelligence and Unified Architectures: Papers like Gemini (Paper 1) highlight the drive to create highly capable models that process and integrate various modalities (text, images, audio) within a unified framework. This trend enables models to solve complex tasks requiring cross-modal understanding, advancing generalist AI systems. +1. Multimodal Foundation Models: Two papers (Gemini, Qwen-VL) highlight rapid advancements in large-scale multimodal models capable of processing and reasoning across text, images, audio, and video. Researchers are pursuing unified architectures and alignment techniques to extend the capabilities of foundation models beyond single modalities, resulting in superior performance on a variety of benchmarks and tasks. -2. Open and Responsible Large Language Models: Llama 2 (Paper 2) demonstrates a commitment to openness, transparency, and responsible deployment in LLMs. The emphasis on public availability, ethical considerations, and alignment with human preferences underlines the shift toward accessible AI research and safer model release practices. +2. Systematic Benchmarking and Evaluation: The introduction of benchmark suites such as MEGAVERSE signals an increased emphasis on objective, reproducible assessment of AI models, particularly in evaluating general agentic capabilities across diverse and complex tasks. There is a clear trend towards establishing broad, flexible, and extensible benchmarks for both understanding and comparing the performance of foundation models in autonomous settings. -3. Efficient and Scalable Model Design: Both EfficientViT (Paper 3) and RetNet (Paper 5) focus on architectural innovation to address the growing computational and memory demands of state-of-the-art models. EfficientViT introduces cascaded group attention for resource-constrained vision transformers, while RetNet targets scalable transformers for processing long sequences in NLP, collectively marking a broader trend toward improving model efficiency and hardware accessibility. +3. Efficient Model Adaptation and Fine-tuning: Resource-efficient techniques are gaining traction, as evidenced by the Eagle paper's focus on layer-wise fine-tuning. The goal is to maximize adaptability and knowledge transfer of large models while minimizing computational and memory costs, which has important implications for scaling and deploying AI systems in practice. -4. Advances in Self-Supervised and Unsupervised Learning: SwAV (Paper 4) showcases the impact of self-supervised learning and clustering for visual representations. Moving beyond labeled data and traditional contrastive methods, SwAV's cluster-based self-supervision enables sample efficiency and strong benchmark performance, reflecting a larger pivot to less supervised paradigm in representation learning. +4. Data Quality and Model Robustness: The LLM-Augmented Learning paper introduces an innovative use of LLMs for iterative data refinement and cleaning, highlighting a growing recognition that high-quality data remains critical, especially as models become larger. Using advanced models to improve the data they learn from creates a virtuous cycle that enhances downstream robustness and reliability. -5. Benchmarking and Empirical Evaluation: All papers underscore rigorous empirical evaluation across diverse benchmarks as essential for validating new methods and architectures. This trend solidifies the importance of reproducible research and standardized comparison in propelling the field forward. - -Overall, the AI/ML research landscape is marked by a convergence of open, multimodal, and efficient model architectures; responsible AI deployment; and a continual shift towards less supervised, self-sufficient learning paradigms. Architectural and algorithmic innovations are tightly coupled with accessibility, scalability, and ethical considerations, driving the next generation of AI systems. +Research Trends: +- Unification and scalability of models across modalities is a major thrust, with both commercial (Gemini) and open-source (Qwen-VL) efforts driving the field forward. +- Comprehensive, agent-centric benchmarking (MEGAVERSE) reflects the community's recognition of the need for systematic, real-world relevant evaluation tools. +- Methodological innovations targeting the efficiency of adaptation (Eagle) and data curation (LLM-Augmented Learning) strive to make large models more practical and accessible. +- There is a move toward openness, with the release of models and code (Qwen-VL) expected to accelerate research progress. +In summary, the current landscape is characterized by rapid progress in multimodal modeling, a maturing focus on standardization and reproducibility, and practical innovations addressing efficiency and data quality. These trends point toward more capable, robust, and accessible AI systems in the near future. ====== -🎯 Task pipeline_task_6 completed successfully. +Task pipeline_task_6 completed successfully (quality score: 97). Task literature_analysis completed: Result: --- Task pipeline_task_0 Result --- [Paper 1] "Gemini: A Family of Highly Capable Multimodal Models" -Authors: F. Zhai, Z. Ma, Y. Du, and others (Google DeepMind, 2023) -Description: This paper introduc... +Authors: Google DeepMind (Peter Barham, Jeff Dean, et al.) +Description: Introduces the Gemini famil... -[TIME] Execution time: 39.61 seconds +[TIME] Execution time: 56.37 seconds ============================================================ -Example 2: API Documentation Processing Pipeline +Pipeline Example Summary ============================================================ -API documentation pipeline built: 1 discovery β†’ 4 parallel analyses β†’ 1 synthesis -Mode: WorkforceMode.PIPELINE -Worker node c79877ae-537b-440b-adb1-e9ccce756540 (API Researcher) get task pipeline_task_0: Generate 4 core RESTful API endpoint types (GET, POST, PUT, DELETE) with descriptions and format as [API 1] to [API 4] -====== -Response from Worker node c79877ae-537b-440b-adb1-e9ccce756540 (API Researcher): +Pipeline pattern demonstrated: + βœ“ FORK-JOIN: Single task β†’ 5 parallel processing β†’ 1 synthesis + βœ“ PARALLEL SCALING: Easy adjustment of parallel worker count + βœ“ DEPENDENCY CHAINS: Automatic synchronization and dependency resolution + βœ“ STRUCTURED OUTPUT: Using [markers] for smart task distribution -[API 1] GET /resources - Retrieves a list of resources or a single resource by ID. Typically used to read or fetch data from the server. - -[API 2] POST /resources - Creates a new resource. Sends data to the server and returns the newly created resource or its unique identifier. - -[API 3] PUT /resources/{id} - Updates an existing resource identified by its ID. Used to fully replace the contents of the resource with the provided data. - -[API 4] DELETE /resources/{id} - Deletes the specified resource by its ID. Removes the resource from the server. -====== -🎯 Task pipeline_task_0 completed successfully. -Worker node a1506367-5bb1-47ba-8532-4e0603830e4f (API Analyst 1) get task parallel_0_1: Analyze [API 1] functionality, parameters, responses and use cases -Worker node cef71fcc-1304-461b-8267-9217cb3e9cac (API Analyst 2) get task parallel_1_1: Analyze [API 2] functionality, parameters, responses and use cases -Worker node a786426e-ff39-458e-95b3-14453f37028c (API Analyst 3) get task parallel_2_1: Analyze [API 3] functionality, parameters, responses and use cases -Worker node bb1cf632-8df3-438d-afe3-f9499e6a3aea (API Analyst 4) get task parallel_3_1: Analyze [API 4] functionality, parameters, responses and use cases -====== -Response from Worker node a1506367-5bb1-47ba-8532-4e0603830e4f (API Analyst 1): - -[API 1] GET /resources - This endpoint is used to retrieve information about resources. - -Functionality: -- Allows clients to fetch a list of all resources. -- Can also be used to fetch a single resource by specifying an ID (often via query parameter or path, e.g., /resources/{id}). - -Parameters: -- Optional query parameters such as filters (e.g., ?type=active), pagination (e.g., ?page=2&limit=10), sorting (e.g., ?sort=name), and specific resource ID if retrieving a single resource. -- Authorization headers may be required if resources are protected. - -Responses: -- 200 OK: Success, returns an array of resource objects for a list, or a resource object for a specific ID. -- 404 Not Found: Returned when a specific resource ID does not exist. -- 401 Unauthorized/403 Forbidden: If access is denied or the client is not authenticated. -- Typical response body for list: [{"id":1, "name":"Resource 1", ...}, ...] -- Typical response body for single: {"id":1, "name":"Resource 1", ...} - -Use Cases: -- Displaying a list of available resources in an application dashboard. -- Fetching detailed information of a specific resource for viewing or editing. -- Supporting features like searching, filtering, and pagination in front-end apps consuming the resource list. - -In summary, [API 1] implements standard RESTful 'read' behavior for resources, supporting querying and retrieval for various client needs. -====== -🎯 Task parallel_0_1 completed successfully. -====== -Response from Worker node a786426e-ff39-458e-95b3-14453f37028c (API Analyst 3): - -[API 3] PUT /resources/{id} is an endpoint designed to update an existing resource on the server, where the resource is identified by its unique 'id'. - -Functionality: -- This endpoint replaces the current contents of the resource with the data provided in the request body. -- It expects the full representation of the resource in the request payload. - -Parameters: -- Path Parameter: 'id' (required) – the unique identifier of the resource to be updated. -- Request Body: A JSON (or other accepted format) object that represents the complete, updated state of the resource. The specific fields depend on the resource model (not specified here). -- Headers: Typically expects 'Content-Type: application/json' (can vary). - -Responses: -- 200 OK: Resource was successfully fully updated; the response typically returns the updated resource or a success message. -- 204 No Content: Resource updated successfully with no content returned (varies by implementation). -- 400 Bad Request: Invalid input data or malformed request. -- 404 Not Found: A resource with the specified 'id' does not exist. -- 401 Unauthorized/403 Forbidden: The client is not authorized to perform the action (if authentication is required). - -Use Cases: -- Updating all properties of a user's profile information by sending a complete updated profile to PUT /resources/123. -- Replacing an entire configuration object for a device or application. -- Correcting all fields of a previously submitted data record by resubmitting the entire object. - -Best Practices: -- Clients should send the complete representation of the resource, not just the fields that are changing. -- For partial updates, PATCH requests are more appropriate; PUT is intended for full replacement. -- Validate input data to avoid overwriting existing fields with null or default values inadvertently. - -Overall, [API 3] PUT /resources/{id} is used where full replacement of a resource's data is required, ensuring that the resource's state on the server exactly matches the client's submitted data. -====== -🎯 Task parallel_2_1 completed successfully. -====== -Response from Worker node bb1cf632-8df3-438d-afe3-f9499e6a3aea (API Analyst 4): - -Analysis of [API 4]: - -Functionality: -[API 4] is a RESTful endpoint defined as DELETE /resources/{id}. It removes a specific resource identified by its unique ID from the server. This operation is destructive and irreversible, making it essential for data management and cleanup workflows. - -Parameters: -- Path Parameter: id (required) – the unique identifier of the resource to be deleted. - -Typical endpoint format: DELETE /resources/12345 - -Responses: -- Success (typically 200 OK or 204 No Content): Indicates the resource was successfully deleted. Some implementations may return the deleted resource data or just a confirmation message. -- 404 Not Found: Returned if the resource with the provided ID does not exist. -- 400 Bad Request: Returned if the ID is improperly formatted or invalid. -- 401 Unauthorized / 403 Forbidden: Indicate lack of access permissions for the operation. - -Use Cases: -- Removing obsolete or irrelevant resources (e.g., deleting a user, file, or record). -- Supporting administrative functions (e.g., batch deletions). -- Implementing resource lifecycle management. -- Enabling users to retract or remove their own content or data. - -Best Practice: -- Ensure the operation is idempotent: multiple DELETE requests for the same non-existent resource should result in the same response (typically 404 Not Found). -- Consider implementing role-based access control to prevent unauthorized deletions. - -Summary: [API 4] enables clients to permanently remove resources by ID. Appropriate checks, confirmations, and security measures should be implemented to avoid accidental or unauthorized deletions. -====== -🎯 Task parallel_3_1 completed successfully. -====== -Response from Worker node cef71fcc-1304-461b-8267-9217cb3e9cac (API Analyst 2): - -[API 2] Functionality, Parameters, Responses, and Use Cases Analysis: - -Functionality: -[API 2] is a POST endpoint at /resources. Its primary function is to create a new resource on the server. The client sends the details of the resource to be created in the request body, and the server processes this request and returns either the created resource object or a unique identifier for the new resource. - -Parameters: -- Request Body: The specific properties or fields required for resource creation will depend on the type of resource the API manages, but typically this would be a JSON object containing required and optional fields (e.g., name, description, attributes of the resource). No path or query parameters are typically used for a create endpoint. -- Headers: Usually includes 'Content-Type: application/json' to indicate JSON payload; may also require authentication/authorization headers (e.g., 'Authorization: Bearer '). - -Responses: -- Success (201 Created): Returns the full details of the created resource (as a JSON object), or a JSON object with at least the unique identifier assigned (e.g., { "id": 123 }). -- Failure (4xx/5xx): - - 400 Bad Request: Returned if the input data is invalid or required fields are missing. - - 401 Unauthorized/403 Forbidden: If the user is not authenticated/authorized. - - 409 Conflict: If a resource with the same unique value already exists. - - 500 Internal Server Error: For unexpected server errors. - -Use Cases: -- Creating a new data entry (e.g., a new user profile, an article, a product in inventory). -- Submitting a form by an end user to add a new entity. -- Backend tools or automation scripts populating the system with new resources. - -Best Practices: -- Client should validate input before making the request. -- Server should return appropriate status codes and error messages for all failure scenarios. -- Use idempotency keys if creating resources where duplicate submissions should not create duplicate records. - -In summary, [API 2] POST /resources is the standard endpoint for resource creation, expects a well-formed JSON body with all mandatory fields, and returns the result of the operation with relevant HTTP status codes and a response object. -====== -🎯 Task parallel_1_1 completed successfully. -Worker node 743a178b-9d2a-4e42-901d-fdba755319fc (Documentation Writer) get task pipeline_task_5: Generate complete API usage guide based on 4 endpoint analyses -2025-09-21 10:50:54,339 - camel.models.model_manager - ERROR - Error processing with model: -2025-09-21 10:50:54,339 - camel.camel.agents.chat_agent - WARNING - Rate limit hit (attempt 1/3). Retrying in 0.1s -2025-09-21 10:51:11,829 - camel.models.model_manager - ERROR - Error processing with model: -2025-09-21 10:51:11,829 - camel.camel.agents.chat_agent - WARNING - Rate limit hit (attempt 2/3). Retrying in 0.3s -2025-09-21 10:51:44,868 - camel.camel.societies.workforce.structured_output_handler - WARNING - Failed to parse TaskResult from response. Using fallback values. -====== -Response from Worker node 743a178b-9d2a-4e42-901d-fdba755319fc (Documentation Writer): - -Task processing failed -====== -2025-09-21 10:51:44,868 - camel.camel.societies.workforce.workforce - ERROR - Task pipeline_task_5 failed (attempt 1/3): Task processing failed (assigned to worker: 743a178b-9d2a-4e42-901d-fdba755319fc) -Task api_documentation completed: -Result: --- Task pipeline_task_0 Result --- -[API 1] GET /resources - Retrieves a list of resources or a single resource by ID. Typically used to read or fetch data from the server. - -[API 2] POST /resources - ... +Technical features: + β€’ Pipeline mode with pipeline_fork() and pipeline_join() + β€’ Automatic worker assignment and task routing + β€’ Multi-agent parallel coordination + β€’ Structured task dependencies -[TIME] Execution time: 80.34 seconds +[TIME] Execution time: 56.37 seconds ============================================================ -Example 3: Code Review Pipeline +Pipeline Example Summary ============================================================ -Code review pipeline built: Scan β†’ 3 parallel reviews β†’ Summary report -Mode: WorkforceMode.PIPELINE -Worker node d7079fe4-be5b-49b9-b3c6-2712633de9db (Code Scanner) get task pipeline_task_0: Scan these code files and format as [File 1], [File 2], [File 3]: +Pipeline pattern demonstrated: + βœ“ FORK-JOIN: Single task β†’ 5 parallel processing β†’ 1 synthesis + βœ“ PARALLEL SCALING: Easy adjustment of parallel worker count + βœ“ DEPENDENCY CHAINS: Automatic synchronization and dependency resolution + βœ“ STRUCTURED OUTPUT: Using [markers] for smart task distribution -{ - "file1.py": "\ndef calculate_total(items):\n total = 0\n for item in items:\n total += item.price * item.quantity\n return total\n\nclass ShoppingCart:\n def __init__(self):\n self.items = []\n \n def add_item(self, item):\n self.items.append(item)\n \n def get_total(self):\n return calculate_total(self.items)\n", - "file2.py": "\nimport requests\nimport json\n\ndef fetch_user_data(user_id):\n url = f\"https://api.example.com/users/{user_id}\"\n response = requests.get(url)\n if response.status_code == 200:\n return json.loads(response.text)\n return None\n\ndef process_users(user_ids):\n results = []\n for uid in user_ids:\n data = fetch_user_data(uid)\n if data:\n results.append(data)\n return results\n", - "file3.py": "\nclass DatabaseConnection:\n def __init__(self, host, port, database):\n self.host = host\n self.port = port \n self.database = database\n self.connection = None\n \n def connect(self):\n # Simulate database connection\n self.connection = f\"connected to {self.host}:{self.port}/{self.database}\"\n return True\n \n def execute_query(self, query):\n if not self.connection:\n raise Exception(\"Not connected to database\")\n return f\"Executed: {query}\"\n" -} -====== -Response from Worker node d7079fe4-be5b-49b9-b3c6-2712633de9db (Code Scanner): - -[File 1] -file1.py: - -``` -def calculate_total(items): - total = 0 - for item in items: - total += item.price * item.quantity - return total - -class ShoppingCart: - def __init__(self): - self.items = [] - - def add_item(self, item): - self.items.append(item) - - def get_total(self): - return calculate_total(self.items) -``` - -[File 2] -file2.py: - -``` -import requests -import json - -def fetch_user_data(user_id): - url = f"https://api.example.com/users/{user_id}" - response = requests.get(url) - if response.status_code == 200: - return json.loads(response.text) - return None - -def process_users(user_ids): - results = [] - for uid in user_ids: - data = fetch_user_data(uid) - if data: - results.append(data) - return results -``` - -[File 3] -file3.py: - -``` -class DatabaseConnection: - def __init__(self, host, port, database): - self.host = host - self.port = port - self.database = database - self.connection = None - - def connect(self): - # Simulate database connection - self.connection = f"connected to {self.host}:{self.port}/{self.database}" - return True - - def execute_query(self, query): - if not self.connection: - raise Exception("Not connected to database") - return f"Executed: {query}" -``` -====== -🎯 Task pipeline_task_0 completed successfully. -Worker node f24b0d6e-22ea-4029-a16d-ba88a271e316 (Code Reviewer 1) get task parallel_0_1: Review [File 1] for code quality, bugs, and improvements -Worker node c54b9e75-b1f6-449d-bc57-80c5c32a1fed (Code Reviewer 2) get task parallel_1_1: Review [File 2] for code quality, bugs, and improvements -Worker node 3993ab6f-8fa5-4176-8eee-1c3109c250ab (Code Reviewer 3) get task parallel_2_1: Review [File 3] for code quality, bugs, and improvements -====== -Response from Worker node f24b0d6e-22ea-4029-a16d-ba88a271e316 (Code Reviewer 1): - -Review of [File 1]: - -Code: -``` -def calculate_total(items): - total = 0 - for item in items: - total += item.price * item.quantity - return total - -class ShoppingCart: - def __init__(self): - self.items = [] - - def add_item(self, item): - self.items.append(item) - - def get_total(self): - return calculate_total(self.items) -``` - -Code Quality: -- The code is simple, readable, and follows standard Python coding conventions. -- Class and function names are descriptive. -- Method and function signatures are clear. - -Potential Bugs: -- There is an implicit assumption that `item` objects passed to `calculate_total` have `price` and `quantity` attributes. No type-checking or error handling is present for invalid items. -- There is no check for non-numeric or negative values in `price` or `quantity`. - -Performance: -- For reasonably sized lists, the current implementation is efficient. There are no obvious performance concerns. - -Improvements: -1. Add type and value checking in `calculate_total` for `price` and `quantity` fields. -2. Optionally, consider supporting property checking or try/except block in case items are malformed. -3. Optionally, provide typing annotations to improve code clarity and assist with static analysis. -4. Consider defining an `Item` class or namedtuple for items, to ensure structure and type safety in the cart. -5. Consider handling situations where an item might be added more than once (aggregate quantities). -6. Docstrings could be added for better documentation. - -Example of an improved calculate_total function: -```python -def calculate_total(items): - total = 0 - for item in items: - try: - price = float(item.price) - quantity = int(item.quantity) - if price < 0 or quantity < 0: - continue # or raise ValueError - total += price * quantity - except (AttributeError, ValueError, TypeError): - continue # or handle/log the error appropriately - return total -``` - -Overall, the current implementation is functional but can be made more robust with the suggested improvements. -====== -🎯 Task parallel_0_1 completed successfully. -====== -Response from Worker node c54b9e75-b1f6-449d-bc57-80c5c32a1fed (Code Reviewer 2): - -Review of [File 2] (file2.py): - -**Code Quality:** -- The code is readable and functionally straightforward. -- Usage of requests and json modules is appropriate. -- Function and variable naming are clear and concise. - -**Bugs:** -- No outright bugs are present in the code as written. Both functions perform their jobs as described. -- However, there is no exception handling for the HTTP request. If the GET request fails (e.g., due to network issues), raises an exception, or non-2xx responses other than 200 are received, the code does not handle these cases, which may cause the program to crash. - -**Performance Issues:** -- The process_users function runs fetch_user_data serially for each user_id. If there are many user_ids, this could lead to slow performance due to sequential network calls. Using concurrency (e.g., via threading, asyncio, or concurrent.futures) could considerably speed up processing large user_id lists. - -**Best Practices & Improvements:** -1. **Exception Handling:** Add exception handling to catch requests.exceptions.RequestException and possibly json.JSONDecodeError inside fetch_user_data to prevent crashes. -2. **Timeouts:** The requests.get() call lacks a timeout. This may potentially cause the call to hang indefinitely if the remote server does not respond. It is best practice to specify a timeout (e.g., requests.get(url, timeout=5)). -3. **Direct Usage of response.json():** Instead of manually decoding JSON with json.loads(response.text), you can use response.json(), which is cleaner and more efficient, and already parses JSON (though it can still throw exceptions if JSON is invalid). - -**Suggested Improved Version:** -```python -import requests - -def fetch_user_data(user_id): - url = f"https://api.example.com/users/{user_id}" - try: - response = requests.get(url, timeout=5) - response.raise_for_status() - return response.json() - except (requests.RequestException, ValueError): # ValueError for JSON decoding issues - return None - -def process_users(user_ids): - results = [] - for uid in user_ids: - data = fetch_user_data(uid) - if data: - results.append(data) - return results -``` -For improved performance, consider parallelizing requests for large lists. - -**Summary:** -- The code is functionally correct but lacks exception handling and timeout safety. -- Consider using response.json(), add try/except blocks, and set a timeout on network requests. -- For significant performance increases with large lists, parallel processing should be considered. - -====== -🎯 Task parallel_1_1 completed successfully. -====== -Response from Worker node 3993ab6f-8fa5-4176-8eee-1c3109c250ab (Code Reviewer 3): - -Review for [File 3] (file3.py): - -Code Quality: -- The class `DatabaseConnection` is written in a readable way and adheres to typical Python conventions. -- Method and variable naming is clear and descriptive. -- Docstrings and comments are minimal. Only one comment is present in `connect()`; adding docstrings to each method and the class itself would improve understandability and maintainability. - -Bugs: -- There are no functional bugs in this mock implementation, as it's simply simulating connections and query execution with string manipulations. -- However, the constructor does not validate host, port, or database arguments; passing invalid values could cause unexpected behaviors in a real-world scenario. - -Performance: -- Since this is a mock/simulated connection and not a real database connection, there are no significant performance concerns. In a real implementation, connecting to a database inside the object constructor or failing to close connections would be a problem, but those do not apply here. - -Best Practices & Improvements: -1. **Docstrings**: Add method and class docstrings for clarity. -2. **Connection State**: Consider adding a `disconnect()` method to fully simulate a connection lifecycle. -3. **Exception Specificity**: Raise a more specific exception (e.g., a custom `DatabaseConnectionError`) rather than a generic `Exception` when not connected. -4. **Validate Parameters**: Optionally, validate constructor parameters (`host`, `port`, `database`) to ensure they meet expected types/values. -5. **PEP8**: The code is PEP8-compliant except for an extra space after `self.port = port `; remove trailing white space. - -Example incorporating improvements: -```python -class DatabaseConnection: - """Simulates a database connection.""" - - def __init__(self, host, port, database): - self.host = host - self.port = port - self.database = database - self.connection = None - - def connect(self): - """Simulate establishing a database connection.""" - self.connection = f"connected to {self.host}:{self.port}/{self.database}" - return True - - def disconnect(self): - """Simulate closing a database connection.""" - self.connection = None - - def execute_query(self, query): - """Simulate executing a database query.""" - if not self.connection: - raise ConnectionError("Not connected to database") - return f"Executed: {query}" -``` - -Summary: The code is clean and correct for a fake/mock connection class. The suggestions above are for further robustness, readability, and code quality in cases where a more realistic simulation or extension is needed. -====== -🎯 Task parallel_2_1 completed successfully. -Worker node 24b9cc1f-a802-4789-a422-1b09fff96e07 (Review Summarizer) get task pipeline_task_4: Generate comprehensive code review report with all findings -====== -Response from Worker node 24b9cc1f-a802-4789-a422-1b09fff96e07 (Review Summarizer): - -Comprehensive Code Review Report - -1. [File 1] Shopping Cart Logic ---- -Code Quality: -- The code is readable and uses descriptive naming for classes and methods. -- The implementation is straightforward, but lacks documentation (docstrings) for functions and classes. - -Bugs & Reliability: -- Assumes items in the cart possess `price` and `quantity` attributes, but performs no type or attribute validation. This may lead to attribute errors if invalid objects are added. -- Negative or non-numeric values for `price` and `quantity` are not checked. - -Improvements Suggested: -- Add explicit attribute and type checking in `calculate_total`. -- Consider using try/except to catch malformed item entries. -- Implement type annotations for better code clarity. -- Provide an `Item` class or structure (e.g., namedtuple) for stronger type safety. -- Optionally, aggregate identical items rather than allowing duplicates. -- Add docstrings for all methods and classes for maintainability. - -2. [File 2] User Data Retrieval Functions ---- -Code Quality: -- Code is readable, function and variable names are clear. -- Makes appropriate use of the `requests` and `json` modules. - -Bugs & Reliability: -- Lacks exception handling for relevant failure scenarios, such as network errors or malformed JSON, leading to potential program crashes. - -Performance: -- The current implementation fetches user data serially for each ID, which is inefficient for large lists. Consider concurrency or parallelization for improved performance. - -Best Practices & Improvements: -- Add try/except blocks for network and JSON decoding errors (e.g., `requests.RequestException`, `ValueError`). -- Set a timeout in HTTP requests to prevent hanging. -- Use `response.json()` instead of manual JSON parsing for safety and simplicity. -- For large user lists, investigate parallel execution (e.g., threading or async calls). - -3. [File 3] Mock DatabaseConnection Class ---- -Code Quality: -- Code is well organized, readable, and mostly PEP8 compliant (except one minor whitespace issue). - -Bugs & Reliability: -- Class is a mock and not functionally problematic, but lacks validation of constructor parameters (host, port, database). -- Only one comment is present and there are no docstrings; code documentation should be added. -- The `execute_query` method raises a generic Exception for not being connected. - -Improvements Suggested: -- Add class-level and method-level docstrings for maintainability. -- Validate constructor arguments for expected types and values. -- Add a `disconnect` method to fully represent the connection lifecycle. -- Raise more specific exceptions (e.g., custom `ConnectionError`) for better error handling. -- Remove extra whitespace to maintain PEP8 compliance. - ---- - -General Recommendations: -- Across all files, improve input validation, type safety, and add comprehensive docstrings. -- Consistent and specific exception handling will improve robustness and maintainability. -- Consider performance implications and use concurrency where relevant when interacting with external systems (e.g., web APIs). - -All code reviewed is functional within scope, but can be improved in terms of error handling, input validation, and code documentation for production use. -====== -🎯 Task pipeline_task_4 completed successfully. -Task code_review completed: -Result: --- Task pipeline_task_0 Result --- -[File 1] -file1.py: - -``` -def calculate_total(items): - total = 0 - for item in items: - total += item.price * item.quantity - return total - -class Shoppin... +Technical features: + β€’ Pipeline mode with pipeline_fork() and pipeline_join() + β€’ Automatic worker assignment and task routing + β€’ Multi-agent parallel coordination + β€’ Structured task dependencies -[TIME] Execution time: 33.88 seconds ============================================================ -Pipeline Examples Summary +Pipeline Example Summary ============================================================ -Examples demonstrated: - 1. Literature Analysis - Fork/join pattern with 5 parallel agents - 2. API Documentation - 4 parallel analysts with structured parsing - 3. Data Analysis - Multi-stage workflow with dataset branching - -Pipeline patterns tested: - SEARCH BRANCH: Single search distributed to multiple specialists - FORK-JOIN: Parallel processing with automatic synchronization - STRUCTURED OUTPUT: Using [markers] for smart task distribution - SMART ROUTING: Agents identify their target data sections - PARALLEL SCALING: Easy adjustment of parallel worker count - DEPENDENCY CHAINS: Multi-stage workflows with proper sequencing +Pipeline pattern demonstrated: + βœ“ FORK-JOIN: Single task β†’ 5 parallel processing β†’ 1 synthesis + βœ“ PARALLEL SCALING: Easy adjustment of parallel worker count + βœ“ DEPENDENCY CHAINS: Automatic synchronization and dependency resolution + βœ“ STRUCTURED OUTPUT: Using [markers] for smart task distribution Technical features: - ModelFactory.create() for agent initialization - SearchToolkit integration - Pipeline mode with fork_pipeline() and join_pipeline() - Automatic dependency resolution - Structured output parsing - Multi-agent coordination + β€’ Pipeline mode with pipeline_fork() and pipeline_join() + β€’ Automatic worker assignment and task routing + β€’ Multi-agent parallel coordination + β€’ Structured task dependencies + + β€’ Multi-agent parallel coordination + β€’ Structured task dependencies -Pipeline examples completed successfully! -Total examples: 3 +Pipeline example completed successfully! """ \ No newline at end of file From ad771b632fbc12a067b1de5b271eeae0544148a2 Mon Sep 17 00:00:00 2001 From: Ol1ver0413 <790221864@qq.com> Date: Tue, 21 Oct 2025 21:43:55 -0400 Subject: [PATCH 4/6] solve merge conflict --- "\\" | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 "\\" diff --git "a/\\" "b/\\" new file mode 100644 index 0000000000..bd41cc15eb --- /dev/null +++ "b/\\" @@ -0,0 +1,6 @@ +Merge branch 'workforce_pipeline' of https://github.com/camel-ai/camel into workforce_pipeline +# Please enter a commit message to explain why this merge is necessary, +# especially if it merges an updated upstream into a topic branch. +# +# Lines starting with '#' will be ignored, and an empty message aborts +# the commit. From 3bc8da5e5470651722bbc5a252a5e446197a551a Mon Sep 17 00:00:00 2001 From: Ol1ver0413 <790221864@qq.com> Date: Tue, 21 Oct 2025 21:45:17 -0400 Subject: [PATCH 5/6] del unnecessary file --- "\\" | 6 ------ 1 file changed, 6 deletions(-) delete mode 100644 "\\" diff --git "a/\\" "b/\\" deleted file mode 100644 index bd41cc15eb..0000000000 --- "a/\\" +++ /dev/null @@ -1,6 +0,0 @@ -Merge branch 'workforce_pipeline' of https://github.com/camel-ai/camel into workforce_pipeline -# Please enter a commit message to explain why this merge is necessary, -# especially if it merges an updated upstream into a topic branch. -# -# Lines starting with '#' will be ignored, and an empty message aborts -# the commit. From 4386e93920fde38b8840ec8c2595c007da9d618e Mon Sep 17 00:00:00 2001 From: Ol1ver0413 <790221864@qq.com> Date: Sun, 26 Oct 2025 22:36:31 +0800 Subject: [PATCH 6/6] wk_pipeline_0.4 --- camel/societies/workforce/utils.py | 7 ++-- camel/societies/workforce/workforce.py | 49 ++++++++++++++++++-------- 2 files changed, 38 insertions(+), 18 deletions(-) diff --git a/camel/societies/workforce/utils.py b/camel/societies/workforce/utils.py index 570a46e6cb..9101fd317b 100644 --- a/camel/societies/workforce/utils.py +++ b/camel/societies/workforce/utils.py @@ -380,10 +380,10 @@ def add_sync_task( """ # Auto-infer wait_for from last parallel tasks if wait_for is None: - if hasattr(self, '_last_parallel_tasks') and self._last_parallel_tasks: + if self._last_parallel_tasks: wait_for = self._last_parallel_tasks # Clear the parallel tasks after using them - delattr(self, '_last_parallel_tasks') + self._last_parallel_tasks = [] else: raise ValueError("wait_for cannot be empty for sync task and no parallel tasks found") @@ -416,8 +416,7 @@ def clear(self) -> None: self._task_registry.clear() self.task_counter = 0 self._last_task_id = None - if hasattr(self, '_last_parallel_tasks'): - delattr(self, '_last_parallel_tasks') + self._last_parallel_tasks = [] def fork(self, task_contents: List[str]) -> 'PipelineTaskBuilder': """Create parallel branches from the current task (alias for add_parallel_tasks). diff --git a/camel/societies/workforce/workforce.py b/camel/societies/workforce/workforce.py index cf18122842..1ba4260cac 100644 --- a/camel/societies/workforce/workforce.py +++ b/camel/societies/workforce/workforce.py @@ -2045,12 +2045,6 @@ async def _process_task_with_pipeline(self, task: Task) -> Task: return task - async def _process_task_with_hybrid(self, task: Task) -> Task: - """Process task using hybrid approach (partial auto-decomposition).""" - # For now, fall back to auto-decompose mode - # This can be extended to support more sophisticated hybrid logic - return await self._process_task_with_auto_decompose(task) - def _collect_pipeline_results(self) -> str: """Collect results from all completed pipeline tasks.""" results = [] @@ -2060,16 +2054,43 @@ def _collect_pipeline_results(self) -> str: return "\n\n".join(results) if results else "Pipeline completed" def _all_pipeline_tasks_successful(self) -> bool: - """Check if all pipeline tasks completed successfully.""" - expected_task_ids = {task.id for task in self._pending_tasks} - expected_task_ids.update(task.id for task in self._completed_tasks) + """Check if all pipeline tasks completed successfully. - completed_successful_ids = { - task.id for task in self._completed_tasks - if task.state == TaskState.DONE - } + This method determines the FINAL STATE of the entire pipeline but does + NOT affect task execution flow. In PIPELINE mode: + + - Failed tasks still pass their results (including errors) to dependent + tasks, allowing join tasks to execute even when upstream tasks fail. + - This is handled in _post_ready_tasks() where dependencies are checked. + + This method only runs AFTER all tasks have been processed to determine + whether the overall pipeline should be marked as DONE or FAILED. + + Returns: + bool: True if all tasks completed successfully (DONE state), + False if any tasks failed or are still pending. + + Example: + Fork-Join pattern with one failed branch: + - Task A (search) β†’ DONE + - Task B (parallel summary 1) β†’ DONE + - Task C (parallel summary 2) β†’ FAILED + - Task D (join/synthesis) β†’ DONE (receives B's result + C's error) + + Result: _all_pipeline_tasks_successful() returns False + Main pipeline task marked as FAILED + But Task D still executed and got all information + """ + # 1. If there are still pending tasks, pipeline is not complete + if self._pending_tasks: + return False + + # 2. If no tasks were completed, consider it a failure + if not self._completed_tasks: + return False - return expected_task_ids.issubset(completed_successful_ids) + # 3. Check if all completed tasks succeeded + return all(task.state == TaskState.DONE for task in self._completed_tasks) def process_task(self, task: Task) -> Task: r"""Synchronous wrapper for process_task that handles async operations