diff --git a/scrapegraphai/utils/copy.py b/scrapegraphai/utils/copy.py index 2defbfa3..99cf8419 100644 --- a/scrapegraphai/utils/copy.py +++ b/scrapegraphai/utils/copy.py @@ -1,17 +1,39 @@ import copy -from typing import Any, Dict, Optional -from pydantic.v1 import BaseModel +from typing import Any + class DeepCopyError(Exception): """Custom exception raised when an object cannot be deep-copied.""" + pass + +def is_boto3_client(obj): + import sys + + # check boto3 module imprort + boto3_module = sys.modules.get("boto3") + + if boto3_module: + # boto3 use botocore client so here we import botocore + # if boto3 was imported the botocore will be import automatically normally + try: + from botocore.client import BaseClient + + return isinstance(obj, BaseClient) + except (AttributeError, ImportError): + # if the module is not imported, or the BaseClient class does not exist, return False + # if custome module name is boto3, the BaseClient class does not exist, + return False + return False + + def safe_deepcopy(obj: Any) -> Any: """ Attempts to create a deep copy of the object using `copy.deepcopy` whenever possible. If that fails, it falls back to custom deep copy logic. If that also fails, it raises a `DeepCopyError`. - + Args: obj (Any): The object to be copied, which can be of any type. @@ -24,7 +46,7 @@ def safe_deepcopy(obj: Any) -> Any: """ try: - + # Try to use copy.deepcopy first return copy.deepcopy(obj) except (TypeError, AttributeError) as e: @@ -33,7 +55,7 @@ def safe_deepcopy(obj: Any) -> Any: # Handle dictionaries if isinstance(obj, dict): new_obj = {} - + for k, v in obj.items(): new_obj[k] = safe_deepcopy(v) return new_obj @@ -41,7 +63,7 @@ def safe_deepcopy(obj: Any) -> Any: # Handle lists elif isinstance(obj, list): new_obj = [] - + for v in obj: new_obj.append(safe_deepcopy(v)) return new_obj @@ -49,7 +71,7 @@ def safe_deepcopy(obj: Any) -> Any: # Handle tuples (immutable, but might contain mutable objects) elif isinstance(obj, tuple): new_obj = tuple(safe_deepcopy(v) for v in obj) - + return new_obj # Handle frozensets (immutable, but might contain mutable objects) @@ -57,19 +79,16 @@ def safe_deepcopy(obj: Any) -> Any: new_obj = frozenset(safe_deepcopy(v) for v in obj) return new_obj + elif is_boto3_client(obj): + return obj + # Handle objects with attributes - elif hasattr(obj, "__dict__"): + else: # If an object cannot be deep copied, then the sub-properties of \ # the object will not be analyzed and shallow copy will be used directly. try: return copy.copy(obj) except (TypeError, AttributeError): - raise DeepCopyError(f"Cannot deep copy the object of type {type(obj)}") from e - - - # Attempt shallow copy as a fallback - try: - return copy.copy(obj) - except (TypeError, AttributeError): - raise DeepCopyError(f"Cannot deep copy the object of type {type(obj)}") from e - + raise DeepCopyError( + f"Cannot deep copy the object of type {type(obj)}" + ) from e diff --git a/tests/utils/copy_utils_test.py b/tests/utils/copy_utils_test.py index 90c85d34..4f684088 100644 --- a/tests/utils/copy_utils_test.py +++ b/tests/utils/copy_utils_test.py @@ -184,3 +184,9 @@ def test_with_pydantic(): copy_obj = safe_deepcopy(original) assert copy_obj.value == original.value assert copy_obj is not original + +def test_with_boto3(): + import boto3 + boto_client = boto3.client("bedrock-runtime", region_name="us-west-2") + copy_obj = safe_deepcopy(boto_client) + assert copy_obj == boto_client \ No newline at end of file