From 546e59abe4f709378f0a30d475134edf0d7b09e5 Mon Sep 17 00:00:00 2001 From: Marco Perini Date: Wed, 12 Jun 2024 22:25:36 +0200 Subject: [PATCH] fix(deepcopy): TypeError related to pickling '_thread.RLock' objects --- .../graphs/smart_scraper_multi_graph.py | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/scrapegraphai/graphs/smart_scraper_multi_graph.py b/scrapegraphai/graphs/smart_scraper_multi_graph.py index 51e18739..5cfa2f55 100644 --- a/scrapegraphai/graphs/smart_scraper_multi_graph.py +++ b/scrapegraphai/graphs/smart_scraper_multi_graph.py @@ -2,7 +2,6 @@ SmartScraperMultiGraph Module """ -from copy import copy, deepcopy from typing import List, Optional from .base_graph import BaseGraph @@ -46,10 +45,21 @@ def __init__(self, prompt: str, source: List[str], config: dict, schema: Optiona self.max_results = config.get("max_results", 3) - if all(isinstance(value, str) for value in config.values()): - self.copy_config = copy(config) - else: - self.copy_config = deepcopy(config) + # Custom deepcopy function to handle non-picklable objects + def custom_deepcopy(obj): + if isinstance(obj, dict): + return {k: custom_deepcopy(v) for k, v in obj.items()} + elif isinstance(obj, list): + return [custom_deepcopy(v) for v in obj] + elif hasattr(obj, '__dict__'): + new_obj = obj.__class__() + for attr in obj.__dict__: + setattr(new_obj, attr, custom_deepcopy(getattr(obj, attr))) + return new_obj + else: + return obj + + self.copy_config = custom_deepcopy(config) super().__init__(prompt, config, source, schema)