plugins for processors

changedetectionio/flask_app.py

@@ -712,23 +712,63 @@ def edit_page(uuid):
         # Does it use some custom form? does one exist?
         processor_name = datastore.data['watching'][uuid].get('processor', '')
         processor_classes = next((tpl for tpl in find_processors() if tpl[1] == processor_name), None)
+
+        # If it's not found in traditional processors, check if it's a pluggy plugin
         if not processor_classes:
-            flash(f"Cannot load the edit form for processor/plugin '{processor_classes[1]}', plugin missing?", 'error')
-            return redirect(url_for('index'))
-
-        parent_module = get_parent_module(processor_classes[0])
-
-        try:
-            # Get the parent of the "processor.py" go up one, get the form (kinda spaghetti but its reusing existing code)
-            forms_module = importlib.import_module(f"{parent_module.__name__}.forms")
-            # Access the 'processor_settings_form' class from the 'forms' module
-            form_class = getattr(forms_module, 'processor_settings_form')
-        except ModuleNotFoundError as e:
-            # .forms didnt exist
-            form_class = forms.processor_text_json_diff_form
-        except AttributeError as e:
-            # .forms exists but no useful form
-            form_class = forms.processor_text_json_diff_form
+            try:
+                from changedetectionio.processors.processor_registry import get_processor_form, _get_plugin_name_map
+
+                # Get all available plugins for debugging
+                available_plugins = list(_get_plugin_name_map().keys())
+                logger.debug(f"Available processor plugins: {available_plugins}")
+
+                # Try to get the processor form
+                plugin_form_class = get_processor_form(processor_name)
+
+                if plugin_form_class:
+                    # Use default text_json_diff_form as parent module for plugins
+                    from changedetectionio.processors.text_json_diff import processor as text_json_diff_processor
+                    form_class = forms.processor_text_json_diff_form
+                    parent_module = get_parent_module(text_json_diff_processor)
+
+                    # Skip the normal form loading code path
+                    use_plugin_form = True
+                    logger.debug(f"Successfully loaded form for plugin '{processor_name}'")
+                else:
+                    # Check if the plugin is registered but doesn't have a form
+                    if processor_name in available_plugins:
+                        logger.error(f"Plugin '{processor_name}' is registered but has no form class")
+                        flash(f"Plugin '{processor_name}' is registered but has no form class", 'error')
+                    else:
+                        logger.error(f"Cannot find plugin '{processor_name}'. Available plugins: {available_plugins}")
+                        flash(f"Cannot load the edit form for processor/plugin '{processor_name}', plugin missing?", 'error')
+                    return redirect(url_for('index'))
+            except ImportError as e:
+                logger.error(f"Import error when loading plugin form: {str(e)}")
+                flash(f"Cannot load the edit form for processor/plugin '{processor_name}', plugin system not available?", 'error')
+                return redirect(url_for('index'))
+            except Exception as e:
+                logger.error(f"Unexpected error loading plugin form: {str(e)}")
+                flash(f"Error loading plugin form: {str(e)}", 'error')
+                return redirect(url_for('index'))
+        else:
+            # Traditional processor - continue with normal flow
+            parent_module = get_parent_module(processor_classes[0])
+            use_plugin_form = False
+
+        # Only follow this path for traditional processors
+        if not use_plugin_form:
+            try:
+                # Get the parent of the "processor.py" go up one, get the form (kinda spaghetti but its reusing existing code)
+                forms_module = importlib.import_module(f"{parent_module.__name__}.forms")
+                # Access the 'processor_settings_form' class from the 'forms' module
+                form_class = getattr(forms_module, 'processor_settings_form')
+            except ModuleNotFoundError as e:
+                # .forms didnt exist
+                form_class = forms.processor_text_json_diff_form
+            except AttributeError as e:
+                # .forms exists but no useful form
+                form_class = forms.processor_text_json_diff_form
 
         form = form_class(formdata=request.form if request.method == 'POST' else None,
                           data=default,

changedetectionio/model/Watch.py

@@ -67,7 +67,6 @@ def ensure_data_dir_exists(self):
 
     @property
     def link(self):
-
         url = self.get('url', '')
         if not is_safe_url(url):
             return 'DISABLED'
@@ -93,6 +92,19 @@ def link(self):
         # Also double check it after any Jinja2 formatting just incase
         if not is_safe_url(ready_url):
             return 'DISABLED'
+
+        # Check if a processor wants to customize the display link
+        processor_name = self.get('processor')
+        if processor_name:
+            try:
+                # Import here to avoid circular imports
+                from changedetectionio.processors.processor_registry import get_display_link
+                custom_link = get_display_link(url=ready_url, processor_name=processor_name)
+                if custom_link:
+                    return custom_link
+            except Exception as e:
+                logger.error(f"Error getting custom display link for processor {processor_name}: {str(e)}")
+
         return ready_url
 
     def clear_watch(self):

changedetectionio/processors/__init__.py

@@ -3,13 +3,18 @@
 from changedetectionio.strtobool import strtobool
 from copy import deepcopy
 from loguru import logger
+
 import hashlib
 import importlib
 import inspect
 import os
 import pkgutil
 import re
 
+# Import the plugin manager
+from .pluggy_interface import plugin_manager
+
+
 class difference_detection_processor():
 
     browser_steps = None
@@ -26,9 +31,95 @@ def __init__(self, *args, datastore, watch_uuid, **kwargs):
         self.watch = deepcopy(self.datastore.data['watching'].get(watch_uuid))
         # Generic fetcher that should be extended (requests, playwright etc)
         self.fetcher = Fetcher()
+
+    def _get_proxy_for_watch(self, preferred_proxy_id=None):
+        """Get proxy configuration based on watch settings and preferred proxy ID
+
+        Args:
+            preferred_proxy_id: Optional explicit proxy ID to use
+
+        Returns:
+            dict: Proxy configuration or None if no proxy should be used
+            str: Proxy URL or None if no proxy should be used
+        """
+        # Default to no proxy config
+        proxy_config = None
+        proxy_url = None
+
+        # Check if datastore is available and has get_preferred_proxy_for_watch method
+        if hasattr(self, 'datastore') and self.datastore:
+            try:
+                # Get preferred proxy ID if not provided
+                if not preferred_proxy_id and hasattr(self.datastore, 'get_preferred_proxy_for_watch'):
+                    # Get the watch UUID if available
+                    watch_uuid = None
+                    if hasattr(self.watch, 'get'):
+                        watch_uuid = self.watch.get('uuid')
+                    elif hasattr(self.watch, 'uuid'):
+                        watch_uuid = self.watch.uuid
+
+                    if watch_uuid:
+                        preferred_proxy_id = self.datastore.get_preferred_proxy_for_watch(uuid=watch_uuid)
+
+                # Check if we have a proxy list and a valid proxy ID
+                if preferred_proxy_id and hasattr(self.datastore, 'proxy_list') and self.datastore.proxy_list:
+                    proxy_info = self.datastore.proxy_list.get(preferred_proxy_id)
+
+                    if proxy_info and 'url' in proxy_info:
+                        proxy_url = proxy_info.get('url')
+                        logger.debug(f"Selected proxy key '{preferred_proxy_id}' as proxy URL '{proxy_url}'")
+
+                        # Parse the proxy URL to build a proxy dict for requests
+                        import urllib.parse
+                        parsed_proxy = urllib.parse.urlparse(proxy_url)
+                        proxy_type = parsed_proxy.scheme
+
+                        # Extract credentials if present
+                        username = None
+                        password = None
+                        if parsed_proxy.username:
+                            username = parsed_proxy.username
+                            if parsed_proxy.password:
+                                password = parsed_proxy.password
+
+                        # Build the proxy URL without credentials for the proxy dict
+                        netloc = parsed_proxy.netloc
+                        if '@' in netloc:
+                            netloc = netloc.split('@')[1]
+
+                        proxy_addr = f"{proxy_type}://{netloc}"
+
+                        # Create the proxy configuration
+                        proxy_config = {
+                            'http': proxy_addr,
+                            'https': proxy_addr
+                        }
+
+                        # Add credentials if present
+                        if username:
+                            proxy_config['username'] = username
+                            if password:
+                                proxy_config['password'] = password
+            except Exception as e:
+                # Log the error but continue without a proxy
+                logger.error(f"Error setting up proxy: {str(e)}")
+                proxy_config = None
+                proxy_url = None
+
+        return proxy_config, proxy_url
 
     def call_browser(self, preferred_proxy_id=None):
-
+        """Fetch content using the appropriate browser/fetcher
+
+        This method will:
+        1. Determine the appropriate fetcher to use based on watch settings
+        2. Set up proxy configuration if needed
+        3. Initialize the fetcher with the correct parameters
+        4. Configure any browser steps if needed
+
+        Args:
+            preferred_proxy_id: Optional explicit proxy ID to use
+        """
         from requests.structures import CaseInsensitiveDict
 
         url = self.watch.link
@@ -43,8 +134,8 @@ def call_browser(self, preferred_proxy_id=None):
         # Requests, playwright, other browser via wss:// etc, fetch_extra_something
         prefer_fetch_backend = self.watch.get('fetch_backend', 'system')
 
-        # Proxy ID "key"
-        preferred_proxy_id = preferred_proxy_id if preferred_proxy_id else self.datastore.get_preferred_proxy_for_watch(uuid=self.watch.get('uuid'))
+        # Get proxy configuration
+        proxy_config, proxy_url = self._get_proxy_for_watch(preferred_proxy_id)
 
         # Pluggable content self.fetcher
         if not prefer_fetch_backend or prefer_fetch_backend == 'system':
@@ -82,14 +173,10 @@ def call_browser(self, preferred_proxy_id=None):
             # What it referenced doesnt exist, Just use a default
             fetcher_obj = getattr(content_fetchers, "html_requests")
 
-        proxy_url = None
-        if preferred_proxy_id:
-            # Custom browser endpoints should NOT have a proxy added
-            if not prefer_fetch_backend.startswith('extra_browser_'):
-                proxy_url = self.datastore.proxy_list.get(preferred_proxy_id).get('url')
-                logger.debug(f"Selected proxy key '{preferred_proxy_id}' as proxy URL '{proxy_url}' for {url}")
-            else:
-                logger.debug(f"Skipping adding proxy data when custom Browser endpoint is specified. ")
+        # Custom browser endpoints should NOT have a proxy added
+        if proxy_url and prefer_fetch_backend.startswith('extra_browser_'):
+            logger.debug(f"Skipping adding proxy data when custom Browser endpoint is specified.")
+            proxy_url = None
 
         # Now call the fetcher (playwright/requests/etc) with arguments that only a fetcher would need.
         # When browser_connection_url is None, it method should default to working out whats the best defaults (os env vars etc)
@@ -185,16 +272,17 @@ def find_sub_packages(package_name):
 
 def find_processors():
     """
-    Find all subclasses of DifferenceDetectionProcessor in the specified package.
+    Find all subclasses of DifferenceDetectionProcessor in the specified package
+    and also include processors from the plugin system.
 
-    :param package_name: The name of the package to scan for processor modules.
     :return: A list of (module, class) tuples.
     """
     package_name = "changedetectionio.processors"  # Name of the current package/module
 
     processors = []
     sub_packages = find_sub_packages(package_name)
 
+    # Find traditional processors
     for sub_package in sub_packages:
         module_name = f"{package_name}.{sub_package}.processor"
         try:
@@ -207,6 +295,15 @@ def find_processors():
         except (ModuleNotFoundError, ImportError) as e:
             logger.warning(f"Failed to import module {module_name}: {e} (find_processors())")
 
+    # Also include processors from the plugin system
+    try:
+        from .processor_registry import get_plugin_processor_modules
+        plugin_modules = get_plugin_processor_modules()
+        if plugin_modules:
+            processors.extend(plugin_modules)
+    except (ImportError, ModuleNotFoundError) as e:
+        logger.warning(f"Failed to import plugin modules: {e} (find_processors())")
+
     return processors
 
 
@@ -223,8 +320,22 @@ def get_parent_module(module):
     return False
 
 
-
 def get_custom_watch_obj_for_processor(processor_name):
+    """
+    Get the custom watch object for a processor
+    :param processor_name: Name of the processor
+    :return: Watch class or None
+    """
+    # First, try to get the watch model from the pluggy system
+    try:
+        from .processor_registry import get_processor_watch_model
+        watch_model = get_processor_watch_model(processor_name)
+        if watch_model:
+            return watch_model
+    except Exception as e:
+        logger.warning(f"Error getting processor watch model from pluggy: {e}")
+
+    # Fall back to the traditional approach
     from changedetectionio.model import Watch
     watch_class = Watch.model
     processor_classes = find_processors()
@@ -241,14 +352,47 @@ def get_custom_watch_obj_for_processor(processor_name):
 def available_processors():
     """
     Get a list of processors by name and description for the UI elements
-    :return: A list :)
+    :return: A list of tuples (processor_name, description)
     """
-
-    processor_classes = find_processors()
-
-    available = []
-    for package, processor_class in processor_classes:
-        available.append((processor_class, package.name))
-
-    return available
-
+    # Get processors from the pluggy system
+    pluggy_processors = []
+    try:
+        from .processor_registry import get_all_processors
+        pluggy_processors = get_all_processors()
+    except Exception as e:
+        logger.error(f"Error getting processors from pluggy: {str(e)}")
+
+    # Get processors from the traditional file-based system
+    traditional_processors = []
+    try:
+        # Let's not use find_processors() directly since it now also includes pluggy processors
+        package_name = "changedetectionio.processors"
+        sub_packages = find_sub_packages(package_name)
+
+        for sub_package in sub_packages:
+            module_name = f"{package_name}.{sub_package}.processor"
+            try:
+                module = importlib.import_module(module_name)
+                # Get the name and description from the module if available
+                name = getattr(module, 'name', f"Traditional processor: {sub_package}")
+                description = getattr(module, 'description', sub_package)
+                traditional_processors.append((sub_package, name))
+            except (ModuleNotFoundError, ImportError, AttributeError) as e:
+                logger.warning(f"Failed to import module {module_name}: {e} (available_processors())")
+    except Exception as e:
+        logger.error(f"Error getting traditional processors: {str(e)}")
+
+    # Combine the lists, ensuring no duplicates
+    # Pluggy processors take precedence
+    all_processors = []
+
+    # Add all pluggy processors
+    all_processors.extend(pluggy_processors)
+
+    # Add traditional processors that aren't already registered via pluggy
+    pluggy_processor_names = [name for name, _ in pluggy_processors]
+    for processor_class, name in traditional_processors:
+        if processor_class not in pluggy_processor_names:
+            all_processors.append((processor_class, name))
+
+    return all_processors

changedetectionio/processors/form.py

@@ -0,0 +1,17 @@
+from wtforms import (
+    BooleanField,
+    validators,
+    RadioField
+)
+from wtforms.fields.choices import SelectField
+from wtforms.fields.form import FormField
+from wtforms.form import Form
+
+class BaseProcessorForm(Form):
+    """Base class for processor forms"""
+
+    def extra_tab_content(self):
+        return None
+
+    def extra_form_content(self):
+        return None

changedetectionio/processors/forms.py

@@ -0,0 +1,4 @@
+"""
+Forms for processors
+"""
+from changedetectionio.forms import processor_text_json_diff_form