Allow users to run crawls with 1 or 2 browser windows

backend/btrixcloud/crawlconfigs.py

@@ -52,6 +52,7 @@
     validate_regexes,
     validate_language_code,
     is_url,
+    browser_windows_from_scale,
 )
 
 if TYPE_CHECKING:
@@ -222,6 +223,12 @@ async def add_crawl_config(
     ) -> CrawlConfigAddedResponse:
         """Add new crawl config"""
 
+        # Overrides scale if set
+        if config_in.browserWindows is None:
+            config_in.browserWindows = browser_windows_from_scale(
+                cast(int, config_in.scale)
+            )
+
         # ensure crawlChannel is valid
         if not self.get_channel_crawler_image(config_in.crawlerChannel):
             raise HTTPException(status_code=404, detail="crawler_not_found")
@@ -272,7 +279,7 @@ async def add_crawl_config(
             jobType=config_in.jobType,
             crawlTimeout=config_in.crawlTimeout,
             maxCrawlSize=config_in.maxCrawlSize,
-            scale=config_in.scale,
+            browserWindows=config_in.browserWindows,
             autoAddCollections=config_in.autoAddCollections,
             profileid=profileid,
             crawlerChannel=config_in.crawlerChannel,
@@ -408,6 +415,10 @@ async def update_crawl_config(
 
         orig_crawl_config = await self.get_crawl_config(cid, org.id)
 
+        if update.scale:
+            update.browserWindows = browser_windows_from_scale(cast(int, update.scale))
+            update.scale = None
+
         if update.config and update.config.exclude:
             exclude = update.config.exclude
             if isinstance(exclude, str):
@@ -441,7 +452,9 @@ async def update_crawl_config(
         changed = changed or (
             self.check_attr_changed(orig_crawl_config, update, "crawlFilenameTemplate")
         )
-        changed = changed or self.check_attr_changed(orig_crawl_config, update, "scale")
+        changed = changed or self.check_attr_changed(
+            orig_crawl_config, update, "browserWindows"
+        )
 
         schedule_changed = self.check_attr_changed(
             orig_crawl_config, update, "schedule"

backend/btrixcloud/crawlmanager.py

@@ -8,7 +8,7 @@
 
 from fastapi import HTTPException
 
-from .utils import dt_now, date_to_str
+from .utils import dt_now, date_to_str, scale_from_browser_windows
 from .k8sapi import K8sAPI
 
 from .models import StorageRef, CrawlConfig, BgJobType
@@ -227,13 +227,16 @@ async def create_crawl_job(
 
         await self.has_storage_secret(storage_secret)
 
+        scale = scale_from_browser_windows(crawlconfig.browserWindows)
+
         return await self.new_crawl_job(
             cid,
             userid,
             str(crawlconfig.oid),
             str(storage),
             crawlconfig.crawlerChannel,
-            crawlconfig.scale,
+            scale,
+            crawlconfig.browserWindows,
             crawlconfig.crawlTimeout,
             crawlconfig.maxCrawlSize,
             manual=True,
@@ -258,7 +261,8 @@ async def update_running_crawl_config(
         # pylint: disable=use-dict-literal
         patch = dict(
             crawlerChannel=crawlconfig.crawlerChannel,
-            scale=crawlconfig.scale,
+            scale=scale_from_browser_windows(crawlconfig.browserWindows),
+            browserWindows=crawlconfig.browserWindows,
             timeout=crawlconfig.crawlTimeout,
             maxCrawlSize=crawlconfig.maxCrawlSize,
             proxyId=crawlconfig.proxyId or DEFAULT_PROXY_ID,
@@ -373,9 +377,13 @@ async def rollover_restart_crawl(self, crawl_id: str) -> dict:
         update = date_to_str(dt_now())
         return await self._patch_job(crawl_id, {"restartTime": update})
 
-    async def scale_crawl(self, crawl_id: str, scale: int = 1) -> dict:
+    async def scale_crawl(
+        self, crawl_id: str, scale: int = 1, browser_windows: int = 1
+    ) -> dict:
         """Set the crawl scale (job parallelism) on the specified job"""
-        return await self._patch_job(crawl_id, {"scale": scale})
+        return await self._patch_job(
+            crawl_id, {"scale": scale, "browserWindows": browser_windows}
+        )
 
     async def shutdown_crawl(self, crawl_id: str, graceful=True) -> dict:
         """Request a crawl cancelation or stop by calling an API

backend/btrixcloud/crawls.py

@@ -25,6 +25,8 @@
     parse_jsonl_log_messages,
     stream_dict_list_as_csv,
     validate_regexes,
+    scale_from_browser_windows,
+    browser_windows_from_scale,
 )
 from .basecrawls import BaseCrawlOps
 from .crawlmanager import CrawlManager
@@ -368,7 +370,8 @@ async def add_new_crawl(
             oid=crawlconfig.oid,
             cid=crawlconfig.id,
             cid_rev=crawlconfig.rev,
-            scale=crawlconfig.scale,
+            scale=scale_from_browser_windows(crawlconfig.browserWindows),
+            browserWindows=crawlconfig.browserWindows,
             jobType=crawlconfig.jobType,
             config=crawlconfig.config,
             profileid=crawlconfig.profileid,
@@ -392,16 +395,27 @@ async def add_new_crawl(
             pass
 
     async def update_crawl_scale(
-        self, crawl_id: str, org: Organization, crawl_scale: CrawlScale, user: User
+        self,
+        crawl_id: str,
+        org: Organization,
+        scale: int,
+        browser_windows: int,
+        user: User,
     ) -> bool:
         """Update crawl scale in the db"""
         crawl = await self.get_crawl(crawl_id, org)
-        update = UpdateCrawlConfig(scale=crawl_scale.scale)
+
+        update = UpdateCrawlConfig(browserWindows=browser_windows)
         await self.crawl_configs.update_crawl_config(crawl.cid, org, user, update)
 
         result = await self.crawls.find_one_and_update(
             {"_id": crawl_id, "type": "crawl", "oid": org.id},
-            {"$set": {"scale": crawl_scale.scale}},
+            {
+                "$set": {
+                    "scale": scale,
+                    "browserWindows": browser_windows,
+                }
+            },
             return_document=pymongo.ReturnDocument.AFTER,
         )
 
@@ -529,7 +543,7 @@ async def add_or_remove_exclusion(
 
         cid = crawl.cid
 
-        scale = crawl.scale or 1
+        browser_windows = crawl.browserWindows or 2
 
         async with self.get_redis(crawl_id) as redis:
             query = {
@@ -538,6 +552,7 @@ async def add_or_remove_exclusion(
             }
             query_str = json.dumps(query)
 
+            scale = scale_from_browser_windows(browser_windows)
             for i in range(0, scale):
                 await redis.rpush(f"crawl-{crawl_id}-{i}:msg", query_str)
 
@@ -1524,20 +1539,31 @@ async def update_crawl_api(
         response_model=CrawlScaleResponse,
     )
     async def scale_crawl(
-        scale: CrawlScale,
+        crawl_scale: CrawlScale,
         crawl_id,
         user: User = Depends(user_dep),
         org: Organization = Depends(org_crawl_dep),
     ):
-        await ops.update_crawl_scale(crawl_id, org, scale, user)
+        if crawl_scale.browserWindows:
+            browser_windows = crawl_scale.browserWindows
+            scale = scale_from_browser_windows(browser_windows)
+        elif crawl_scale.scale:
+            scale = crawl_scale.scale
+            browser_windows = browser_windows_from_scale(scale)
+        else:
+            raise HTTPException(
+                status_code=400, detail="browser_windows_or_scale_required"
+            )
+
+        await ops.update_crawl_scale(crawl_id, org, scale, browser_windows, user)
 
-        result = await ops.crawl_manager.scale_crawl(crawl_id, scale.scale)
+        result = await ops.crawl_manager.scale_crawl(crawl_id, scale, browser_windows)
         if not result or not result.get("success"):
             raise HTTPException(
                 status_code=400, detail=result.get("error") or "unknown"
             )
 
-        return {"scaled": scale.scale}
+        return {"scaled": True, "browserWindows": browser_windows}
 
     @app.get(
         "/orgs/{oid}/crawls/{crawl_id}/access",

backend/btrixcloud/db.py

@@ -32,7 +32,7 @@
     ) = PageOps = BackgroundJobOps = object
 
 
-CURR_DB_VERSION = "0046"
+CURR_DB_VERSION = "0047"
 
 
 # ============================================================================

backend/btrixcloud/k8sapi.py

@@ -85,6 +85,7 @@ def new_crawl_job_yaml(
         storage: str,
         crawler_channel: Optional[str] = "",
         scale: Optional[int] = 1,
+        browser_windows: Optional[int] = 1,
         crawl_timeout: Optional[int] = 0,
         max_crawl_size: Optional[int] = 0,
         manual: bool = True,
@@ -109,6 +110,7 @@ def new_crawl_job_yaml(
             "storage_name": storage,
             "crawler_channel": crawler_channel,
             "scale": scale,
+            "browser_windows": browser_windows,
             "timeout": crawl_timeout,
             "max_crawl_size": max_crawl_size or 0,
             "manual": "1" if manual else "0",
@@ -130,6 +132,7 @@ async def new_crawl_job(
         storage: str,
         crawler_channel: Optional[str] = "",
         scale: Optional[int] = 1,
+        browser_windows: Optional[int] = 1,
         crawl_timeout: Optional[int] = 0,
         max_crawl_size: Optional[int] = 0,
         manual: bool = True,
@@ -148,6 +151,7 @@ async def new_crawl_job(
             storage=storage,
             crawler_channel=crawler_channel,
             scale=scale,
+            browser_windows=browser_windows,
             crawl_timeout=crawl_timeout,
             max_crawl_size=max_crawl_size,
             manual=manual,

backend/btrixcloud/main.py

@@ -112,8 +112,8 @@ class SettingsResponse(BaseModel):
     defaultPageLoadTimeSeconds: int
 
     maxPagesPerCrawl: int
-    numBrowsers: int
-    maxScale: int
+    numBrowsersPerInstance: int
+    maxBrowserWindows: int
 
     billingEnabled: bool
 
@@ -149,8 +149,8 @@ def main() -> None:
             os.environ.get("DEFAULT_PAGE_LOAD_TIME_SECONDS", 120)
         ),
         maxPagesPerCrawl=int(os.environ.get("MAX_PAGES_PER_CRAWL", 0)),
-        numBrowsers=int(os.environ.get("NUM_BROWSERS", 1)),
-        maxScale=int(os.environ.get("MAX_CRAWL_SCALE", 3)),
+        numBrowsersPerInstance=int(os.environ.get("NUM_BROWSERS", 1)),
+        maxBrowserWindows=int(os.environ.get("MAX_BROWSER_WINDOWS", 8)),
         billingEnabled=is_bool(os.environ.get("BILLING_ENABLED")),
         signUpUrl=os.environ.get("SIGN_UP_URL", ""),
         salesEmail=os.environ.get("SALES_EMAIL", ""),

backend/btrixcloud/migrations/migration_0005_operator_scheduled_jobs.py

@@ -43,7 +43,7 @@ async def migrate_up(self):
             config = CrawlConfig.from_dict(config_dict)
             print(
                 f"Updating Crawl Config {config.id}: schedule: {config.schedule}, "
-                + f"timeout: {config.crawlTimeout}, scale: {config.scale}"
+                + f"timeout: {config.crawlTimeout}"
             )
             try:
                 await crawl_manager.update_scheduled_job(config)

backend/btrixcloud/migrations/migration_0047_scale_to_browser_windows.py

@@ -0,0 +1,62 @@
+"""
+Migration 0047 - Convert scale to browserWindows
+"""
+
+from btrixcloud.migrations import BaseMigration
+from btrixcloud.utils import browser_windows_from_scale
+
+
+MIGRATION_VERSION = "0047"
+
+
+# pylint: disable=duplicate-code
+class Migration(BaseMigration):
+    """Migration class."""
+
+    # pylint: disable=unused-argument
+    def __init__(self, mdb, **kwargs):
+        super().__init__(mdb, migration_version=MIGRATION_VERSION)
+
+    async def migrate_up(self):
+        """Perform migration up.
+
+        Calculate and store browserWindows from existing scale on workflows and crawls
+        """
+        configs_mdb = self.mdb["crawl_configs"]
+        crawls_mdb = self.mdb["crawls"]
+
+        async for config_raw in configs_mdb.find({"browserWindows": None}):
+            config_id = config_raw["_id"]
+            scale = config_raw.get("scale", 1)
+
+            try:
+                await configs_mdb.find_one_and_update(
+                    {"_id": config_id},
+                    {
+                        "$set": {"browserWindows": browser_windows_from_scale(scale)},
+                    },
+                )
+            # pylint: disable=broad-exception-caught
+            except Exception as err:
+                print(
+                    f"Unable to set browser windows from scale for workflow {config_id}: {err}",
+                    flush=True,
+                )
+
+        async for crawl_raw in crawls_mdb.find({"browserWindows": None}):
+            crawl_id = crawl_raw["_id"]
+            scale = crawl_raw.get("scale", 1)
+
+            try:
+                await crawls_mdb.find_one_and_update(
+                    {"_id": crawl_id},
+                    {
+                        "$set": {"browserWindows": browser_windows_from_scale(scale)},
+                    },
+                )
+            # pylint: disable=broad-exception-caught
+            except Exception as err:
+                print(
+                    f"Unable to set browser windows from scale for crawl {crawl_id}: {err}",
+                    flush=True,
+                )