From 1ab6ec325b124512f48ad65b895d557c87d27d0b Mon Sep 17 00:00:00 2001 From: Ilya Kreymer Date: Tue, 11 Jun 2024 16:28:40 -0700 Subject: [PATCH 1/7] version: bump to 1.10.2 --- backend/btrixcloud/version.py | 2 +- chart/Chart.yaml | 2 +- chart/values.yaml | 4 ++-- frontend/package.json | 2 +- version.txt | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/backend/btrixcloud/version.py b/backend/btrixcloud/version.py index c826faf14f..a54057875d 100644 --- a/backend/btrixcloud/version.py +++ b/backend/btrixcloud/version.py @@ -1,3 +1,3 @@ """ current version """ -__version__ = "1.10.1" +__version__ = "1.10.2" diff --git a/chart/Chart.yaml b/chart/Chart.yaml index a33f2b05bd..e480a79dbf 100644 --- a/chart/Chart.yaml +++ b/chart/Chart.yaml @@ -5,7 +5,7 @@ type: application icon: https://webrecorder.net/assets/icon.png # Browsertrix and Chart Version -version: v1.10.1 +version: v1.10.2 dependencies: - name: btrix-admin-logging diff --git a/chart/values.yaml b/chart/values.yaml index 14cb21b917..c58f25a56a 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -90,7 +90,7 @@ default_org: "My Organization" # API Image # ========================================= -backend_image: "docker.io/webrecorder/browsertrix-backend:1.10.1" +backend_image: "docker.io/webrecorder/browsertrix-backend:1.10.2" backend_pull_policy: "Always" backend_password_secret: "PASSWORD!" @@ -129,7 +129,7 @@ backend_avg_memory_threshold: 95 # Nginx Image # ========================================= -frontend_image: "docker.io/webrecorder/browsertrix-frontend:1.10.1" +frontend_image: "docker.io/webrecorder/browsertrix-frontend:1.10.2" frontend_pull_policy: "Always" frontend_cpu: "10m" diff --git a/frontend/package.json b/frontend/package.json index 4b9eb12980..0cd10dc69c 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "browsertrix-frontend", - "version": "1.10.1", + "version": "1.10.2", "main": "index.ts", "license": "AGPL-3.0-or-later", "dependencies": { diff --git a/version.txt b/version.txt index 4dae2985b5..5ad2491cf8 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.10.1 +1.10.2 From d501fb3fe75c119f5e6a07489ce75b0db8dc9092 Mon Sep 17 00:00:00 2001 From: sua yoo Date: Wed, 12 Jun 2024 09:50:19 -0500 Subject: [PATCH 2/7] fix: Correct progress ring for analysis run in Chrome (#1860) Switches `sl-progress-ring` and `sl-icon` to inherit font size from table cell for consistent sizing as base size, while also fixing `var(--font-size-base)` not being available when progress ring size is first calculated. Fixes #1835 --- .../archived-items/archived-item-list.ts | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/frontend/src/features/archived-items/archived-item-list.ts b/frontend/src/features/archived-items/archived-item-list.ts index 2f43a8b5d5..0bc397b30e 100644 --- a/frontend/src/features/archived-items/archived-item-list.ts +++ b/frontend/src/features/archived-items/archived-item-list.ts @@ -45,6 +45,14 @@ export class ArchivedItemListItem extends TailwindElement { var(--btrix-border-radius-bottom, 0); height: 2.5rem; } + + sl-progress-ring { + /* Setting size to var(--font-size-base) breaks in chrome, + have cell contents inherit size from cell instead */ + --size: 1em; + --track-width: 1px; + --indicator-width: 2px; + } `; @property({ type: Object, attribute: false }) @@ -141,7 +149,7 @@ export class ArchivedItemListItem extends TailwindElement { ` : nothing} - + ${this.showStatus ? html` ` : html` Date: Wed, 12 Jun 2024 11:46:15 -0400 Subject: [PATCH 3/7] Reorder sections to match workflow creator (#1862) --- frontend/src/components/ui/config-details.ts | 40 ++++++++++++-------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/frontend/src/components/ui/config-details.ts b/frontend/src/components/ui/config-details.ts index 35158b4ee8..6c8a3e9378 100644 --- a/frontend/src/components/ui/config-details.ts +++ b/frontend/src/components/ui/config-details.ts @@ -141,6 +141,9 @@ export class ConfigDetails extends LiteElement { `, () => this.renderSetting(msg("Exclusions"), msg("None")), )} + +

${msg("Per-Crawl Limits")}

+ ${this.renderSetting( msg("Max Pages"), when( @@ -157,6 +160,21 @@ export class ConfigDetails extends LiteElement { : undefined, ), )} + ${this.renderSetting( + msg("Crawl Time Limit"), + renderTimeLimit(crawlConfig?.crawlTimeout, Infinity), + )} + ${this.renderSetting( + msg("Crawl Size Limit"), + renderSize(crawlConfig?.maxCrawlSize), + )} + ${this.renderSetting( + msg("Crawler Instances"), + crawlConfig?.scale ? `${crawlConfig.scale}×` : "", + )} + +

${msg("Per-Page Limits")}

+ ${this.renderSetting( msg("Page Load Timeout"), renderTimeLimit( @@ -169,7 +187,7 @@ export class ConfigDetails extends LiteElement { renderTimeLimit(crawlConfig?.config.postLoadDelay, 0), )} ${this.renderSetting( - msg("Page Behavior Timeout"), + msg("Behavior Timeout"), renderTimeLimit( crawlConfig?.config.behaviorTimeout, this.orgDefaults?.behaviorTimeoutSeconds ?? Infinity, @@ -188,18 +206,6 @@ export class ConfigDetails extends LiteElement { msg("Delay Before Next Page"), renderTimeLimit(crawlConfig?.config.pageExtraDelay, 0), )} - ${this.renderSetting( - msg("Crawl Time Limit"), - renderTimeLimit(crawlConfig?.crawlTimeout, Infinity), - )} - ${this.renderSetting( - msg("Crawl Size Limit"), - renderSize(crawlConfig?.maxCrawlSize), - )} - ${this.renderSetting( - msg("Crawler Instances"), - crawlConfig?.scale ? `${crawlConfig.scale}×` : "", - )}
@@ -221,7 +227,11 @@ export class ConfigDetails extends LiteElement { > ${crawlConfig?.profileName} `, - () => crawlConfig?.profileName || msg("Default Profile"), + () => + crawlConfig?.profileName || + html`${msg("Default Profile")}`, ), )} ${this.renderSetting( @@ -237,7 +247,7 @@ export class ConfigDetails extends LiteElement { msg("User Agent"), crawlConfig?.config.userAgent ? crawlConfig.config.userAgent - : msg("Default User Agent"), + : html`${msg("Default")}`, )} ${crawlConfig?.config.lang ? this.renderSetting( From 9d69001c237c74e4a01fde8249553e46ce9ca597 Mon Sep 17 00:00:00 2001 From: sua yoo Date: Wed, 12 Jun 2024 11:20:49 -0500 Subject: [PATCH 4/7] fix: Prevent ReplayWeb.Page shoelace components from impacting browsertrix (#1861) Resolves https://github.com/webrecorder/browsertrix/issues/1836 Related to https://github.com/webrecorder/replayweb.page/pull/329 ### Changes Loads `replay/ui.js` after the browsertrix-app component is defined to prevent browsertrix from using buttons defined by replayweb.page shoelace imports. This was the most evident in button group styles not being applied. --- frontend/package.json | 2 +- frontend/src/index.ejs | 11 ++++++++++- frontend/yarn.lock | 8 ++++---- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/frontend/package.json b/frontend/package.json index 0cd10dc69c..4f195bb4e7 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -10,7 +10,7 @@ "@lit/task": "^1.0.0", "@novnc/novnc": "^1.4.0-beta", "@rollup/plugin-commonjs": "^18.0.0", - "@shoelace-style/shoelace": "~2.15.0", + "@shoelace-style/shoelace": "~2.15.1", "@tailwindcss/container-queries": "^0.1.1", "@types/color": "^3.0.2", "@types/diff": "^5.0.9", diff --git a/frontend/src/index.ejs b/frontend/src/index.ejs index d3e21b6b76..27d20362b7 100644 --- a/frontend/src/index.ejs +++ b/frontend/src/index.ejs @@ -8,7 +8,7 @@ /> Browsertrix - + diff --git a/frontend/yarn.lock b/frontend/yarn.lock index cf2bed85fa..06ec59a74c 100644 --- a/frontend/yarn.lock +++ b/frontend/yarn.lock @@ -1135,10 +1135,10 @@ resolved "https://registry.yarnpkg.com/@shoelace-style/localize/-/localize-3.1.2.tgz#2c63f16d8aa80842dbe5127845c76ed53f6a5e8e" integrity sha512-Hf45HeO+vdQblabpyZOTxJ4ZeZsmIUYXXPmoYrrR4OJ5OKxL+bhMz5mK8JXgl7HsoEowfz7+e248UGi861de9Q== -"@shoelace-style/shoelace@~2.15.0": - version "2.15.0" - resolved "https://registry.yarnpkg.com/@shoelace-style/shoelace/-/shoelace-2.15.0.tgz#3410d6bb50811fad001b2c2fbd455cb60d6341a9" - integrity sha512-Lcg938Y8U2VsHqIYewzlt+H1rbrXC4GRSUkTJgXyF8/0YAOlI+srd5OSfIw+/LYmwLP2Peyh398Kae/6tg4PDA== +"@shoelace-style/shoelace@~2.15.1": + version "2.15.1" + resolved "https://registry.yarnpkg.com/@shoelace-style/shoelace/-/shoelace-2.15.1.tgz#2fa6bd8e493801f5b5b4744fab0fa108bbc01934" + integrity sha512-3ecUw8gRwOtcZQ8kWWkjk4FTfObYQ/XIl3aRhxprESoOYV1cYhloYPsmQY38UoL3+pwJiZb5+LzX0l3u3Zl0GA== dependencies: "@ctrl/tinycolor" "^4.0.2" "@floating-ui/dom" "^1.5.3" From fa6627ce7069983c62c2562ecb65e7de4f82be8f Mon Sep 17 00:00:00 2001 From: Ilya Kreymer Date: Wed, 12 Jun 2024 10:51:35 -0700 Subject: [PATCH 5/7] ensure QA configmap is updated for long running QA runs: (#1865) - add a 'expire_at_duration_seconds' which is 75% of actual presign duration time, or <25% remaining until presigned URL actually expires to ensure presigned URLs are updated early than when they actually expire - set cached expireAt time to the renew at time for more frequent updates - update QA configmap in place with updated presigned URLs when expireAt time is reached - mount qa config volume under /tmp/qa/ without subPath to get automatic updates, which crawler will handle - tests: fix qa test typo (from main) - fixes #1864 --- backend/btrixcloud/basecrawls.py | 8 ++++++-- backend/btrixcloud/operator/crawls.py | 27 +++++++++++++++++++++++++-- backend/test/test_qa.py | 2 +- chart/app-templates/crawler.yaml | 5 ++--- chart/templates/operators.yaml | 2 +- 5 files changed, 35 insertions(+), 9 deletions(-) diff --git a/backend/btrixcloud/basecrawls.py b/backend/btrixcloud/basecrawls.py index cf03bdeafa..5aec6be483 100644 --- a/backend/btrixcloud/basecrawls.py +++ b/backend/btrixcloud/basecrawls.py @@ -64,7 +64,8 @@ class BaseCrawlOps: background_job_ops: BackgroundJobOps page_ops: PageOps - presign_duration: int + presign_duration_seconds: int + expire_at_duration_seconds: int def __init__( self, @@ -95,6 +96,9 @@ def __init__( min(presign_duration_minutes, PRESIGN_MINUTES_MAX) * 60 ) + # renew when <25% of time remaining + self.expire_at_duration_seconds = int(self.presign_duration_seconds * 0.75) + def set_page_ops(self, page_ops): """set page ops reference""" self.page_ops = page_ops @@ -434,7 +438,7 @@ async def _resolve_signed_urls( print("no files") return [] - delta = timedelta(seconds=self.presign_duration_seconds) + delta = timedelta(seconds=self.expire_at_duration_seconds) out_files = [] diff --git a/backend/btrixcloud/operator/crawls.py b/backend/btrixcloud/operator/crawls.py index 9962c827e8..e6eee4df65 100644 --- a/backend/btrixcloud/operator/crawls.py +++ b/backend/btrixcloud/operator/crawls.py @@ -325,8 +325,15 @@ async def _load_qa_configmap(self, params, children): qa_source_crawl_id = params["qa_source_crawl_id"] name = f"qa-replay-{qa_source_crawl_id}" - if name in children[CMAP]: - return [children[CMAP][name]] + configmap = children[CMAP].get(name) + if configmap and not self._qa_configmap_update_needed(name, configmap): + metadata = configmap["metadata"] + configmap["metadata"] = { + "name": metadata["name"], + "namespace": metadata["namespace"], + "labels": metadata["labels"], + } + return [configmap] crawl_replay = await self.crawl_ops.get_internal_crawl_out(qa_source_crawl_id) @@ -364,6 +371,22 @@ def _load_crawler(self, params, i, status, children): return self.load_from_yaml("crawler.yaml", params) + def _qa_configmap_update_needed(self, name, configmap): + try: + now = dt_now() + resources = json.loads(configmap["data"]["qa-config.json"])["resources"] + for resource in resources: + expire_at = datetime.fromisoformat(resource["expireAt"]) + if expire_at <= now: + print(f"Refreshing QA configmap for QA run: {name}") + return True + + # pylint: disable=broad-exception-caught + except Exception as e: + print(e) + + return False + # pylint: disable=too-many-arguments async def _resolve_scale( self, diff --git a/backend/test/test_qa.py b/backend/test/test_qa.py index 85f09e1dfd..e9861b313c 100644 --- a/backend/test/test_qa.py +++ b/backend/test/test_qa.py @@ -383,7 +383,7 @@ def test_run_qa_not_running( count = 0 while count < MAX_ATTEMPTS: r = requests.get( - f"{API_PREFIX}/orgs/{default_org_id}/crawls/{crawler_crawl_id}/activeQA", + f"{API_PREFIX}/orgs/{default_org_id}/crawls/{crawler_crawl_id}/qa/activeQA", headers=crawler_auth_headers, ) data = r.json() diff --git a/chart/app-templates/crawler.yaml b/chart/app-templates/crawler.yaml index acd6e1b9b5..6dc618d069 100644 --- a/chart/app-templates/crawler.yaml +++ b/chart/app-templates/crawler.yaml @@ -124,7 +124,7 @@ spec: - {{ redis_url }} {% if qa_source_crawl_id %} - --qaSource - - /tmp/qa-config.json + - /tmp/qa/qa-config.json {% elif profile_filename %} - --profile - "@{{ profile_filename }}" @@ -137,8 +137,7 @@ spec: {% if qa_source_crawl_id %} - name: qa-config - mountPath: /tmp/qa-config.json - subPath: qa-config.json + mountPath: /tmp/qa/ readOnly: True {% endif %} diff --git a/chart/templates/operators.yaml b/chart/templates/operators.yaml index b7126edb4c..92e6a3a864 100644 --- a/chart/templates/operators.yaml +++ b/chart/templates/operators.yaml @@ -23,7 +23,7 @@ spec: - apiVersion: v1 resource: configmaps updateStrategy: - method: OnDelete + method: InPlace hooks: sync: From f8cde4bd763c569b2b4263710f35f78b070295a5 Mon Sep 17 00:00:00 2001 From: sua yoo Date: Wed, 12 Jun 2024 15:05:01 -0500 Subject: [PATCH 6/7] fix: Show correct max depth in settings (#1858) Fixes https://github.com/webrecorder/browsertrix/issues/1626 --- frontend/src/components/ui/config-details.ts | 7 ++++--- frontend/src/pages/org/workflow-editor.ts | 4 +--- frontend/src/utils/crawler.ts | 8 ++++++++ 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/frontend/src/components/ui/config-details.ts b/frontend/src/components/ui/config-details.ts index 6c8a3e9378..298d9ff5ad 100644 --- a/frontend/src/components/ui/config-details.ts +++ b/frontend/src/components/ui/config-details.ts @@ -13,6 +13,7 @@ import type { CrawlConfig, Seed, SeedConfig } from "@/pages/org/types"; import type { Collection } from "@/types/collection"; import { isApiError } from "@/utils/api"; import type { AuthState } from "@/utils/AuthService"; +import { DEPTH_SUPPORTED_SCOPES } from "@/utils/crawler"; import { humanizeSchedule } from "@/utils/cron"; import LiteElement, { html } from "@/utils/LiteElement"; @@ -406,15 +407,15 @@ export class ConfigDetails extends LiteElement { true, )} ${when( - ["host", "domain", "custom", "any"].includes( + DEPTH_SUPPORTED_SCOPES.includes( primarySeedConfig!.scopeType || seedsConfig.scopeType!, ), () => this.renderSetting( msg("Max Depth"), - primarySeedConfig?.depth + primarySeedConfig && primarySeedConfig.depth !== null ? msg(str`${primarySeedConfig.depth} hop(s)`) - : msg("None"), + : msg("Unlimited (default)"), ), )} ${this.renderSetting( diff --git a/frontend/src/pages/org/workflow-editor.ts b/frontend/src/pages/org/workflow-editor.ts index ddd25b3eb9..e2500180f3 100644 --- a/frontend/src/pages/org/workflow-editor.ts +++ b/frontend/src/pages/org/workflow-editor.ts @@ -57,7 +57,7 @@ import type { } from "@/features/crawl-workflows/queue-exclusion-table"; import { isApiError, type Detail } from "@/utils/api"; import type { AuthState } from "@/utils/AuthService"; -import { DEFAULT_MAX_SCALE } from "@/utils/crawler"; +import { DEFAULT_MAX_SCALE, DEPTH_SUPPORTED_SCOPES } from "@/utils/crawler"; import { getNextDate, getScheduleInterval, @@ -135,8 +135,6 @@ type FormState = { crawlerChannel: string; }; -const DEPTH_SUPPORTED_SCOPES = ["prefix", "host", "domain", "custom", "any"]; - const getDefaultProgressState = (hasConfigId = false): ProgressState => { let activeTab: StepName = "crawlSetup"; if (window.location.hash) { diff --git a/frontend/src/utils/crawler.ts b/frontend/src/utils/crawler.ts index dcf84e06af..dfdf5204bc 100644 --- a/frontend/src/utils/crawler.ts +++ b/frontend/src/utils/crawler.ts @@ -31,6 +31,14 @@ export const inactiveCrawlStates: CrawlState[] = [ export const DEFAULT_MAX_SCALE = 3; +export const DEPTH_SUPPORTED_SCOPES = [ + "prefix", + "host", + "domain", + "custom", + "any", +]; + export function isActive(state: CrawlState | null) { return state && activeCrawlStates.includes(state); } From dc9069d1bfc02d195b43c19f0d2b40ce15b567f9 Mon Sep 17 00:00:00 2001 From: Henry Wilkinson Date: Wed, 12 Jun 2024 21:32:43 -0400 Subject: [PATCH 7/7] Improves accessibility for QA features (#1863) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Switches from spans with classes to `ins` and `del` tags for text comparison - Adds `aria-label` attributes for screen readers. Text comparison is accessible now! 🎉 - I kept the strikethrough text on the `del` tag but can remove? Only visible on hover. - Removes autofocus on the delete analysis run cancel button - Adds `aria-label` attribute for the heuristic nav bar to describe the context of the content users will be seeing on each page. --- .../pages/org/archived-item-detail/ui/qa.ts | 1 - .../org/archived-item-qa/archived-item-qa.ts | 1 + .../src/pages/org/archived-item-qa/ui/text.ts | 52 ++++++++++++------- 3 files changed, 34 insertions(+), 20 deletions(-) diff --git a/frontend/src/pages/org/archived-item-detail/ui/qa.ts b/frontend/src/pages/org/archived-item-detail/ui/qa.ts index af0559db2d..24e5f38d44 100644 --- a/frontend/src/pages/org/archived-item-detail/ui/qa.ts +++ b/frontend/src/pages/org/archived-item-detail/ui/qa.ts @@ -425,7 +425,6 @@ export class ArchivedItemDetailQA extends TailwindElement {
void this.deleteQADialog?.hide()} > ${msg("Cancel")} diff --git a/frontend/src/pages/org/archived-item-qa/archived-item-qa.ts b/frontend/src/pages/org/archived-item-qa/archived-item-qa.ts index a5ea3936db..1eb2f19e01 100644 --- a/frontend/src/pages/org/archived-item-qa/archived-item-qa.ts +++ b/frontend/src/pages/org/archived-item-qa/archived-item-qa.ts @@ -508,6 +508,7 @@ export class ArchivedItemQA extends TailwindElement {
${diff.map((part) => { - return html` - ${part.value}`; + } else if (part.removed) { + return html`${part.value}`; + } else { + return html`${part.value} - `; + >`; + } })}
`;