diff --git a/backend/btrixcloud/basecrawls.py b/backend/btrixcloud/basecrawls.py index cf03bdeafa..5aec6be483 100644 --- a/backend/btrixcloud/basecrawls.py +++ b/backend/btrixcloud/basecrawls.py @@ -64,7 +64,8 @@ class BaseCrawlOps: background_job_ops: BackgroundJobOps page_ops: PageOps - presign_duration: int + presign_duration_seconds: int + expire_at_duration_seconds: int def __init__( self, @@ -95,6 +96,9 @@ def __init__( min(presign_duration_minutes, PRESIGN_MINUTES_MAX) * 60 ) + # renew when <25% of time remaining + self.expire_at_duration_seconds = int(self.presign_duration_seconds * 0.75) + def set_page_ops(self, page_ops): """set page ops reference""" self.page_ops = page_ops @@ -434,7 +438,7 @@ async def _resolve_signed_urls( print("no files") return [] - delta = timedelta(seconds=self.presign_duration_seconds) + delta = timedelta(seconds=self.expire_at_duration_seconds) out_files = [] diff --git a/backend/btrixcloud/operator/crawls.py b/backend/btrixcloud/operator/crawls.py index 9962c827e8..e6eee4df65 100644 --- a/backend/btrixcloud/operator/crawls.py +++ b/backend/btrixcloud/operator/crawls.py @@ -325,8 +325,15 @@ async def _load_qa_configmap(self, params, children): qa_source_crawl_id = params["qa_source_crawl_id"] name = f"qa-replay-{qa_source_crawl_id}" - if name in children[CMAP]: - return [children[CMAP][name]] + configmap = children[CMAP].get(name) + if configmap and not self._qa_configmap_update_needed(name, configmap): + metadata = configmap["metadata"] + configmap["metadata"] = { + "name": metadata["name"], + "namespace": metadata["namespace"], + "labels": metadata["labels"], + } + return [configmap] crawl_replay = await self.crawl_ops.get_internal_crawl_out(qa_source_crawl_id) @@ -364,6 +371,22 @@ def _load_crawler(self, params, i, status, children): return self.load_from_yaml("crawler.yaml", params) + def _qa_configmap_update_needed(self, name, configmap): + try: + now = dt_now() + resources = json.loads(configmap["data"]["qa-config.json"])["resources"] + for resource in resources: + expire_at = datetime.fromisoformat(resource["expireAt"]) + if expire_at <= now: + print(f"Refreshing QA configmap for QA run: {name}") + return True + + # pylint: disable=broad-exception-caught + except Exception as e: + print(e) + + return False + # pylint: disable=too-many-arguments async def _resolve_scale( self, diff --git a/chart/app-templates/crawler.yaml b/chart/app-templates/crawler.yaml index acd6e1b9b5..6dc618d069 100644 --- a/chart/app-templates/crawler.yaml +++ b/chart/app-templates/crawler.yaml @@ -124,7 +124,7 @@ spec: - {{ redis_url }} {% if qa_source_crawl_id %} - --qaSource - - /tmp/qa-config.json + - /tmp/qa/qa-config.json {% elif profile_filename %} - --profile - "@{{ profile_filename }}" @@ -137,8 +137,7 @@ spec: {% if qa_source_crawl_id %} - name: qa-config - mountPath: /tmp/qa-config.json - subPath: qa-config.json + mountPath: /tmp/qa/ readOnly: True {% endif %} diff --git a/chart/templates/operators.yaml b/chart/templates/operators.yaml index b7126edb4c..92e6a3a864 100644 --- a/chart/templates/operators.yaml +++ b/chart/templates/operators.yaml @@ -23,7 +23,7 @@ spec: - apiVersion: v1 resource: configmaps updateStrategy: - method: OnDelete + method: InPlace hooks: sync: diff --git a/frontend/package.json b/frontend/package.json index f1c5a8c598..31fb6fe49a 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -10,7 +10,7 @@ "@lit/task": "^1.0.0", "@novnc/novnc": "^1.4.0-beta", "@rollup/plugin-commonjs": "^18.0.0", - "@shoelace-style/shoelace": "~2.15.0", + "@shoelace-style/shoelace": "~2.15.1", "@tailwindcss/container-queries": "^0.1.1", "@types/color": "^3.0.2", "@types/diff": "^5.0.9", diff --git a/frontend/src/components/ui/config-details.ts b/frontend/src/components/ui/config-details.ts index 35158b4ee8..298d9ff5ad 100644 --- a/frontend/src/components/ui/config-details.ts +++ b/frontend/src/components/ui/config-details.ts @@ -13,6 +13,7 @@ import type { CrawlConfig, Seed, SeedConfig } from "@/pages/org/types"; import type { Collection } from "@/types/collection"; import { isApiError } from "@/utils/api"; import type { AuthState } from "@/utils/AuthService"; +import { DEPTH_SUPPORTED_SCOPES } from "@/utils/crawler"; import { humanizeSchedule } from "@/utils/cron"; import LiteElement, { html } from "@/utils/LiteElement"; @@ -141,6 +142,9 @@ export class ConfigDetails extends LiteElement { `, () => this.renderSetting(msg("Exclusions"), msg("None")), )} + +

${msg("Per-Crawl Limits")}

+ ${this.renderSetting( msg("Max Pages"), when( @@ -157,6 +161,21 @@ export class ConfigDetails extends LiteElement { : undefined, ), )} + ${this.renderSetting( + msg("Crawl Time Limit"), + renderTimeLimit(crawlConfig?.crawlTimeout, Infinity), + )} + ${this.renderSetting( + msg("Crawl Size Limit"), + renderSize(crawlConfig?.maxCrawlSize), + )} + ${this.renderSetting( + msg("Crawler Instances"), + crawlConfig?.scale ? `${crawlConfig.scale}×` : "", + )} + +

${msg("Per-Page Limits")}

+ ${this.renderSetting( msg("Page Load Timeout"), renderTimeLimit( @@ -169,7 +188,7 @@ export class ConfigDetails extends LiteElement { renderTimeLimit(crawlConfig?.config.postLoadDelay, 0), )} ${this.renderSetting( - msg("Page Behavior Timeout"), + msg("Behavior Timeout"), renderTimeLimit( crawlConfig?.config.behaviorTimeout, this.orgDefaults?.behaviorTimeoutSeconds ?? Infinity, @@ -188,18 +207,6 @@ export class ConfigDetails extends LiteElement { msg("Delay Before Next Page"), renderTimeLimit(crawlConfig?.config.pageExtraDelay, 0), )} - ${this.renderSetting( - msg("Crawl Time Limit"), - renderTimeLimit(crawlConfig?.crawlTimeout, Infinity), - )} - ${this.renderSetting( - msg("Crawl Size Limit"), - renderSize(crawlConfig?.maxCrawlSize), - )} - ${this.renderSetting( - msg("Crawler Instances"), - crawlConfig?.scale ? `${crawlConfig.scale}×` : "", - )}
@@ -221,7 +228,11 @@ export class ConfigDetails extends LiteElement { > ${crawlConfig?.profileName} `, - () => crawlConfig?.profileName || msg("Default Profile"), + () => + crawlConfig?.profileName || + html`${msg("Default Profile")}`, ), )} ${this.renderSetting( @@ -237,7 +248,7 @@ export class ConfigDetails extends LiteElement { msg("User Agent"), crawlConfig?.config.userAgent ? crawlConfig.config.userAgent - : msg("Default User Agent"), + : html`${msg("Default")}`, )} ${crawlConfig?.config.lang ? this.renderSetting( @@ -396,15 +407,15 @@ export class ConfigDetails extends LiteElement { true, )} ${when( - ["host", "domain", "custom", "any"].includes( + DEPTH_SUPPORTED_SCOPES.includes( primarySeedConfig!.scopeType || seedsConfig.scopeType!, ), () => this.renderSetting( msg("Max Depth"), - primarySeedConfig?.depth + primarySeedConfig && primarySeedConfig.depth !== null ? msg(str`${primarySeedConfig.depth} hop(s)`) - : msg("None"), + : msg("Unlimited (default)"), ), )} ${this.renderSetting( diff --git a/frontend/src/features/archived-items/archived-item-list.ts b/frontend/src/features/archived-items/archived-item-list.ts index b00bfbc954..1750f0569d 100644 --- a/frontend/src/features/archived-items/archived-item-list.ts +++ b/frontend/src/features/archived-items/archived-item-list.ts @@ -45,6 +45,14 @@ export class ArchivedItemListItem extends TailwindElement { var(--btrix-border-radius-bottom, 0); height: 2.5rem; } + + sl-progress-ring { + /* Setting size to var(--font-size-base) breaks in chrome, + have cell contents inherit size from cell instead */ + --size: 1em; + --track-width: 1px; + --indicator-width: 2px; + } `; @property({ type: Object, attribute: false }) @@ -141,7 +149,7 @@ export class ArchivedItemListItem extends TailwindElement { ` : nothing} - + ${this.showStatus ? html` ` : html` Browsertrix - + diff --git a/frontend/src/pages/org/archived-item-detail/ui/qa.ts b/frontend/src/pages/org/archived-item-detail/ui/qa.ts index 25a5dd1cb0..3747ec9030 100644 --- a/frontend/src/pages/org/archived-item-detail/ui/qa.ts +++ b/frontend/src/pages/org/archived-item-detail/ui/qa.ts @@ -440,7 +440,6 @@ export class ArchivedItemDetailQA extends TailwindElement {
void this.deleteQADialog?.hide()} > ${msg("Cancel")} diff --git a/frontend/src/pages/org/archived-item-qa/archived-item-qa.ts b/frontend/src/pages/org/archived-item-qa/archived-item-qa.ts index a5ea3936db..1eb2f19e01 100644 --- a/frontend/src/pages/org/archived-item-qa/archived-item-qa.ts +++ b/frontend/src/pages/org/archived-item-qa/archived-item-qa.ts @@ -508,6 +508,7 @@ export class ArchivedItemQA extends TailwindElement {
${diff.map((part) => { - return html` - ${part.value}`; + } else if (part.removed) { + return html`${part.value}`; + } else { + return html`${part.value} - `; + >`; + } })}
`; diff --git a/frontend/src/pages/org/workflow-editor.ts b/frontend/src/pages/org/workflow-editor.ts index ddd25b3eb9..e2500180f3 100644 --- a/frontend/src/pages/org/workflow-editor.ts +++ b/frontend/src/pages/org/workflow-editor.ts @@ -57,7 +57,7 @@ import type { } from "@/features/crawl-workflows/queue-exclusion-table"; import { isApiError, type Detail } from "@/utils/api"; import type { AuthState } from "@/utils/AuthService"; -import { DEFAULT_MAX_SCALE } from "@/utils/crawler"; +import { DEFAULT_MAX_SCALE, DEPTH_SUPPORTED_SCOPES } from "@/utils/crawler"; import { getNextDate, getScheduleInterval, @@ -135,8 +135,6 @@ type FormState = { crawlerChannel: string; }; -const DEPTH_SUPPORTED_SCOPES = ["prefix", "host", "domain", "custom", "any"]; - const getDefaultProgressState = (hasConfigId = false): ProgressState => { let activeTab: StepName = "crawlSetup"; if (window.location.hash) { diff --git a/frontend/src/utils/crawler.ts b/frontend/src/utils/crawler.ts index dcf84e06af..dfdf5204bc 100644 --- a/frontend/src/utils/crawler.ts +++ b/frontend/src/utils/crawler.ts @@ -31,6 +31,14 @@ export const inactiveCrawlStates: CrawlState[] = [ export const DEFAULT_MAX_SCALE = 3; +export const DEPTH_SUPPORTED_SCOPES = [ + "prefix", + "host", + "domain", + "custom", + "any", +]; + export function isActive(state: CrawlState | null) { return state && activeCrawlStates.includes(state); } diff --git a/frontend/yarn.lock b/frontend/yarn.lock index cf2bed85fa..06ec59a74c 100644 --- a/frontend/yarn.lock +++ b/frontend/yarn.lock @@ -1135,10 +1135,10 @@ resolved "https://registry.yarnpkg.com/@shoelace-style/localize/-/localize-3.1.2.tgz#2c63f16d8aa80842dbe5127845c76ed53f6a5e8e" integrity sha512-Hf45HeO+vdQblabpyZOTxJ4ZeZsmIUYXXPmoYrrR4OJ5OKxL+bhMz5mK8JXgl7HsoEowfz7+e248UGi861de9Q== -"@shoelace-style/shoelace@~2.15.0": - version "2.15.0" - resolved "https://registry.yarnpkg.com/@shoelace-style/shoelace/-/shoelace-2.15.0.tgz#3410d6bb50811fad001b2c2fbd455cb60d6341a9" - integrity sha512-Lcg938Y8U2VsHqIYewzlt+H1rbrXC4GRSUkTJgXyF8/0YAOlI+srd5OSfIw+/LYmwLP2Peyh398Kae/6tg4PDA== +"@shoelace-style/shoelace@~2.15.1": + version "2.15.1" + resolved "https://registry.yarnpkg.com/@shoelace-style/shoelace/-/shoelace-2.15.1.tgz#2fa6bd8e493801f5b5b4744fab0fa108bbc01934" + integrity sha512-3ecUw8gRwOtcZQ8kWWkjk4FTfObYQ/XIl3aRhxprESoOYV1cYhloYPsmQY38UoL3+pwJiZb5+LzX0l3u3Zl0GA== dependencies: "@ctrl/tinycolor" "^4.0.2" "@floating-ui/dom" "^1.5.3"