From 9eec53cdbbaa8aea26d2ddf6a811c1fdb86f1654 Mon Sep 17 00:00:00 2001 From: sua yoo Date: Wed, 14 May 2025 10:53:13 -0700 Subject: [PATCH 01/34] switch to task --- frontend/src/pages/org/workflow-detail.ts | 29 +++++++++++++++++------ 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/frontend/src/pages/org/workflow-detail.ts b/frontend/src/pages/org/workflow-detail.ts index 2714cee192..17befe85f8 100644 --- a/frontend/src/pages/org/workflow-detail.ts +++ b/frontend/src/pages/org/workflow-detail.ts @@ -1,4 +1,5 @@ import { localized, msg, str } from "@lit/localize"; +import { Task, TaskStatus } from "@lit/task"; import type { SlSelect } from "@shoelace-style/shoelace"; import clsx from "clsx"; import { html, nothing, type PropertyValues, type TemplateResult } from "lit"; @@ -111,6 +112,15 @@ export class WorkflowDetail extends BtrixElement { private getWorkflowPromise?: Promise; private getSeedsPromise?: Promise>; + private readonly runNowTask = new Task(this, { + autoRun: false, + task: async (_args, { signal }) => { + await this.runNow({ signal }); + await this.fetchWorkflow(); + }, + args: () => [] as const, + }); + private get isExplicitRunning() { return ( this.workflow?.isCrawlRunning && @@ -656,7 +666,7 @@ export class WorkflowDetail extends BtrixElement { void this.runNow()} + @click=${() => void this.runNowTask.run()} > ${msg("Run Crawl")} @@ -1086,7 +1096,7 @@ export class WorkflowDetail extends BtrixElement { ${noData} @@ -1295,10 +1305,12 @@ export class WorkflowDetail extends BtrixElement { size="small" variant="primary" ?disabled=${this.org?.storageQuotaReached || - this.org?.execMinutesQuotaReached} - @click=${() => void this.runNow()} + this.org?.execMinutesQuotaReached || + this.runNowTask.status === TaskStatus.PENDING} + ?loading=${this.runNowTask.status === TaskStatus.PENDING} + @click=${() => void this.runNowTask.run()} > - + ${msg("Run Crawl")} @@ -1779,17 +1791,20 @@ export class WorkflowDetail extends BtrixElement { this.isCancelingOrStoppingCrawl = false; } - private async runNow(): Promise { + private async runNow({ + signal, + }: { signal?: AbortSignal } = {}): Promise { try { const data = await this.api.fetch<{ started: string | null }>( `/orgs/${this.orgId}/crawlconfigs/${this.workflowId}/run`, { method: "POST", + signal, }, ); this.lastCrawlId = data.started; this.lastCrawlStartTime = new Date().toISOString(); - void this.fetchWorkflow(); + this.navigate.to(`${this.basePath}/${WorkflowTab.LatestCrawl}`); this.notify.toast({ From 1a21ec235d97d1a34009e0b6fe5809f1d22e8541 Mon Sep 17 00:00:00 2001 From: sua yoo Date: Wed, 14 May 2025 15:32:08 -0700 Subject: [PATCH 02/34] add pause notice --- frontend/src/components/ui/alert.ts | 48 +++++---- frontend/src/components/ui/badge.ts | 25 ++--- .../src/features/admin/super-admin-banner.ts | 4 +- .../src/features/archived-items/crawl-logs.ts | 1 + .../crawl-workflows/workflow-editor.ts | 2 +- frontend/src/layouts/pageSectionsWithNav.ts | 2 +- frontend/src/pages/org/workflow-detail.ts | 99 ++++++++++++++++++- frontend/src/types/crawler.ts | 2 + 8 files changed, 149 insertions(+), 34 deletions(-) diff --git a/frontend/src/components/ui/alert.ts b/frontend/src/components/ui/alert.ts index b6f3a34687..fa82379eca 100644 --- a/frontend/src/components/ui/alert.ts +++ b/frontend/src/components/ui/alert.ts @@ -1,6 +1,6 @@ import clsx from "clsx"; import { css, html } from "lit"; -import { customElement, property } from "lit/decorators.js"; +import { customElement, property, state } from "lit/decorators.js"; import { TailwindElement } from "@/classes/TailwindElement"; @@ -18,28 +18,42 @@ export class Alert extends TailwindElement { @property({ type: String }) variant: "success" | "warning" | "danger" | "info" = "info"; + @state() + open = true; + static styles = css` :host { display: block; } `; + public hide() { + // TODO Animate for nicer transition + this.open = false; + } + + public show() { + // TODO Animate for nicer transition + this.open = true; + } + render() { - return html` - - `; + if (!this.open) return; + + return html``; } } diff --git a/frontend/src/components/ui/badge.ts b/frontend/src/components/ui/badge.ts index 0b92966018..c66fc6c7dc 100644 --- a/frontend/src/components/ui/badge.ts +++ b/frontend/src/components/ui/badge.ts @@ -1,3 +1,4 @@ +import clsx from "clsx"; import { css, html } from "lit"; import { customElement, property } from "lit/decorators.js"; @@ -38,17 +39,19 @@ export class Badge extends TailwindElement { render() { return html` diff --git a/frontend/src/features/admin/super-admin-banner.ts b/frontend/src/features/admin/super-admin-banner.ts index 2b3929a345..5c47efd00f 100644 --- a/frontend/src/features/admin/super-admin-banner.ts +++ b/frontend/src/features/admin/super-admin-banner.ts @@ -22,7 +22,7 @@ export class SuperAdminBanner extends TailwindElement { ${msg("You are logged in as a superadmin")} – ${msg("please be careful.")} -
+
+ `, + })} `; } } diff --git a/frontend/src/features/archived-items/crawl-status.ts b/frontend/src/features/archived-items/crawl-status.ts index 1025f26812..450fdf2f3b 100644 --- a/frontend/src/features/archived-items/crawl-status.ts +++ b/frontend/src/features/archived-items/crawl-status.ts @@ -331,12 +331,12 @@ export class CrawlStatus extends TailwindElement {
`; } if (label) { - return html`
+ return html`
${icon}
${label}
`; } - return html`
+ return html`
${icon}
`; } diff --git a/frontend/src/layouts/pageError.ts b/frontend/src/layouts/pageError.ts new file mode 100644 index 0000000000..2dfeef5218 --- /dev/null +++ b/frontend/src/layouts/pageError.ts @@ -0,0 +1,31 @@ +import { html, nothing, type TemplateResult } from "lit"; + +/** + * Render a full page error, like 404 or 500 for primary resources. + */ +export function pageError({ + heading, + detail, + primaryAction, + secondaryAction, +}: { + heading: string | TemplateResult; + detail: string | TemplateResult; + primaryAction: TemplateResult; + secondaryAction?: TemplateResult; +}) { + return html` +
+

+ ${heading} +

+

${detail}

+
${primaryAction}
+ ${secondaryAction + ? html`

${secondaryAction}

` + : nothing} +
+ `; +} diff --git a/frontend/src/pages/org/workflow-detail.ts b/frontend/src/pages/org/workflow-detail.ts index 4e86d9caab..eeaefaa39b 100644 --- a/frontend/src/pages/org/workflow-detail.ts +++ b/frontend/src/pages/org/workflow-detail.ts @@ -6,6 +6,7 @@ import { html, nothing, type PropertyValues, type TemplateResult } from "lit"; import { customElement, property, query, state } from "lit/decorators.js"; import { choose } from "lit/directives/choose.js"; import { ifDefined } from "lit/directives/if-defined.js"; +import { until } from "lit/directives/until.js"; import { when } from "lit/directives/when.js"; import queryString from "query-string"; @@ -16,6 +17,7 @@ import type { Alert } from "@/components/ui/alert"; import { ClipboardController } from "@/controllers/clipboard"; import { CrawlStatus } from "@/features/archived-items/crawl-status"; import { ExclusionEditor } from "@/features/crawl-workflows/exclusion-editor"; +import { pageError } from "@/layouts/pageError"; import { pageNav, type Breadcrumb } from "@/layouts/pageHeader"; import { WorkflowTab } from "@/routes"; import { tooltipFor } from "@/strings/archived-items/tooltips"; @@ -35,6 +37,8 @@ import { tw } from "@/utils/tailwind"; const POLL_INTERVAL_SECONDS = 10; +const isLoading = (task: Task) => task.status === TaskStatus.PENDING; + /** * Usage: * ```ts @@ -48,7 +52,7 @@ export class WorkflowDetail extends BtrixElement { workflowId!: string; @property({ type: String }) - workflowTab = WorkflowTab.LatestCrawl; + workflowTab?: WorkflowTab; @property({ type: Boolean }) isEditing = false; @@ -69,62 +73,212 @@ export class WorkflowDetail extends BtrixElement { maxScale = DEFAULT_MAX_SCALE; @state() - private workflow?: Workflow; + private lastCrawlId: Workflow["lastCrawlId"] = null; @state() - private seeds?: APIPaginatedList; + private isDialogVisible = false; @state() - private crawls?: APIPaginatedList; // Only inactive crawls + private crawlToDelete: Crawl | null = null; @state() - private lastCrawlId: Workflow["lastCrawlId"] = null; + private filterBy: Partial> = {}; - @state() - private lastCrawlStartTime: Workflow["lastCrawlStartTime"] = null; + @query("#pausedNotice") + private readonly pausedNotice?: Alert | null; - @state() - private lastCrawl?: Pick; + // Keep previous values to use when editing + private readonly prevValues: { + workflow?: Awaited>; + latestCrawl?: Awaited>; + logTotals?: Awaited>; + seeds?: APIPaginatedList; + } = {}; - @state() - private logTotals?: { errors: number; behaviors: number }; + // Get workflow and supplementary data, like latest crawl and logs + private readonly workflowDataTask = new Task(this, { + task: async ([workflowId, isEditing], { signal }) => { + if (!workflowId) throw new Error("required `workflowId` missing"); - @state() - private isLoading = false; + this.stopPoll(); - @state() - private isSubmittingUpdate = false; + if (isEditing && this.prevValues.workflow) { + return { workflow: this.prevValues.workflow }; + } - @state() - private isDialogVisible = false; + const workflow = await this.getWorkflow(workflowId, signal).then( + (workflow) => { + this.prevValues.workflow = workflow; + return workflow; + }, + ); - @state() - private isCancelingOrStoppingCrawl = false; + if ( + // Last crawl ID and crawl time can also be set in `runNow()` + workflow.lastCrawlId !== this.lastCrawlId + ) { + this.lastCrawlId = workflow.lastCrawlId; + } else if ( + workflow.isCrawlRunning && + this.groupedWorkflowTab === WorkflowTab.Crawls + ) { + void this.crawlsTask.run(); + } - @state() - private crawlToDelete: Crawl | null = null; + if ( + workflow.lastCrawlId && + (!this.workflowDataTask.value || + (workflow.isCrawlRunning && + this.groupedWorkflowTab === WorkflowTab.LatestCrawl)) + ) { + // Fill in crawl information that's not provided by workflow + const [{ stats, pageCount, reviewStatus }, logTotals] = + await Promise.all([ + this.getCrawl(workflow.lastCrawlId, signal).then((latestCrawl) => { + this.prevValues.latestCrawl = latestCrawl; + return latestCrawl; + }), + this.getLogTotals(workflow.lastCrawlId, signal).then( + (logTotals) => { + this.prevValues.logTotals = logTotals; + return logTotals; + }, + ), + ]); + return { + workflow, + latestCrawl: { stats, pageCount, reviewStatus }, + logTotals, + }; + } - @state() - private filterBy: Partial> = {}; + return { + workflow, + latestCrawl: this.prevValues.latestCrawl, + logTotals: this.prevValues.logTotals, + }; + }, + args: () => [this.workflowId, this.isEditing] as const, + }); - @state() - private timerId?: number; + private readonly pollTask = new Task(this, { + task: async ([workflow, isEditing]) => { + if (!workflow || isEditing) { + return; + } - @query("#pausedNotice") - private readonly pausedNotice?: Alert | null; + return window.setTimeout(() => { + void this.workflowDataTask.run(); + }, POLL_INTERVAL_SECONDS * 1000); + }, + args: () => [this.workflowDataTask.value, this.isEditing] as const, + }); - private getWorkflowPromise?: Promise; - private getSeedsPromise?: Promise>; + private readonly seedsTask = new Task(this, { + task: async ([workflowId, isEditing], { signal }) => { + if (!workflowId) throw new Error("required `workflowId` missing"); + + if (isEditing && this.prevValues.seeds) { + return this.prevValues.seeds; + } + + return await this.getSeeds(workflowId, signal); + }, + args: () => [this.workflowId, this.isEditing] as const, + }); + + private readonly crawlsTask = new Task(this, { + task: async ([workflowId, filterBy], { signal }) => { + if (!workflowId) throw new Error("required `workflowId` missing"); + + return await this.getCrawls(workflowId, filterBy, signal); + }, + args: () => [this.workflowId, this.filterBy, this.lastCrawlId] as const, + }); private readonly runNowTask = new Task(this, { - autoRun: false, task: async (_args, { signal }) => { - await this.runNow({ signal }); - await this.fetchWorkflow(); + this.stopPoll(); + + await this.runNow(signal); + + await this.workflowDataTask.run(); + + return this.workflow; }, - args: () => [] as const, }); + private readonly scaleTask = new Task(this, { + task: async ([value], { signal }) => { + this.stopPoll(); + + await this.scale(value as Crawl["scale"], signal); + + await this.workflowDataTask.run(); + + return this.workflow; + }, + }); + + private readonly pauseResumeTask = new Task(this, { + task: async (_args, { signal }) => { + this.stopPoll(); + + await this.pauseResume(signal); + + void this.crawlsTask.run(); + + await this.workflowDataTask.run(); + + return this.workflow; + }, + }); + + private readonly stopTask = new Task(this, { + task: async (_args, { signal }) => { + this.stopPoll(); + + await this.stop(signal); + + void this.crawlsTask.run(); + + await this.workflowDataTask.run(); + + return this.workflow; + }, + }); + + private readonly cancelTask = new Task(this, { + task: async (_args, { signal }) => { + this.stopPoll(); + + await this.cancel(signal); + + void this.crawlsTask.run(); + + await this.workflowDataTask.run(); + + return this.workflow; + }, + }); + + // TODO Use task render function + private get workflow() { + return this.workflowDataTask.value?.workflow; + } + private get seeds() { + return this.seedsTask.value; + } + private get crawls() { + return this.crawlsTask.value; + } + private get isLoading() { + return this.workflowDataTask.status === TaskStatus.INITIAL; + } + private get isCancelingOrStoppingCrawl() { + return isLoading(this.stopTask) || isLoading(this.cancelTask); + } + private get isExplicitRunning() { return ( this.workflow?.isCrawlRunning && @@ -160,94 +314,36 @@ export class WorkflowDetail extends BtrixElement { super.disconnectedCallback(); } - firstUpdated() { + protected willUpdate(changedProperties: PropertyValues): void { if ( - this.openDialogName && - (this.openDialogName === "scale" || this.openDialogName === "exclusions") + (changedProperties.has("workflowTab") || + changedProperties.has("isEditing")) && + !this.isEditing && + !this.workflowTab ) { - void this.showDialog(); + this.workflowTab = WorkflowTab.LatestCrawl; } } - willUpdate(changedProperties: PropertyValues & Map) { + firstUpdated() { if ( - (changedProperties.has("workflowId") && this.workflowId) || - (changedProperties.get("isEditing") === true && !this.isEditing) + this.openDialogName && + (this.openDialogName === "scale" || this.openDialogName === "exclusions") ) { - void this.fetchWorkflow(); - void this.fetchSeeds(); - void this.fetchCrawls(); - } else if (changedProperties.has("workflowTab")) { - void this.fetchDataForTab(); - } - - if (changedProperties.has("isEditing") && this.isEditing) { - this.stopPoll(); - } - } - - private async fetchDataForTab() { - switch (this.groupedWorkflowTab) { - case WorkflowTab.LatestCrawl: - void this.fetchWorkflow(); - break; - - case WorkflowTab.Crawls: { - void this.fetchCrawls(); - break; - } - default: - break; + void this.showDialog(); } } - private async fetchWorkflow() { - this.stopPoll(); - this.isLoading = true; - - try { - this.getWorkflowPromise = this.getWorkflow(); - this.workflow = await this.getWorkflowPromise; - this.lastCrawlId = this.workflow.lastCrawlId; - this.lastCrawlStartTime = this.workflow.lastCrawlStartTime; - - if ( - this.lastCrawlId && - this.groupedWorkflowTab === WorkflowTab.LatestCrawl - ) { - void this.fetchLastCrawl(); - } - - // TODO: Check if storage quota has been exceeded here by running - // crawl?? - } catch (e) { - this.notify.toast({ - message: - isApiError(e) && e.statusCode === 404 - ? msg("Workflow not found.") - : msg("Sorry, couldn't retrieve workflow at this time."), - variant: "danger", - icon: "exclamation-octagon", - id: "data-retrieve-error", + render() { + if (this.workflowDataTask.status === TaskStatus.ERROR) { + return this.workflowDataTask.render({ + error: this.renderPageError, }); } - this.isLoading = false; - - if (!this.isEditing) { - // Restart timer for next poll - this.timerId = window.setTimeout(() => { - void this.fetchWorkflow(); - }, 1000 * POLL_INTERVAL_SECONDS); - } - } - - render() { if (this.isEditing && this.isCrawler) { return html` -
- ${when(this.workflow, this.renderEditor)} -
+
${this.renderEditor()}
`; } @@ -282,7 +378,11 @@ export class WorkflowDetail extends BtrixElement {
- ${when(this.workflow, this.renderTabList, this.renderLoading)} + ${when( + this.workflow && this.groupedWorkflowTab, + this.renderTabList, + this.renderLoading, + )}
{ - await this.stop(); + await this.stopTask.run(); this.openDialogName = undefined; }} >${msg("Stop Crawling")} { - await this.cancel(); + await this.cancelTask.run(); this.openDialogName = undefined; }} >${msg(html`Cancel & Discard Crawl`)} { + if (isApiError(err) && err.statusCode >= 400 && err.statusCode < 500) { + // API returns a 422 for non existing CIDs + return html``; + } + + console.error(err); + + const email = this.appState.settings?.supportEmail; + + return pageError({ + heading: msg("Sorry, something unexpected went wrong"), + detail: msg("Try reloading the page."), + primaryAction: html` window.location.reload()} + size="small" + >${msg("Reload")}`, + secondaryAction: email + ? html` + ${msg("If the problem persists, please reach out to us.")} +
+ + ${msg("Contact Support")} + + ` + : undefined, + }); + }; + private renderBreadcrumbs() { const breadcrumbs: Breadcrumb[] = [ { @@ -438,12 +568,12 @@ export class WorkflowDetail extends BtrixElement { return pageNav(breadcrumbs); } - private readonly renderTabList = () => html` - + private readonly renderTabList = (tab: WorkflowTab) => html` +
-

${this.tabLabels[this.groupedWorkflowTab]}

+

${this.tabLabels[tab]}

${this.renderPanelAction()}
@@ -476,7 +606,7 @@ export class WorkflowDetail extends BtrixElement { size="small" href="${this.basePath}/crawls/${this.lastCrawlId}#qa" @click=${this.navigate.link} - ?loading=${!this.lastCrawl} + ?loading=${this.isLoading} > ${msg("QA Crawl")} @@ -558,18 +688,22 @@ export class WorkflowDetail extends BtrixElement { - ${when( - !this.isLoading && this.seeds && this.workflow, - (workflow) => html` - this.navigate.to(this.basePath)} - > - `, - this.renderLoading, - )} + ${this.workflow && this.seeds + ? html` + this.navigate.to(this.basePath)} + > + ` + : until( + Promise.all([ + this.workflowDataTask.taskComplete, + this.seedsTask.taskComplete, + ]).catch(this.renderPageError), + this.renderLoading(), + )} `; private readonly renderActions = () => { @@ -601,7 +735,7 @@ export class WorkflowDetail extends BtrixElement { () => html` void this.pauseResumeTask.run()} ?disabled=${disablePauseResume} variant=${ifDefined(paused ? "primary" : undefined)} > @@ -856,7 +990,6 @@ export class WorkflowDetail extends BtrixElement { ...this.filterBy, state: value, }; - void this.fetchCrawls(); }} > ${inactiveCrawlStates.map(this.renderStatusMenuItem)} @@ -887,8 +1020,8 @@ export class WorkflowDetail extends BtrixElement { ${when( this.crawls, - () => - this.crawls!.items.map( + (crawls) => + crawls.items.map( (crawl: Crawl) => html` `, ), () => - // TODO Handle laoding state in crawls list - html` -
- -
- `, + this.crawlsTask.render({ + pending: () => html` +
+ +
+ `, + error: () => html` +
+ ${msg( + "Sorry, couldn't retrieve crawls at this time", + )} +
+ `, + }), )} @@ -962,6 +1105,7 @@ export class WorkflowDetail extends BtrixElement { return this.renderInactiveCrawlMessage(); } + const logTotals = this.workflowDataTask.value?.logTotals; const showReplay = this.workflow && (!this.workflow.isCrawlRunning || @@ -972,7 +1116,7 @@ export class WorkflowDetail extends BtrixElement { ${this.renderCrawlDetails()} - + ${this.tabLabels.logs} - ${this.logTotals?.errors + ${logTotals?.errors ? html` - ${this.localize.number(this.logTotals.errors)} - ${pluralOf("errors", this.logTotals.errors)} + ${this.localize.number(logTotals.errors)} + ${pluralOf("errors", logTotals.errors)} ` : nothing} @@ -1170,9 +1314,11 @@ export class WorkflowDetail extends BtrixElement { `; } + const logTotals = this.workflowDataTask.value?.logTotals; + if ( this.workflowTab === WorkflowTab.Logs && - (this.logTotals?.errors || this.logTotals?.behaviors) + (logTotals?.errors || logTotals?.behaviors) ) { return html` { + const latestCrawl = this.workflowDataTask.value?.latestCrawl; const skeleton = html``; + const duration = (workflow: Workflow) => { + if (!workflow.lastCrawlStartTime) return skeleton; + + return this.localize.humanizeDuration( + (workflow.lastCrawlTime && !workflow.isCrawlRunning + ? new Date(workflow.lastCrawlTime) + : new Date() + ).valueOf() - new Date(workflow.lastCrawlStartTime).valueOf(), + ); + }; + const pages = (workflow: Workflow) => { - if (!this.lastCrawl) return skeleton; + if (!latestCrawl) return skeleton; if (workflow.isCrawlRunning) { return [ - this.localize.number(+(this.lastCrawl.stats?.done || 0)), - this.localize.number(+(this.lastCrawl.stats?.found || 0)), + this.localize.number(+(latestCrawl.stats?.done || 0)), + this.localize.number(+(latestCrawl.stats?.found || 0)), ].join(` ${msg("of")} `); } - return this.localize.number(this.lastCrawl.pageCount || 0); + return this.localize.number(latestCrawl.pageCount || 0); }; const qa = (workflow: Workflow) => { - if (!this.lastCrawl) - return html``; + if (!latestCrawl) return html``; if (workflow.isCrawlRunning) { return html` @@ -1222,21 +1379,21 @@ export class WorkflowDetail extends BtrixElement { } return html``; }; return html` ${this.renderDetailItem(msg("Elapsed Time"), (workflow) => - this.lastCrawlStartTime - ? this.localize.humanizeDuration( - (workflow.lastCrawlTime && !workflow.isCrawlRunning - ? new Date(workflow.lastCrawlTime) - : new Date() - ).valueOf() - new Date(this.lastCrawlStartTime).valueOf(), - ) - : skeleton, + isLoading(this.runNowTask) + ? html`${until( + this.runNowTask.taskComplete.then((workflow) => + workflow ? duration(workflow) : noData, + ), + html``, + )}` + : duration(workflow), )} ${this.renderDetailItem(msg("Pages Crawled"), pages)} ${this.renderDetailItem(msg("Size"), (workflow) => @@ -1373,8 +1530,6 @@ export class WorkflowDetail extends BtrixElement { private renderReplay() { if (!this.workflow || !this.lastCrawlId) return; - console.log(this.lastCrawlId); - const replaySource = `/api/orgs/${this.workflow.oid}/crawls/${this.lastCrawlId}/replay.json`; const headers = this.authState?.headers; const config = JSON.stringify({ headers }); @@ -1401,7 +1556,7 @@ export class WorkflowDetail extends BtrixElement { @@ -1424,8 +1579,8 @@ export class WorkflowDetail extends BtrixElement { variant="primary" ?disabled=${this.org?.storageQuotaReached || this.org?.execMinutesQuotaReached || - this.runNowTask.status === TaskStatus.PENDING} - ?loading=${this.runNowTask.status === TaskStatus.PENDING} + isLoading(this.runNowTask)} + ?loading=${isLoading(this.runNowTask)} @click=${() => void this.runNowTask.run()} > @@ -1534,10 +1689,10 @@ export class WorkflowDetail extends BtrixElement { value=${value} size="small" @click=${async () => { - await this.scale(value); + await this.scaleTask.run([value]); this.openDialogName = undefined; }} - ?disabled=${this.isSubmittingUpdate} + ?disabled=${isLoading(this.scaleTask)} >${label} `, @@ -1548,7 +1703,10 @@ export class WorkflowDetail extends BtrixElement { (this.openDialogName = undefined)} + @click=${() => { + this.scaleTask.abort(); + this.openDialogName = undefined; + }} >${msg("Cancel")} @@ -1575,17 +1733,16 @@ export class WorkflowDetail extends BtrixElement { `; private readonly showDialog = async () => { - await this.getWorkflowPromise; + await this.workflowDataTask.taskComplete; this.isDialogVisible = true; }; private handleExclusionChange() { - void this.fetchWorkflow(); + void this.workflowDataTask.run(); } - private async scale(value: Crawl["scale"]) { + private async scale(value: Crawl["scale"], signal: AbortSignal) { if (!this.lastCrawlId) return; - this.isSubmittingUpdate = true; try { const data = await this.api.fetch<{ scaled: boolean }>( @@ -1593,11 +1750,11 @@ export class WorkflowDetail extends BtrixElement { { method: "POST", body: JSON.stringify({ scale: +value }), + signal, }, ); if (data.scaled) { - void this.fetchWorkflow(); this.notify.toast({ message: msg("Updated number of browser windows."), variant: "success", @@ -1617,13 +1774,12 @@ export class WorkflowDetail extends BtrixElement { id: "browser-windows-update-status", }); } - - this.isSubmittingUpdate = false; } - private async getWorkflow(): Promise { - const data: Workflow = await this.api.fetch( - `/orgs/${this.orgId}/crawlconfigs/${this.workflowId}`, + private async getWorkflow(workflowId: string, signal: AbortSignal) { + const data = await this.api.fetch( + `/orgs/${this.orgId}/crawlconfigs/${workflowId}`, + { signal }, ); return data; } @@ -1636,47 +1792,23 @@ export class WorkflowDetail extends BtrixElement { this.openDialogName = undefined; } - private async fetchSeeds(): Promise { - try { - this.getSeedsPromise = this.getSeeds(); - this.seeds = await this.getSeedsPromise; - } catch { - this.notify.toast({ - message: msg( - "Sorry, couldn't retrieve all crawl settings at this time.", - ), - variant: "danger", - icon: "exclamation-octagon", - id: "data-retrieve-error", - }); - } - } - - private async getSeeds() { + private async getSeeds(workflowId: string, signal: AbortSignal) { const data = await this.api.fetch>( - `/orgs/${this.orgId}/crawlconfigs/${this.workflowId}/seeds`, + `/orgs/${this.orgId}/crawlconfigs/${workflowId}/seeds`, + { signal }, ); return data; } - private async fetchCrawls() { - try { - this.crawls = await this.getCrawls(); - } catch { - this.notify.toast({ - message: msg("Sorry, couldn't get crawls at this time."), - variant: "danger", - icon: "exclamation-octagon", - id: "data-retrieve-error", - }); - } - } - - private async getCrawls() { + private async getCrawls( + workflowId: string, + { state }: WorkflowDetail["filterBy"], + signal: AbortSignal, + ) { const query = queryString.stringify( { - state: this.filterBy.state, - cid: this.workflowId, + state, + cid: workflowId, sortBy: "started", }, { @@ -1685,69 +1817,38 @@ export class WorkflowDetail extends BtrixElement { ); const data = await this.api.fetch>( `/orgs/${this.orgId}/crawls?${query}`, + { signal }, ); return data; } private stopPoll() { - window.clearTimeout(this.timerId); - } - - private async fetchLastCrawl() { - if (!this.lastCrawlId) return; - - let crawlState: CrawlState | null = null; - - try { - const { stats, pageCount, reviewStatus, state } = await this.getCrawl( - this.lastCrawlId, - ); - this.lastCrawl = { stats, pageCount, reviewStatus }; - - crawlState = state; - } catch { - this.notify.toast({ - message: msg("Sorry, couldn't retrieve latest crawl at this time."), - variant: "danger", - icon: "exclamation-octagon", - id: "data-retrieve-error", - }); - } - - if ( - !this.logTotals || - (crawlState && isActive({ state: crawlState })) || - this.workflowTab === WorkflowTab.Logs - ) { - try { - this.logTotals = await this.getLogTotals(this.lastCrawlId); - } catch (err) { - // Fail silently, since we're fetching just the total - console.debug(err); - } + if (this.pollTask.value) { + window.clearTimeout(this.pollTask.value); } } - private async getCrawl(crawlId: Crawl["id"]): Promise { + private async getCrawl(crawlId: Crawl["id"], signal: AbortSignal) { const data = await this.api.fetch( `/orgs/${this.orgId}/crawls/${crawlId}/replay.json`, + { signal }, ); return data; } - private async getLogTotals( - crawlId: Crawl["id"], - ): Promise { + private async getLogTotals(crawlId: Crawl["id"], signal: AbortSignal) { const query = queryString.stringify({ pageSize: 1 }); const [errors, behaviors] = await Promise.all([ this.api.fetch>( `/orgs/${this.orgId}/crawls/${crawlId}/errors?${query}`, + { signal }, ), this.api.fetch>( `/orgs/${this.orgId}/crawls/${crawlId}/behaviorLogs?${query}`, + { signal }, ), ]); @@ -1761,8 +1862,8 @@ export class WorkflowDetail extends BtrixElement { * Create a new template using existing template data */ private async duplicateConfig() { - if (!this.workflow) await this.getWorkflowPromise; - if (!this.seeds) await this.getSeedsPromise; + if (!this.workflow) await this.workflowDataTask.taskComplete; + if (!this.seeds) await this.seedsTask.taskComplete; await this.updateComplete; if (!this.workflow) return; @@ -1815,7 +1916,7 @@ export class WorkflowDetail extends BtrixElement { } } - private async pauseResume() { + private async pauseResume(signal: AbortSignal) { if (!this.lastCrawlId) return; const pause = this.workflow?.lastCrawlState !== "paused"; @@ -1825,20 +1926,21 @@ export class WorkflowDetail extends BtrixElement { `/orgs/${this.orgId}/crawls/${this.lastCrawlId}/${pause ? "pause" : "resume"}`, { method: "POST", + signal, }, ); if (data.success) { - void this.fetchWorkflow(); + this.notify.toast({ + message: pause + ? msg("Pausing crawl.") + : msg("Resuming paused crawl."), + variant: "success", + icon: "check2-circle", + id: "crawl-action-status", + }); } else { throw data; } - - this.notify.toast({ - message: pause ? msg("Pausing crawl.") : msg("Resuming paused crawl."), - variant: "success", - icon: "check2-circle", - id: "crawl-pause-resume-status", - }); } catch { this.notify.toast({ message: pause @@ -1846,25 +1948,29 @@ export class WorkflowDetail extends BtrixElement { : msg("Something went wrong, couldn't resume paused crawl."), variant: "danger", icon: "exclamation-octagon", - id: "crawl-pause-resume-status", + id: "crawl-action-status", }); } } - private async cancel() { + private async cancel(signal: AbortSignal) { if (!this.lastCrawlId) return; - this.isCancelingOrStoppingCrawl = true; - try { const data = await this.api.fetch<{ success: boolean }>( `/orgs/${this.orgId}/crawls/${this.lastCrawlId}/cancel`, { method: "POST", + signal, }, ); if (data.success) { - void this.fetchWorkflow(); + this.notify.toast({ + message: msg("Canceling crawl."), + variant: "success", + icon: "check2-circle", + id: "crawl-action-status", + }); } else { throw data; } @@ -1873,27 +1979,29 @@ export class WorkflowDetail extends BtrixElement { message: msg("Something went wrong, couldn't cancel crawl."), variant: "danger", icon: "exclamation-octagon", - id: "crawl-stop-error", + id: "crawl-action-status", }); } - - this.isCancelingOrStoppingCrawl = false; } - private async stop() { + private async stop(signal: AbortSignal) { if (!this.lastCrawlId) return; - this.isCancelingOrStoppingCrawl = true; - try { const data = await this.api.fetch<{ success: boolean }>( `/orgs/${this.orgId}/crawls/${this.lastCrawlId}/stop`, { method: "POST", + signal, }, ); if (data.success) { - void this.fetchWorkflow(); + this.notify.toast({ + message: msg("Stopping crawl."), + variant: "success", + icon: "check2-circle", + id: "crawl-action-status", + }); } else { throw data; } @@ -1902,16 +2010,12 @@ export class WorkflowDetail extends BtrixElement { message: msg("Something went wrong, couldn't stop crawl."), variant: "danger", icon: "exclamation-octagon", - id: "crawl-stop-error", + id: "crawl-action-status", }); } - - this.isCancelingOrStoppingCrawl = false; } - private async runNow({ - signal, - }: { signal?: AbortSignal } = {}): Promise { + private async runNow(signal: AbortSignal): Promise { try { const data = await this.api.fetch<{ started: string | null }>( `/orgs/${this.orgId}/crawlconfigs/${this.workflowId}/run`, @@ -1921,7 +2025,6 @@ export class WorkflowDetail extends BtrixElement { }, ); this.lastCrawlId = data.started; - this.lastCrawlStartTime = new Date().toISOString(); this.navigate.to(`${this.basePath}/${WorkflowTab.LatestCrawl}`); @@ -1929,7 +2032,7 @@ export class WorkflowDetail extends BtrixElement { message: msg("Starting crawl."), variant: "success", icon: "check2-circle", - id: "crawl-start-status", + id: "crawl-action-status", }); } catch (e) { let message = msg("Sorry, couldn't run crawl at this time."); @@ -1952,7 +2055,7 @@ export class WorkflowDetail extends BtrixElement { message: message, variant: "danger", icon: "exclamation-octagon", - id: "crawl-start-status", + id: "crawl-action-status", }); } } @@ -1971,20 +2074,17 @@ export class WorkflowDetail extends BtrixElement { }), }); this.crawlToDelete = null; - this.crawls = { - ...this.crawls!, - items: this.crawls!.items.filter((c) => c.id !== crawl.id), - }; + void this.crawlsTask.run(); + this.notify.toast({ message: msg(`Successfully deleted crawl`), variant: "success", icon: "check2-circle", id: "archived-item-delete-status", }); - void this.fetchCrawls(); // Update crawl count - void this.fetchWorkflow(); + void this.workflowDataTask.run(); } catch (e) { if (this.crawlToDelete) { this.confirmDeleteCrawl(this.crawlToDelete); From 76ebebfa06a751db852921b4ce26f756ca6e0b2c Mon Sep 17 00:00:00 2001 From: sua yoo Date: Wed, 21 May 2025 18:15:20 -0700 Subject: [PATCH 09/34] fix size not showingf --- frontend/src/features/archived-items/crawl-list.ts | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/frontend/src/features/archived-items/crawl-list.ts b/frontend/src/features/archived-items/crawl-list.ts index 016a15513b..9b23456fbc 100644 --- a/frontend/src/features/archived-items/crawl-list.ts +++ b/frontend/src/features/archived-items/crawl-list.ts @@ -197,9 +197,14 @@ export class CrawlListItem extends BtrixElement { })} - ${this.localize.bytes(this.crawl.fileSize || 0, { - unitDisplay: "narrow", - })} + ${this.safeRender((crawl) => + this.localize.bytes( + crawl.finished ? crawl.fileSize || 0 : +(crawl.stats?.size || 0), + { + unitDisplay: "narrow", + }, + ), + )}
From 23420cb6a9cc0796e282602aca5ecadc3dba95ff Mon Sep 17 00:00:00 2001 From: sua yoo Date: Wed, 21 May 2025 18:17:23 -0700 Subject: [PATCH 10/34] add name to logs --- frontend/src/features/archived-items/crawl-logs.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/frontend/src/features/archived-items/crawl-logs.ts b/frontend/src/features/archived-items/crawl-logs.ts index 071ab6ae8c..5ffd06646d 100644 --- a/frontend/src/features/archived-items/crawl-logs.ts +++ b/frontend/src/features/archived-items/crawl-logs.ts @@ -203,6 +203,7 @@ export class CrawlLogs extends BtrixElement {
Date: Wed, 21 May 2025 19:26:50 -0700 Subject: [PATCH 11/34] paginate crawls --- frontend/src/components/ui/pagination.ts | 42 ++++-- .../src/features/archived-items/crawl-logs.ts | 2 +- frontend/src/pages/org/workflow-detail.ts | 132 ++++++++++++------ 3 files changed, 118 insertions(+), 58 deletions(-) diff --git a/frontend/src/components/ui/pagination.ts b/frontend/src/components/ui/pagination.ts index 59a88256cd..9742bbdacc 100644 --- a/frontend/src/components/ui/pagination.ts +++ b/frontend/src/components/ui/pagination.ts @@ -25,6 +25,20 @@ type PageChangeDetail = { }; export type PageChangeEvent = CustomEvent; +export function calculatePages({ + total, + pageSize, +}: { + total: number; + pageSize: number; +}) { + if (total && pageSize) { + return Math.ceil(total / pageSize); + } else { + return 0; + } +} + /** * Pagination * @@ -200,7 +214,7 @@ export class Pagination extends LitElement { if (parsedPage != this._page) { const page = parsePage(this.searchParams.searchParams.get(this.name)); const constrainedPage = Math.max(1, Math.min(this.pages, page)); - this.onPageChange(constrainedPage); + this.onPageChange(constrainedPage, { dispatch: false }); } if (changedProperties.get("page") && this._page) { @@ -364,15 +378,18 @@ export class Pagination extends LitElement { this.onPageChange(this._page < this.pages ? this._page + 1 : this.pages); } - private onPageChange(page: number) { + private onPageChange(page: number, opts = { dispatch: true }) { if (this._page !== page) { this.setPage(page); - this.dispatchEvent( - new CustomEvent("page-change", { - detail: { page: page, pages: this.pages }, - composed: true, - }), - ); + + if (opts.dispatch) { + this.dispatchEvent( + new CustomEvent("page-change", { + detail: { page: page, pages: this.pages }, + composed: true, + }), + ); + } } this._page = page; } @@ -389,10 +406,9 @@ export class Pagination extends LitElement { } private calculatePages() { - if (this.totalCount && this.size) { - this.pages = Math.ceil(this.totalCount / this.size); - } else { - this.pages = 0; - } + this.pages = calculatePages({ + total: this.totalCount, + pageSize: this.size, + }); } } diff --git a/frontend/src/features/archived-items/crawl-logs.ts b/frontend/src/features/archived-items/crawl-logs.ts index 5ffd06646d..4fa366ad58 100644 --- a/frontend/src/features/archived-items/crawl-logs.ts +++ b/frontend/src/features/archived-items/crawl-logs.ts @@ -203,7 +203,7 @@ export class CrawlLogs extends BtrixElement {
task.status === TaskStatus.PENDING; @@ -82,7 +88,11 @@ export class WorkflowDetail extends BtrixElement { private crawlToDelete: Crawl | null = null; @state() - private filterBy: Partial> = {}; + private crawlsParams: { state?: CrawlState[] } & APIPaginationQuery = { + page: parsePage( + new URLSearchParams(location.search).get(CRAWLS_PAGINATION_NAME), + ), + }; @query("#pausedNotice") private readonly pausedNotice?: Alert | null; @@ -96,7 +106,7 @@ export class WorkflowDetail extends BtrixElement { } = {}; // Get workflow and supplementary data, like latest crawl and logs - private readonly workflowDataTask = new Task(this, { + private readonly workflowTask = new Task(this, { task: async ([workflowId, isEditing], { signal }) => { if (!workflowId) throw new Error("required `workflowId` missing"); @@ -119,6 +129,7 @@ export class WorkflowDetail extends BtrixElement { ) { this.lastCrawlId = workflow.lastCrawlId; } else if ( + this.crawlsTask.value && workflow.isCrawlRunning && this.groupedWorkflowTab === WorkflowTab.Crawls ) { @@ -127,7 +138,7 @@ export class WorkflowDetail extends BtrixElement { if ( workflow.lastCrawlId && - (!this.workflowDataTask.value || + (!this.workflowTask.value || (workflow.isCrawlRunning && this.groupedWorkflowTab === WorkflowTab.LatestCrawl)) ) { @@ -168,10 +179,10 @@ export class WorkflowDetail extends BtrixElement { } return window.setTimeout(() => { - void this.workflowDataTask.run(); + void this.workflowTask.run(); }, POLL_INTERVAL_SECONDS * 1000); }, - args: () => [this.workflowDataTask.value, this.isEditing] as const, + args: () => [this.workflowTask.value, this.isEditing] as const, }); private readonly seedsTask = new Task(this, { @@ -188,12 +199,12 @@ export class WorkflowDetail extends BtrixElement { }); private readonly crawlsTask = new Task(this, { - task: async ([workflowId, filterBy], { signal }) => { + task: async ([workflowId, crawlsParams], { signal }) => { if (!workflowId) throw new Error("required `workflowId` missing"); - return await this.getCrawls(workflowId, filterBy, signal); + return await this.getCrawls(workflowId, crawlsParams, signal); }, - args: () => [this.workflowId, this.filterBy, this.lastCrawlId] as const, + args: () => [this.workflowId, this.crawlsParams] as const, }); private readonly runNowTask = new Task(this, { @@ -202,7 +213,7 @@ export class WorkflowDetail extends BtrixElement { await this.runNow(signal); - await this.workflowDataTask.run(); + await this.workflowTask.run(); return this.workflow; }, @@ -214,7 +225,7 @@ export class WorkflowDetail extends BtrixElement { await this.scale(value as Crawl["scale"], signal); - await this.workflowDataTask.run(); + await this.workflowTask.run(); return this.workflow; }, @@ -228,7 +239,7 @@ export class WorkflowDetail extends BtrixElement { void this.crawlsTask.run(); - await this.workflowDataTask.run(); + await this.workflowTask.run(); return this.workflow; }, @@ -242,7 +253,7 @@ export class WorkflowDetail extends BtrixElement { void this.crawlsTask.run(); - await this.workflowDataTask.run(); + await this.workflowTask.run(); return this.workflow; }, @@ -256,7 +267,7 @@ export class WorkflowDetail extends BtrixElement { void this.crawlsTask.run(); - await this.workflowDataTask.run(); + await this.workflowTask.run(); return this.workflow; }, @@ -264,7 +275,7 @@ export class WorkflowDetail extends BtrixElement { // TODO Use task render function private get workflow() { - return this.workflowDataTask.value?.workflow; + return this.workflowTask.value?.workflow; } private get seeds() { return this.seedsTask.value; @@ -273,7 +284,7 @@ export class WorkflowDetail extends BtrixElement { return this.crawlsTask.value; } private get isLoading() { - return this.workflowDataTask.status === TaskStatus.INITIAL; + return this.workflowTask.status === TaskStatus.INITIAL; } private get isCancelingOrStoppingCrawl() { return isLoading(this.stopTask) || isLoading(this.cancelTask); @@ -335,8 +346,8 @@ export class WorkflowDetail extends BtrixElement { } render() { - if (this.workflowDataTask.status === TaskStatus.ERROR) { - return this.workflowDataTask.render({ + if (this.workflowTask.status === TaskStatus.ERROR) { + return this.workflowTask.render({ error: this.renderPageError, }); } @@ -699,7 +710,7 @@ export class WorkflowDetail extends BtrixElement { ` : until( Promise.all([ - this.workflowDataTask.taskComplete, + this.workflowTask.taskComplete, this.seedsTask.taskComplete, ]).catch(this.renderPageError), this.renderLoading(), @@ -968,13 +979,25 @@ export class WorkflowDetail extends BtrixElement { } private renderCrawls() { + const pageView = (crawls: APIPaginatedList) => { + const pages = calculatePages(crawls); + + if (crawls.page === 1 || pages < 2) return; + + const page = this.localize.number(crawls.page); + const pageCount = this.localize.number(pages); + + return msg(str`Viewing page ${page} of ${pageCount}`); + }; + return html`
+
${when(this.crawls, pageView)}
-
${msg("View:")}
+
${msg("Status:")}
{ const value = (e.target as SlSelect).value as CrawlState[]; await this.updateComplete; - this.filterBy = { - ...this.filterBy, + this.crawlsParams = { + ...this.crawlsParams, + page: 1, state: value, }; }} @@ -1078,17 +1102,34 @@ export class WorkflowDetail extends BtrixElement { )}
- ${when( - this.crawls && !this.crawls.items.length, - () => html` -
-

- ${this.crawls?.total - ? msg("No matching crawls found.") - : msg("No crawls yet.")} -

-
- `, + ${when(this.crawls, (crawls) => + crawls.total + ? html` +
+ { + this.crawlsParams = { + ...this.crawlsParams, + page: e.detail.page, + }; + }} + > + +
+ ` + : html` +
+

+ ${this.crawls?.total + ? msg("No matching crawls found.") + : msg("No crawls yet.")} +

+
+ `, )}
`; @@ -1105,7 +1146,7 @@ export class WorkflowDetail extends BtrixElement { return this.renderInactiveCrawlMessage(); } - const logTotals = this.workflowDataTask.value?.logTotals; + const logTotals = this.workflowTask.value?.logTotals; const showReplay = this.workflow && (!this.workflow.isCrawlRunning || @@ -1314,7 +1355,7 @@ export class WorkflowDetail extends BtrixElement { `; } - const logTotals = this.workflowDataTask.value?.logTotals; + const logTotals = this.workflowTask.value?.logTotals; if ( this.workflowTab === WorkflowTab.Logs && @@ -1334,7 +1375,7 @@ export class WorkflowDetail extends BtrixElement { } private readonly renderCrawlDetails = () => { - const latestCrawl = this.workflowDataTask.value?.latestCrawl; + const latestCrawl = this.workflowTask.value?.latestCrawl; const skeleton = html``; const duration = (workflow: Workflow) => { @@ -1733,12 +1774,12 @@ export class WorkflowDetail extends BtrixElement {
`; private readonly showDialog = async () => { - await this.workflowDataTask.taskComplete; + await this.workflowTask.taskComplete; this.isDialogVisible = true; }; private handleExclusionChange() { - void this.workflowDataTask.run(); + void this.workflowTask.run(); } private async scale(value: Crawl["scale"], signal: AbortSignal) { @@ -1802,19 +1843,22 @@ export class WorkflowDetail extends BtrixElement { private async getCrawls( workflowId: string, - { state }: WorkflowDetail["filterBy"], + params: WorkflowDetail["crawlsParams"], signal: AbortSignal, ) { const query = queryString.stringify( { - state, cid: workflowId, sortBy: "started", + page: params.page ?? this.crawls?.page, + pageSize: this.crawls?.pageSize ?? 10, + ...params, }, { arrayFormat: "comma", }, ); + const data = await this.api.fetch>( `/orgs/${this.orgId}/crawls?${query}`, { signal }, @@ -1862,7 +1906,7 @@ export class WorkflowDetail extends BtrixElement { * Create a new template using existing template data */ private async duplicateConfig() { - if (!this.workflow) await this.workflowDataTask.taskComplete; + if (!this.workflow) await this.workflowTask.taskComplete; if (!this.seeds) await this.seedsTask.taskComplete; await this.updateComplete; if (!this.workflow) return; @@ -2084,7 +2128,7 @@ export class WorkflowDetail extends BtrixElement { }); // Update crawl count - void this.workflowDataTask.run(); + void this.workflowTask.run(); } catch (e) { if (this.crawlToDelete) { this.confirmDeleteCrawl(this.crawlToDelete); From 2731e11b036ce9de41ba81c6811870ee6988efed Mon Sep 17 00:00:00 2001 From: sua yoo Date: Wed, 21 May 2025 19:51:53 -0700 Subject: [PATCH 12/34] tweak layout --- .../features/archived-items/crawl-status.ts | 1 + .../archived-item-detail.ts | 2 -- frontend/src/pages/org/workflow-detail.ts | 22 ++++++++++--------- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/frontend/src/features/archived-items/crawl-status.ts b/frontend/src/features/archived-items/crawl-status.ts index 450fdf2f3b..e2077ea5db 100644 --- a/frontend/src/features/archived-items/crawl-status.ts +++ b/frontend/src/features/archived-items/crawl-status.ts @@ -133,6 +133,7 @@ export class CrawlStatus extends TailwindElement { break; case "resuming": + color = "var(--sl-color-violet-600)"; icon = html` ${msg("QA Crawl")} + - `; + `; } if (this.workflowTab === WorkflowTab.Settings && this.isCrawler) { @@ -1265,7 +1267,7 @@ export class WorkflowDetail extends BtrixElement { return html`
@@ -1302,9 +1304,10 @@ export class WorkflowDetail extends BtrixElement { }; private renderLatestCrawlAction() { + if (!this.workflow || !this.lastCrawlId) return; + if ( this.isCrawler && - this.workflow && this.workflow.isCrawlRunning && this.workflow.lastCrawlState !== "paused" ) { @@ -1313,7 +1316,7 @@ export class WorkflowDetail extends BtrixElement { this.workflow.scale * (this.appState.settings?.numBrowsers || 1); return html` -
+
${msg("Running in")} ${this.localize.number(windowCount)} ${pluralOf("browserWindows", windowCount)}
@@ -1340,8 +1343,7 @@ export class WorkflowDetail extends BtrixElement { if ( this.workflowTab === WorkflowTab.LatestCrawl && - this.lastCrawlId && - this.workflow?.lastCrawlSize + this.workflow.lastCrawlSize ) { return html` - ${this.renderReplay()} + ${guard([this.lastCrawlId], this.renderReplay)}
`; }; @@ -1568,7 +1570,7 @@ export class WorkflowDetail extends BtrixElement { `; } - private renderReplay() { + private readonly renderReplay = () => { if (!this.workflow || !this.lastCrawlId) return; const replaySource = `/api/orgs/${this.workflow.oid}/crawls/${this.lastCrawlId}/replay.json`; @@ -1586,7 +1588,7 @@ export class WorkflowDetail extends BtrixElement { noCache="true" > `; - } + }; private renderLogs() { return html` From a971a88ea0dbcaf62cb9f363d1e319e1fab5934f Mon Sep 17 00:00:00 2001 From: sua yoo Date: Wed, 21 May 2025 21:54:37 -0700 Subject: [PATCH 13/34] simplify poll --- frontend/src/components/ui/desc-list.ts | 8 +- frontend/src/pages/org/workflow-detail.ts | 223 +++++++++++++--------- frontend/src/theme.stylesheet.css | 6 + 3 files changed, 143 insertions(+), 94 deletions(-) diff --git a/frontend/src/components/ui/desc-list.ts b/frontend/src/components/ui/desc-list.ts index 3096f44588..83798d718d 100644 --- a/frontend/src/components/ui/desc-list.ts +++ b/frontend/src/components/ui/desc-list.ts @@ -30,7 +30,7 @@ export class DescListItem extends LitElement { color: var(--sl-color-neutral-500); font-size: var(--sl-font-size-x-small); line-height: 1rem; - margin: 0 0 var(--sl-spacing-2x-small) 0; + margin: 0 0 var(--sl-spacing-3x-small) 0; } dd { @@ -40,8 +40,8 @@ export class DescListItem extends LitElement { font-size: var(--sl-font-size-medium); font-family: var(--font-monostyle-family); font-variation-settings: var(--font-monostyle-variation); - line-height: 1rem; - min-height: calc(1rem + var(--sl-spacing-2x-small)); + line-height: 1.5rem; + min-height: 1.5rem; } .item { @@ -94,7 +94,7 @@ export class DescList extends LitElement { display: inline-block; flex: 1 0 0; min-width: min-content; - padding-top: var(--sl-spacing-2x-small); + padding-top: var(--sl-spacing-x-small); } .horizontal ::slotted(btrix-desc-list-item)::before { diff --git a/frontend/src/pages/org/workflow-detail.ts b/frontend/src/pages/org/workflow-detail.ts index ab57814430..4ac598d924 100644 --- a/frontend/src/pages/org/workflow-detail.ts +++ b/frontend/src/pages/org/workflow-detail.ts @@ -101,12 +101,9 @@ export class WorkflowDetail extends BtrixElement { // Keep previous values to use when editing private readonly prevValues: { workflow?: Awaited>; - latestCrawl?: Awaited>; - logTotals?: Awaited>; seeds?: APIPaginatedList; } = {}; - // Get workflow and supplementary data, like latest crawl and logs private readonly workflowTask = new Task(this, { task: async ([workflowId, isEditing], { signal }) => { if (!workflowId) throw new Error("required `workflowId` missing"); @@ -114,78 +111,25 @@ export class WorkflowDetail extends BtrixElement { this.stopPoll(); if (isEditing && this.prevValues.workflow) { - return { workflow: this.prevValues.workflow }; + return this.prevValues.workflow; } - const workflow = await this.getWorkflow(workflowId, signal).then( - (workflow) => { - this.prevValues.workflow = workflow; - return workflow; - }, - ); + const workflow = await this.getWorkflow(workflowId, signal); + + this.prevValues.workflow = workflow; if ( - // Last crawl ID and crawl time can also be set in `runNow()` + // Last crawl ID can also be set in `runNow()` workflow.lastCrawlId !== this.lastCrawlId ) { this.lastCrawlId = workflow.lastCrawlId; - } else if ( - this.crawlsTask.value && - workflow.isCrawlRunning && - this.groupedWorkflowTab === WorkflowTab.Crawls - ) { - void this.crawlsTask.run(); - } - - if ( - workflow.lastCrawlId && - (!this.workflowTask.value || - (workflow.isCrawlRunning && - this.groupedWorkflowTab === WorkflowTab.LatestCrawl)) - ) { - // Fill in crawl information that's not provided by workflow - const [{ stats, pageCount, reviewStatus }, logTotals] = - await Promise.all([ - this.getCrawl(workflow.lastCrawlId, signal).then((latestCrawl) => { - this.prevValues.latestCrawl = latestCrawl; - return latestCrawl; - }), - this.getLogTotals(workflow.lastCrawlId, signal).then( - (logTotals) => { - this.prevValues.logTotals = logTotals; - return logTotals; - }, - ), - ]); - return { - workflow, - latestCrawl: { stats, pageCount, reviewStatus }, - logTotals, - }; } - return { - workflow, - latestCrawl: this.prevValues.latestCrawl, - logTotals: this.prevValues.logTotals, - }; + return workflow; }, args: () => [this.workflowId, this.isEditing] as const, }); - private readonly pollTask = new Task(this, { - task: async ([workflow, isEditing]) => { - if (!workflow || isEditing) { - return; - } - - return window.setTimeout(() => { - void this.workflowTask.run(); - }, POLL_INTERVAL_SECONDS * 1000); - }, - args: () => [this.workflowTask.value, this.isEditing] as const, - }); - private readonly seedsTask = new Task(this, { task: async ([workflowId, isEditing], { signal }) => { if (!workflowId) throw new Error("required `workflowId` missing"); @@ -199,6 +143,24 @@ export class WorkflowDetail extends BtrixElement { args: () => [this.workflowId, this.isEditing] as const, }); + private readonly latestCrawlTask = new Task(this, { + task: async ([lastCrawlId], { signal }) => { + if (!lastCrawlId) return null; + + return await this.getCrawl(lastCrawlId, signal); + }, + args: () => [this.lastCrawlId] as const, + }); + + private readonly logTotalsTask = new Task(this, { + task: async ([lastCrawlId], { signal }) => { + if (!lastCrawlId) return null; + + return await this.getLogTotals(lastCrawlId, signal); + }, + args: () => [this.lastCrawlId] as const, + }); + private readonly crawlsTask = new Task(this, { task: async ([workflowId, crawlsParams], { signal }) => { if (!workflowId) throw new Error("required `workflowId` missing"); @@ -208,6 +170,49 @@ export class WorkflowDetail extends BtrixElement { args: () => [this.workflowId, this.crawlsParams] as const, }); + private readonly pollTask = new Task(this, { + task: async ([workflow, isEditing]) => { + if (!workflow || isEditing) { + return; + } + + if (workflow.lastCrawlId) { + await Promise.all([ + this.latestCrawlTask.taskComplete, + this.logTotalsTask.taskComplete, + this.crawlsTask.taskComplete, + ]); + } + + return window.setTimeout(async () => { + void this.workflowTask.run(); + const workflow = await this.workflowTask.taskComplete; + + // Retrieve additional data based on current tab + if ( + workflow.isCrawlRunning && + !workflow.lastCrawlShouldPause && + workflow.lastCrawlState !== "paused" + ) { + switch (this.groupedWorkflowTab) { + case WorkflowTab.LatestCrawl: { + void this.latestCrawlTask.run(); + void this.logTotalsTask.run(); + break; + } + case WorkflowTab.Crawls: { + void this.crawlsTask.run(); + break; + } + default: + break; + } + } + }, POLL_INTERVAL_SECONDS * 1000); + }, + args: () => [this.workflowTask.value, this.isEditing] as const, + }); + private readonly runNowTask = new Task(this, { task: async (_args, { signal }) => { this.stopPoll(); @@ -239,7 +244,6 @@ export class WorkflowDetail extends BtrixElement { await this.pauseResume(signal); void this.crawlsTask.run(); - await this.workflowTask.run(); return this.workflow; @@ -253,7 +257,6 @@ export class WorkflowDetail extends BtrixElement { await this.stop(signal); void this.crawlsTask.run(); - await this.workflowTask.run(); return this.workflow; @@ -267,7 +270,6 @@ export class WorkflowDetail extends BtrixElement { await this.cancel(signal); void this.crawlsTask.run(); - await this.workflowTask.run(); return this.workflow; @@ -276,7 +278,7 @@ export class WorkflowDetail extends BtrixElement { // TODO Use task render function private get workflow() { - return this.workflowTask.value?.workflow; + return this.workflowTask.value; } private get seeds() { return this.seedsTask.value; @@ -284,9 +286,27 @@ export class WorkflowDetail extends BtrixElement { private get crawls() { return this.crawlsTask.value; } - private get isLoading() { - return this.workflowTask.status === TaskStatus.INITIAL; + + private get isReady() { + if (!this.workflow) return false; + + if (this.workflow.lastCrawlId) { + if (this.groupedWorkflowTab === WorkflowTab.LatestCrawl) { + return Boolean(this.latestCrawlTask.value); + } + + if (this.groupedWorkflowTab === WorkflowTab.Crawls) { + return Boolean(this.crawlsTask.value); + } + } + + if (this.groupedWorkflowTab === WorkflowTab.Settings) { + return Boolean(this.seedsTask.value); + } + + return true; } + private get isCancelingOrStoppingCrawl() { return isLoading(this.stopTask) || isLoading(this.cancelTask); } @@ -391,7 +411,7 @@ export class WorkflowDetail extends BtrixElement {
${when( - this.workflow && this.groupedWorkflowTab, + this.isReady && this.groupedWorkflowTab, this.renderTabList, this.renderLoading, )} @@ -616,12 +636,10 @@ export class WorkflowDetail extends BtrixElement { return html` - - ${msg("QA Crawl")} + ${msg("View Details")} `; @@ -1034,8 +1052,10 @@ export class WorkflowDetail extends BtrixElement { class="underline hover:no-underline" @click=${this.navigate.link} > - ${this.workflow?.lastCrawlState === "paused" - ? msg("View Paused Crawl") + ${this.workflow && + (this.workflow.lastCrawlShouldPause || + this.workflow.lastCrawlState === "paused") + ? msg("View Crawl") : msg("Watch Crawl")} @@ -1148,7 +1168,7 @@ export class WorkflowDetail extends BtrixElement { return this.renderInactiveCrawlMessage(); } - const logTotals = this.workflowTask.value?.logTotals; + const logTotals = this.logTotalsTask.value; const showReplay = this.workflow && (!this.workflow.isCrawlRunning || @@ -1200,14 +1220,17 @@ export class WorkflowDetail extends BtrixElement { name=${WorkflowTab.LatestCrawl} class="mt-3 block" > - ${when( - showReplay, - this.renderInactiveWatchCrawl, - this.renderWatchCrawl, + + ${when(this.workflowTab === WorkflowTab.LatestCrawl, () => + when( + showReplay, + this.renderInactiveWatchCrawl, + this.renderWatchCrawl, + ), )} - ${this.renderLogs()} + ${when(this.workflowTab === WorkflowTab.Logs, this.renderLogs)} `; @@ -1357,7 +1380,7 @@ export class WorkflowDetail extends BtrixElement { `; } - const logTotals = this.workflowTask.value?.logTotals; + const logTotals = this.logTotalsTask.value; if ( this.workflowTab === WorkflowTab.Logs && @@ -1377,7 +1400,7 @@ export class WorkflowDetail extends BtrixElement { } private readonly renderCrawlDetails = () => { - const latestCrawl = this.workflowTask.value?.latestCrawl; + const latestCrawl = this.latestCrawlTask.value; const skeleton = html``; const duration = (workflow: Workflow) => { @@ -1421,9 +1444,27 @@ export class WorkflowDetail extends BtrixElement { `; } - return html``; + return html`
+ ${latestCrawl.reviewStatus + ? html`` + : html` + + + ${msg("Add")} + + `} +
`; }; return html` @@ -1590,9 +1631,9 @@ export class WorkflowDetail extends BtrixElement { `; }; - private renderLogs() { + private readonly renderLogs = () => { return html` -
+
${when( this.lastCrawlId, (crawlId) => html` @@ -1608,7 +1649,7 @@ export class WorkflowDetail extends BtrixElement { )}
`; - } + }; private readonly renderRunNowButton = () => { return html` @@ -1760,7 +1801,7 @@ export class WorkflowDetail extends BtrixElement { return html`
Date: Wed, 21 May 2025 22:26:10 -0700 Subject: [PATCH 14/34] simplify buttons --- frontend/src/components/ui/copy-button.ts | 3 +- frontend/src/pages/org/workflow-detail.ts | 81 +++++++++++------------ 2 files changed, 42 insertions(+), 42 deletions(-) diff --git a/frontend/src/components/ui/copy-button.ts b/frontend/src/components/ui/copy-button.ts index f25b54b021..ee121b7a8c 100644 --- a/frontend/src/components/ui/copy-button.ts +++ b/frontend/src/components/ui/copy-button.ts @@ -4,6 +4,7 @@ import { customElement, property } from "lit/decorators.js"; import { TailwindElement } from "@/classes/TailwindElement"; import { ClipboardController } from "@/controllers/clipboard"; +import { tw } from "@/utils/tailwind"; /** * Copy text to clipboard on click @@ -69,7 +70,7 @@ export class CopyButton extends TailwindElement { ? this.name : "copy"} label=${msg("Copy to clipboard")} - class="size-3.5" + class=${this.size === "medium" ? tw`size-4` : tw`size-3.5`} > diff --git a/frontend/src/pages/org/workflow-detail.ts b/frontend/src/pages/org/workflow-detail.ts index 4ac598d924..194622b2c3 100644 --- a/frontend/src/pages/org/workflow-detail.ts +++ b/frontend/src/pages/org/workflow-detail.ts @@ -628,34 +628,38 @@ export class WorkflowDetail extends BtrixElement { private renderPanelAction() { if ( this.groupedWorkflowTab === WorkflowTab.LatestCrawl && - this.isCrawler && - this.workflow && - !this.workflow.isCrawlRunning && - this.lastCrawlId + this.lastCrawlId && + this.latestCrawlTask.value?.finished ) { - return html` - - ${msg("View Details")} - - - `; + return html`
+ + + + + + + +
`; } if (this.workflowTab === WorkflowTab.Settings && this.isCrawler) { - return html` - - - - `; + return html` + + + `; } return nothing; @@ -1445,25 +1449,20 @@ export class WorkflowDetail extends BtrixElement { } return html`
- ${latestCrawl.reviewStatus + ${latestCrawl.reviewStatus || !this.isCrawler ? html`` - : html` - - - ${msg("Add")} - - `} + + ${msg("Add Review")} + `}
`; }; @@ -1595,11 +1594,11 @@ export class WorkflowDetail extends BtrixElement { )} ${when( this.lastCrawlId, - () => + (id) => html`
${msg("View Crawl Details")} From 83ac943407ae532fe3f126ff60bef41e63f044aa Mon Sep 17 00:00:00 2001 From: sua yoo Date: Wed, 21 May 2025 22:49:45 -0700 Subject: [PATCH 15/34] show tooltip for entire latest crawl --- .../features/crawl-workflows/workflow-list.ts | 277 +++++++++--------- 1 file changed, 141 insertions(+), 136 deletions(-) diff --git a/frontend/src/features/crawl-workflows/workflow-list.ts b/frontend/src/features/crawl-workflows/workflow-list.ts index f6828f4597..52b8b7a542 100644 --- a/frontend/src/features/crawl-workflows/workflow-list.ts +++ b/frontend/src/features/crawl-workflows/workflow-list.ts @@ -23,6 +23,7 @@ import { import { BtrixElement } from "@/classes/BtrixElement"; import type { OverflowDropdown } from "@/components/ui/overflow-dropdown"; import { WorkflowTab } from "@/routes"; +import { noData } from "@/strings/ui"; import type { ListWorkflow } from "@/types/crawler"; import { humanizeSchedule } from "@/utils/cron"; import { srOnly, truncate } from "@/utils/css"; @@ -71,6 +72,10 @@ const hostVars = css` } `; +const notSpecified = html`${noData}`; + @customElement("btrix-workflow-list-item") @localized() export class WorkflowListItem extends BtrixElement { @@ -208,10 +213,6 @@ export class WorkflowListItem extends BtrixElement { dropdownMenu!: OverflowDropdown; render() { - const notSpecified = html`---`; - return html`
-
-
- ${this.safeRender( - (workflow) => html` - - `, - )} -
-
- ${this.safeRender((workflow) => { - const inDuration = (dur: number) => { - const compactDuration = this.localize.humanizeDuration(dur, { - compact: true, - }); - return msg(str`in ${compactDuration}`, { - desc: "`compactDuration` example: '2h'", - }); - }; - const forDuration = (dur: number) => { - const compactDuration = this.localize.humanizeDuration(dur, { - compact: true, - }); - return msg(str`for ${compactDuration}`, { - desc: "`compactDuration` example: '2h'", - }); - }; - const afterDuration = (dur: number) => { - const verboseDuration = this.localize.humanizeDuration(dur, { - verbose: true, - unitCount: 2, - }); - return msg(str`after ${verboseDuration}`, { - desc: "`verboseDuration` example: '2 hours, 15 seconds'", - }); - }; - - if (workflow.lastCrawlTime && workflow.lastCrawlStartTime) { - const diff = - new Date(workflow.lastCrawlTime).valueOf() - - new Date(workflow.lastCrawlStartTime).valueOf(); - - return html` - - - ${inDuration(diff)} - - - ${msg("Crawl ended on")} - - ${afterDuration(diff)} - - `; - } - if (workflow.lastCrawlStartTime) { - const latestDate = - workflow.lastCrawlShouldPause && workflow.lastCrawlPausedAt - ? new Date(workflow.lastCrawlPausedAt) - : new Date(); - const diff = - latestDate.valueOf() - - new Date(workflow.lastCrawlStartTime).valueOf(); - if (diff < 1000) { - return ""; - } - - if ( - workflow.lastCrawlState === "paused" && - workflow.lastCrawlPausedAt - ) { - const pausedDiff = - new Date().valueOf() - - new Date(workflow.lastCrawlPausedAt).valueOf(); - const runDuration = afterDuration(diff); - - return html` - - - - ${forDuration(pausedDiff)} - - - - ${msg("Crawl paused on")} - - ${msg(str`${runDuration} of run time`, { - desc: "`runDuration` example: 'after 5 minutes'", - })} - - - `; - } - - return html`${msg("Running")} ${forDuration(diff)}`; - } - return notSpecified; - })} -
-
+
${this.renderLatestCrawl()}
${this.safeRender((workflow) => { @@ -457,6 +327,141 @@ export class WorkflowListItem extends BtrixElement {
`; } + private readonly renderLatestCrawl = () => + this.safeRender((workflow) => { + let tooltipContent = html``; + + const status = () => html` + + `; + + const duration = () => { + const inDuration = (dur: number) => { + const compactDuration = this.localize.humanizeDuration(dur, { + compact: true, + }); + return msg(str`in ${compactDuration}`, { + desc: "`compactDuration` example: '2h'", + }); + }; + const forDuration = (dur: number) => { + const compactDuration = this.localize.humanizeDuration(dur, { + compact: true, + }); + return msg(str`for ${compactDuration}`, { + desc: "`compactDuration` example: '2h'", + }); + }; + const afterDuration = (dur: number) => { + const verboseDuration = this.localize.humanizeDuration(dur, { + verbose: true, + unitCount: 2, + }); + return msg(str`after ${verboseDuration}`, { + desc: "`verboseDuration` example: '2 hours, 15 seconds'", + }); + }; + + if (workflow.lastCrawlTime && workflow.lastCrawlStartTime) { + const diff = + new Date(workflow.lastCrawlTime).valueOf() - + new Date(workflow.lastCrawlStartTime).valueOf(); + + tooltipContent = html` + + ${msg("Crawl ended on")} + + ${afterDuration(diff)} + + `; + + return html` + ${inDuration(diff)}`; + } + + if (workflow.lastCrawlStartTime) { + const latestDate = + workflow.lastCrawlShouldPause && workflow.lastCrawlPausedAt + ? new Date(workflow.lastCrawlPausedAt) + : new Date(); + const diff = + latestDate.valueOf() - + new Date(workflow.lastCrawlStartTime).valueOf(); + if (diff < 1000) { + return ""; + } + + if ( + workflow.lastCrawlState === "paused" && + workflow.lastCrawlPausedAt + ) { + const pausedDiff = + new Date().valueOf() - + new Date(workflow.lastCrawlPausedAt).valueOf(); + tooltipContent = html` + + ${msg("Crawl paused on")} + + + `; + + return html` + + ${forDuration(pausedDiff)} + `; + } + + return html`${msg("Running")} ${forDuration(diff)}`; + } + return notSpecified; + }; + + return html` + +
+
${status()}
+
${duration()}
+
+ + ${tooltipContent} + + `; + }); + private safeRender( render: (workflow: ListWorkflow) => string | TemplateResult<1>, ) { From 60f380f5d0eed2221b22426c12f05540dd117bf5 Mon Sep 17 00:00:00 2001 From: sua yoo Date: Thu, 22 May 2025 10:43:15 -0700 Subject: [PATCH 16/34] update tooltips --- .../features/crawl-workflows/workflow-list.ts | 301 +++++++++--------- 1 file changed, 155 insertions(+), 146 deletions(-) diff --git a/frontend/src/features/crawl-workflows/workflow-list.ts b/frontend/src/features/crawl-workflows/workflow-list.ts index 52b8b7a542..7528d705e6 100644 --- a/frontend/src/features/crawl-workflows/workflow-list.ts +++ b/frontend/src/features/crawl-workflows/workflow-list.ts @@ -72,6 +72,28 @@ const hostVars = css` } `; +const shortDate = (date: string) => html` + +`; +const longDate = (date: string) => html` + +`; + const notSpecified = html`${noData}`; @@ -244,7 +266,7 @@ export class WorkflowListItem extends BtrixElement { })}
-
${this.renderLatestCrawl()}
+
${this.safeRender(this.renderLatestCrawl)}
${this.safeRender((workflow) => { @@ -290,28 +312,7 @@ export class WorkflowListItem extends BtrixElement { )}
-
-
- ${this.safeRender( - (workflow) => - html`${workflow.modifiedByName}`, - )} -
-
- ${this.safeRender( - (workflow) => html` - - `, - )} -
-
+
${this.safeRender(this.renderModifiedBy)}
`; } - private readonly renderLatestCrawl = () => - this.safeRender((workflow) => { - let tooltipContent = html``; - - const status = () => html` - - `; - - const duration = () => { - const inDuration = (dur: number) => { - const compactDuration = this.localize.humanizeDuration(dur, { - compact: true, - }); - return msg(str`in ${compactDuration}`, { - desc: "`compactDuration` example: '2h'", - }); - }; - const forDuration = (dur: number) => { - const compactDuration = this.localize.humanizeDuration(dur, { - compact: true, - }); - return msg(str`for ${compactDuration}`, { - desc: "`compactDuration` example: '2h'", - }); - }; - const afterDuration = (dur: number) => { - const verboseDuration = this.localize.humanizeDuration(dur, { - verbose: true, - unitCount: 2, - }); - return msg(str`after ${verboseDuration}`, { - desc: "`verboseDuration` example: '2 hours, 15 seconds'", - }); - }; - - if (workflow.lastCrawlTime && workflow.lastCrawlStartTime) { - const diff = - new Date(workflow.lastCrawlTime).valueOf() - - new Date(workflow.lastCrawlStartTime).valueOf(); + private readonly renderLatestCrawl = (workflow: ListWorkflow) => { + let tooltipContent: TemplateResult | null = null; + + const status = html` + + `; + const renderDuration = () => { + const compactIn = (dur: number) => { + const compactDuration = this.localize.humanizeDuration(dur, { + compact: true, + }); + return msg(str`in ${compactDuration}`, { + desc: "`compactDuration` example: '2h'", + }); + }; + const verboseIn = (dur: number) => { + const verboseDuration = this.localize.humanizeDuration(dur, { + verbose: true, + unitCount: 2, + }); + return msg(str`in ${verboseDuration}`, { + desc: "`verboseDuration` example: '2 hours, 15 seconds'", + }); + }; + const compactFor = (dur: number) => { + const compactDuration = this.localize.humanizeDuration(dur, { + compact: true, + }); + return msg(str`for ${compactDuration}`, { + desc: "`compactDuration` example: '2h'", + }); + }; + const verboseFor = (dur: number) => { + const verboseDuration = this.localize.humanizeDuration(dur, { + verbose: true, + unitCount: 2, + }); + return msg(str`for ${verboseDuration}`, { + desc: "`verboseDuration` example: '2 hours, 15 seconds'", + }); + }; + + if (workflow.lastCrawlTime && workflow.lastCrawlStartTime) { + const diff = + new Date(workflow.lastCrawlTime).valueOf() - + new Date(workflow.lastCrawlStartTime).valueOf(); + + tooltipContent = html` + + ${msg("Finished")} ${longDate(workflow.lastCrawlTime)} + ${verboseIn(diff)} + + `; + + return html`${shortDate(workflow.lastCrawlTime)} ${compactIn(diff)}`; + } + + if (workflow.lastCrawlStartTime) { + const latestDate = + workflow.lastCrawlShouldPause && workflow.lastCrawlPausedAt + ? new Date(workflow.lastCrawlPausedAt) + : new Date(); + const diff = + latestDate.valueOf() - + new Date(workflow.lastCrawlStartTime).valueOf(); + if (diff < 1000) { + return ""; + } + + if ( + workflow.lastCrawlState === "paused" && + workflow.lastCrawlPausedAt + ) { + const pausedDiff = + new Date().valueOf() - + new Date(workflow.lastCrawlPausedAt).valueOf(); tooltipContent = html` - ${msg("Crawl ended on")} - - ${afterDuration(diff)} + ${msg("Crawl paused on")} ${longDate(workflow.lastCrawlPausedAt)} `; - return html` - ${inDuration(diff)}`; + return html` + ${shortDate(workflow.lastCrawlPausedAt)} ${compactFor(pausedDiff)} + `; } - if (workflow.lastCrawlStartTime) { - const latestDate = - workflow.lastCrawlShouldPause && workflow.lastCrawlPausedAt - ? new Date(workflow.lastCrawlPausedAt) - : new Date(); - const diff = - latestDate.valueOf() - - new Date(workflow.lastCrawlStartTime).valueOf(); - if (diff < 1000) { - return ""; - } - - if ( - workflow.lastCrawlState === "paused" && - workflow.lastCrawlPausedAt - ) { - const pausedDiff = - new Date().valueOf() - - new Date(workflow.lastCrawlPausedAt).valueOf(); - tooltipContent = html` - - ${msg("Crawl paused on")} - - - `; - - return html` - - ${forDuration(pausedDiff)} - `; - } - - return html`${msg("Running")} ${forDuration(diff)}`; - } - return notSpecified; - }; + tooltipContent = html` + + ${msg("Running")} ${verboseFor(diff)} ${msg("since")} + ${longDate(workflow.lastCrawlStartTime)} + + `; + + return html`${msg("Running")} ${compactFor(diff)}`; + } + return notSpecified; + }; + + const duration = renderDuration(); + + return html` + +
+
${status}
+
${duration}
+
+ + ${tooltipContent} + + `; + }; + + private readonly renderModifiedBy = (workflow: ListWorkflow) => { + const date = longDate(workflow.modified); - return html` - -
-
${status()}
-
${duration()}
+ return html` + +
+
+ ${workflow.modifiedByName}
+
${shortDate(workflow.modified)}
+
- ${tooltipContent} - - `; - }); + + ${workflow.modified === workflow.created + ? msg("Created by") + : msg("Edited by")} + ${workflow.modifiedByName} + ${msg(html`on ${date}`, { + desc: "`date` example: 'January 1st, 2025 at 05:00 PM EST'", + })} + + + `; + }; private safeRender( render: (workflow: ListWorkflow) => string | TemplateResult<1>, From eb6f1c3afd63779d1e2f9b354a62f6809ee34908 Mon Sep 17 00:00:00 2001 From: sua yoo Date: Thu, 22 May 2025 12:19:58 -0700 Subject: [PATCH 17/34] update menu options --- frontend/src/components/ui/button.ts | 34 ++- .../archived-item-detail.ts | 5 +- frontend/src/pages/org/workflow-detail.ts | 279 +++++++++++++----- .../src/strings/archived-items/tooltips.ts | 6 - 4 files changed, 236 insertions(+), 88 deletions(-) delete mode 100644 frontend/src/strings/archived-items/tooltips.ts diff --git a/frontend/src/components/ui/button.ts b/frontend/src/components/ui/button.ts index 3a77bb909d..97466fb125 100644 --- a/frontend/src/components/ui/button.ts +++ b/frontend/src/components/ui/button.ts @@ -38,6 +38,9 @@ export class Button extends TailwindElement { @property({ type: String }) href?: string; + @property({ type: String }) + download?: string; + @property({ type: Boolean }) raised = false; @@ -63,32 +66,49 @@ export class Button extends TailwindElement { return html`<${tag} type=${this.type === "submit" ? "submit" : "button"} class=${clsx( - tw`flex cursor-pointer items-center justify-center gap-2 text-center font-medium outline-3 outline-offset-1 outline-primary transition focus-visible:outline disabled:cursor-not-allowed disabled:text-neutral-300`, + this.disabled ? tw`cursor-not-allowed opacity-50` : tw`cursor-pointer`, + tw`flex items-center justify-center gap-2 text-center font-medium outline-3 outline-offset-1 outline-primary transition focus-visible:outline`, { "x-small": tw`min-h-4 min-w-4 text-sm`, small: tw`min-h-6 min-w-6 rounded-md text-base`, medium: tw`min-h-8 min-w-8 rounded-sm text-lg`, }[this.size], - this.raised && - tw`shadow ring-1 ring-stone-500/20 hover:shadow-stone-800/20 hover:ring-stone-800/20`, + this.raised && [ + tw`shadow ring-1 ring-stone-500/20`, + !this.disabled && + tw`hover:shadow-stone-800/20 hover:ring-stone-800/20`, + ], this.filled ? [ tw`text-white`, { - neutral: tw`border-primary-800 bg-primary-500 shadow-primary-800/20 hover:bg-primary-600`, - danger: tw`shadow-danger-800/20 border-danger-800 bg-danger-500 hover:bg-danger-600`, + neutral: [ + tw`border-primary-800 bg-primary-500 shadow-primary-800/20`, + !this.disabled && tw`hover:bg-primary-600`, + ], + danger: [ + tw`shadow-danger-800/20 border-danger-800 bg-danger-500`, + !this.disabled && tw`hover:bg-danger-600`, + ], }[this.variant], ] : [ this.raised && tw`bg-white`, { - neutral: tw`border-gray-300 text-gray-600 hover:text-primary-600`, - danger: tw`shadow-danger-800/20 border-danger-300 bg-danger-50 text-danger-600 hover:bg-danger-100`, + neutral: [ + tw`border-gray-300 text-gray-600`, + !this.disabled && tw`hover:text-primary-500`, + ], + danger: [ + tw`shadow-danger-800/20 border-danger-300 bg-danger-50 text-danger-600`, + !this.disabled && tw`hover:bg-danger-100`, + ], }[this.variant], ], )} ?disabled=${this.disabled} href=${ifDefined(this.href)} + download=${ifDefined(this.download)} aria-label=${ifDefined(this.label)} @click=${this.handleClick} > diff --git a/frontend/src/pages/org/archived-item-detail/archived-item-detail.ts b/frontend/src/pages/org/archived-item-detail/archived-item-detail.ts index 468147173c..a293a60266 100644 --- a/frontend/src/pages/org/archived-item-detail/archived-item-detail.ts +++ b/frontend/src/pages/org/archived-item-detail/archived-item-detail.ts @@ -11,7 +11,6 @@ import { type Dialog } from "@/components/ui/dialog"; import { ClipboardController } from "@/controllers/clipboard"; import { pageBack, pageNav, type Breadcrumb } from "@/layouts/pageHeader"; import { WorkflowTab } from "@/routes"; -import { tooltipFor } from "@/strings/archived-items/tooltips"; import type { APIPaginatedList } from "@/types/api"; import type { ArchivedItem, @@ -316,7 +315,7 @@ export class ArchivedItemDetail extends BtrixElement { case "files": sectionContent = this.renderPanel( html` ${this.renderTitle(this.tabLabels.files)} - + + = { [WorkflowTab.LatestCrawl]: msg("Latest Crawl"), crawls: msg("Crawls"), @@ -626,27 +645,79 @@ export class WorkflowDetail extends BtrixElement { `; private renderPanelAction() { - if ( - this.groupedWorkflowTab === WorkflowTab.LatestCrawl && - this.lastCrawlId && - this.latestCrawlTask.value?.finished - ) { - return html`
- + const latestCrawl = this.latestCrawlTask.value; + + if (this.groupedWorkflowTab === WorkflowTab.LatestCrawl && latestCrawl) { + const archivedItemId = this.archivedItemId; + const logTotals = this.logTotalsTask.value; + const hasLogs = logTotals?.errors || logTotals?.behaviors; + const disableDownload = + this.workflow?.isCrawlRunning && + this.workflow.lastCrawlState !== "paused"; - + const authToken = this.authState?.headers.Authorization.split(" ")[1]; + + return html`
+ - + + + ${when( + hasLogs, + () => html` + + + + + + `, + )} + ${when( + this.workflow?.isCrawlRunning || this.archivedItemId, + () => html` + + + + + + `, + )}
`; } @@ -741,23 +812,37 @@ export class WorkflowDetail extends BtrixElement { )} `; + private get disablePauseResume() { + if (!this.workflow) return true; + + // disable pause/resume button if desired state is already in the process of being set. + // if crawl is running, and pause requested (shouldPause is true), don't allow clicking Pausing + // if crawl not running, and resume requested (shouldPause is false), don't allow clicking Resume + + return ( + this.workflow.lastCrawlShouldPause === + (this.workflow.lastCrawlState === "running") || + isLoading(this.pauseResumeTask) + ); + } + private readonly renderActions = () => { if (!this.workflow) return; const workflow = this.workflow; const archivingDisabled = isArchivingDisabled(this.org, true); + const cancelStopLoading = this.isCancelingRun; const paused = workflow.lastCrawlState === "paused"; const hidePauseResume = !this.lastCrawlId || - this.isCancelingOrStoppingCrawl || + this.isCancelingRun || this.workflow.lastCrawlStopping; - // disable pause/resume button if desired state is already in the process of being set. - // if crawl is running, and pause requested (shouldPause is true), don't allow clicking Pausing - // if crawl not running, and resume requested (shouldPause is false), don't allow clicking Resume const disablePauseResume = - this.workflow.lastCrawlShouldPause === - (this.workflow.lastCrawlState === "running"); + this.disablePauseResume || + cancelStopLoading || + (paused && archivingDisabled); + const pauseResumeLoading = isLoading(this.pauseResumeTask); return html` ${this.renderPausedNotice({ truncate: true })} @@ -774,10 +859,14 @@ export class WorkflowDetail extends BtrixElement { ?disabled=${disablePauseResume} variant=${ifDefined(paused ? "primary" : undefined)} > - + ${pauseResumeLoading + ? html`` + : html` + + `} ${paused ? msg("Resume") : msg("Pause")} `, @@ -786,7 +875,7 @@ export class WorkflowDetail extends BtrixElement { size="small" @click=${() => (this.openDialogName = "stop")} ?disabled=${!this.lastCrawlId || - this.isCancelingOrStoppingCrawl || + this.isCancelingRun || this.workflow?.lastCrawlStopping} > @@ -795,7 +884,7 @@ export class WorkflowDetail extends BtrixElement { (this.openDialogName = "cancel")} - ?disabled=${!this.lastCrawlId || this.isCancelingOrStoppingCrawl} + ?disabled=${!this.lastCrawlId || this.isCancelingRun} > ${when( - this.workflow.isCrawlRunning, + workflow.isCrawlRunning, // HACK shoelace doesn't current have a way to override non-hover // color without resetting the --sl-color-neutral-700 variable () => html` + ${when(!hidePauseResume && !disablePauseResume, () => + workflow.lastCrawlState === "paused" + ? html` + void this.pauseResumeTask.run()} + > + + ${msg("Resume Crawl")} + + ` + : html` + void this.pauseResumeTask.run()} + > + + ${msg("Pause Crawl")} + + `, + )} + (this.openDialogName = "stop")} - ?disabled=${workflow.lastCrawlStopping || - this.isCancelingOrStoppingCrawl} + ?disabled=${workflow.lastCrawlStopping || this.isCancelingRun} > ${msg("Stop Crawl")} (this.openDialogName = "cancel")} > @@ -838,7 +947,7 @@ export class WorkflowDetail extends BtrixElement { `, () => html` void this.runNowTask.run()} > @@ -847,10 +956,10 @@ export class WorkflowDetail extends BtrixElement { `, )} + ${when( workflow.isCrawlRunning && !workflow.lastCrawlStopping, () => html` - (this.openDialogName = "scale")}> ${msg("Edit Browser Windows")} @@ -864,7 +973,6 @@ export class WorkflowDetail extends BtrixElement { `, )} - this.navigate.to( @@ -881,6 +989,13 @@ export class WorkflowDetail extends BtrixElement { ${msg("Duplicate Workflow")} + ${when( + workflow.lastCrawlId, + () => html` + + ${this.renderLatestCrawlMenuOptions()} + `, + )} @@ -896,6 +1011,14 @@ export class WorkflowDetail extends BtrixElement { ${msg("Copy Workflow ID")} + + ClipboardController.copyToClipboard(workflow.lastCrawlId || "")} + ?disabled=${!workflow.lastCrawlId} + > + + ${msg("Copy Latest Crawl ID")} + ${when( !workflow.crawlCount, () => html` @@ -914,6 +1037,52 @@ export class WorkflowDetail extends BtrixElement { `; }; + private renderLatestCrawlMenuOptions() { + const authToken = this.authState?.headers.Authorization.split(" ")[1]; + const latestCrawl = this.latestCrawlTask.value; + const logTotals = this.logTotalsTask.value; + + return html` + + + ${msg("Download Latest Crawl")} + ${latestCrawl?.fileSize + ? html` ${this.localize.bytes(latestCrawl.fileSize)}` + : nothing} + + + + + ${msg("Download Latest Crawl Log")} + + + ${when( + this.archivedItemId, + (id) => html` + + this.navigate.to(`${this.basePath}/${WorkflowTab.Crawls}/${id}`)} + > + + ${msg("Go to Archived Item")} + + `, + )} + `; + } + private renderDetails() { return html` @@ -1365,42 +1534,6 @@ export class WorkflowDetail extends BtrixElement { `; } - - const authToken = this.authState?.headers.Authorization.split(" ")[1]; - - if ( - this.workflowTab === WorkflowTab.LatestCrawl && - this.workflow.lastCrawlSize - ) { - return html` - - - `; - } - - const logTotals = this.logTotalsTask.value; - - if ( - this.workflowTab === WorkflowTab.Logs && - (logTotals?.errors || logTotals?.behaviors) - ) { - return html` - - - `; - } } private readonly renderCrawlDetails = () => { @@ -2044,6 +2177,8 @@ export class WorkflowDetail extends BtrixElement { private async cancel(signal: AbortSignal) { if (!this.lastCrawlId) return; + this.isCancelingRun = true; + try { const data = await this.api.fetch<{ success: boolean }>( `/orgs/${this.orgId}/crawls/${this.lastCrawlId}/cancel`, diff --git a/frontend/src/strings/archived-items/tooltips.ts b/frontend/src/strings/archived-items/tooltips.ts deleted file mode 100644 index 9705de3a35..0000000000 --- a/frontend/src/strings/archived-items/tooltips.ts +++ /dev/null @@ -1,6 +0,0 @@ -import { msg } from "@lit/localize"; - -export const tooltipFor = { - downloadMultWacz: msg(msg("Download Files as Multi-WACZ")), - downloadLogs: msg("Download Entire Log File"), -}; From 9863945cf43bd93224901dd2455c5e252fb237ba Mon Sep 17 00:00:00 2001 From: sua yoo Date: Thu, 22 May 2025 12:20:15 -0700 Subject: [PATCH 18/34] update tooltip --- frontend/src/features/org/usage-history-table.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/features/org/usage-history-table.ts b/frontend/src/features/org/usage-history-table.ts index 07bad2ac07..35be985b4e 100644 --- a/frontend/src/features/org/usage-history-table.ts +++ b/frontend/src/features/org/usage-history-table.ts @@ -71,7 +71,7 @@ export class UsageHistoryTable extends BtrixElement { field: Field.ExecutionTime, label: msg("Execution Time"), description: msg( - "Aggregated time across all browser windows that the crawler was actively executing a crawl or QA analysis run, i.e. not in a waiting state", + "Aggregated time across all browser windows that the crawler was actively executing a crawl or QA analysis run, i.e. not waiting or paused", ), }, ]; From 8bcd2f7dbf662fc497ad8de747f1a2fedd2f2f4a Mon Sep 17 00:00:00 2001 From: sua yoo Date: Mon, 26 May 2025 15:17:58 -0700 Subject: [PATCH 19/34] update actions --- frontend/src/pages/org/workflow-detail.ts | 243 ++++++++++++---------- 1 file changed, 130 insertions(+), 113 deletions(-) diff --git a/frontend/src/pages/org/workflow-detail.ts b/frontend/src/pages/org/workflow-detail.ts index 4d202ff407..09d788bbf0 100644 --- a/frontend/src/pages/org/workflow-detail.ts +++ b/frontend/src/pages/org/workflow-detail.ts @@ -625,7 +625,7 @@ export class WorkflowDetail extends BtrixElement { class="mb-2 flex h-7 items-end justify-between text-lg font-medium" >

${this.tabLabels[tab]}

- ${this.renderPanelAction()} +
${this.renderPanelAction()}
${this.renderTab(WorkflowTab.LatestCrawl)} @@ -648,77 +648,55 @@ export class WorkflowDetail extends BtrixElement { const latestCrawl = this.latestCrawlTask.value; if (this.groupedWorkflowTab === WorkflowTab.LatestCrawl && latestCrawl) { - const archivedItemId = this.archivedItemId; const logTotals = this.logTotalsTask.value; - const hasLogs = logTotals?.errors || logTotals?.behaviors; - const disableDownload = - this.workflow?.isCrawlRunning && - this.workflow.lastCrawlState !== "paused"; - const authToken = this.authState?.headers.Authorization.split(" ")[1]; + const disableDownload = this.workflow?.isCrawlRunning; - return html`
- + - - - - - - ${when( - hasLogs, - () => html` - - - - - - `, - )} - ${when( - this.workflow?.isCrawlRunning || this.archivedItemId, - () => html` - + - + ${msg("Download")} + + + - - - - `, - )} -
`; + + ${msg("Item")} + + + + ${msg("Log")} + + + + + `; } if (this.workflowTab === WorkflowTab.Settings && this.isCrawler) { @@ -993,7 +971,9 @@ export class WorkflowDetail extends BtrixElement { workflow.lastCrawlId, () => html` - ${this.renderLatestCrawlMenuOptions()} + + ${this.tabLabels.latest} ${this.renderLatestCrawlMenu()} + `, )} @@ -1011,14 +991,7 @@ export class WorkflowDetail extends BtrixElement { ${msg("Copy Workflow ID")} - - ClipboardController.copyToClipboard(workflow.lastCrawlId || "")} - ?disabled=${!workflow.lastCrawlId} - > - - ${msg("Copy Latest Crawl ID")} - + ${when( !workflow.crawlCount, () => html` @@ -1037,49 +1010,63 @@ export class WorkflowDetail extends BtrixElement { `; }; - private renderLatestCrawlMenuOptions() { + private renderLatestCrawlMenu() { const authToken = this.authState?.headers.Authorization.split(" ")[1]; const latestCrawl = this.latestCrawlTask.value; const logTotals = this.logTotalsTask.value; return html` - - - ${msg("Download Latest Crawl")} - ${latestCrawl?.fileSize - ? html` ${this.localize.bytes(latestCrawl.fileSize)}` - : nothing} - - - - - ${msg("Download Latest Crawl Log")} - + + + + ${msg("Download Item")} + ${latestCrawl?.fileSize + ? html` ${this.localize.bytes(latestCrawl.fileSize)}` + : nothing} + - ${when( - this.archivedItemId, - (id) => html` - - this.navigate.to(`${this.basePath}/${WorkflowTab.Crawls}/${id}`)} - > - - ${msg("Go to Archived Item")} - - `, - )} + + + ${msg("Download Log")} + + + + + ${when( + this.archivedItemId, + (id) => html` + + this.navigate.to( + `${this.basePath}/${WorkflowTab.Crawls}/${id}`, + )} + > + + ${msg("View Item Details")} + + `, + )} + + ClipboardController.copyToClipboard(this.lastCrawlId || "")} + ?disabled=${!this.lastCrawlId} + > + + ${msg("Copy Item ID")} + + `; } @@ -1384,8 +1371,37 @@ export class WorkflowDetail extends BtrixElement { ` : nothing} + ${when( + this.archivedItemId, + (id) => html` + + + + ${msg("More")} + + + + + ${msg("Metadata")} + + + + + ${msg("Quality Assurance")} + + + + + + ${msg("WACZ Files")} + + + + + `, + )} -
+
${this.renderLatestCrawlAction()}
@@ -1503,10 +1519,11 @@ export class WorkflowDetail extends BtrixElement { if (!this.workflow || !this.lastCrawlId) return; if ( - this.isCrawler && this.workflow.isCrawlRunning && this.workflow.lastCrawlState !== "paused" ) { + if (!this.isCrawler) return; + const enableEditBrowserWindows = !this.workflow.lastCrawlStopping; const windowCount = this.workflow.scale * (this.appState.settings?.numBrowsers || 1); From 2278061b225ae90cf1b116276c30d66cab9d0fd0 Mon Sep 17 00:00:00 2001 From: sua yoo Date: Mon, 26 May 2025 15:22:21 -0700 Subject: [PATCH 20/34] handle pause --- frontend/src/pages/org/workflow-detail.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/frontend/src/pages/org/workflow-detail.ts b/frontend/src/pages/org/workflow-detail.ts index 09d788bbf0..aa6c2090bc 100644 --- a/frontend/src/pages/org/workflow-detail.ts +++ b/frontend/src/pages/org/workflow-detail.ts @@ -650,7 +650,9 @@ export class WorkflowDetail extends BtrixElement { if (this.groupedWorkflowTab === WorkflowTab.LatestCrawl && latestCrawl) { const logTotals = this.logTotalsTask.value; const authToken = this.authState?.headers.Authorization.split(" ")[1]; - const disableDownload = this.workflow?.isCrawlRunning; + const disableDownload = + this.workflow?.isCrawlRunning && + this.workflow.lastCrawlState !== "paused"; return html` Date: Tue, 27 May 2025 14:06:05 -0700 Subject: [PATCH 21/34] allow other running states to be set while pausing --- backend/btrixcloud/operator/crawls.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/backend/btrixcloud/operator/crawls.py b/backend/btrixcloud/operator/crawls.py index 9fc69b39a7..7e1b68eac0 100644 --- a/backend/btrixcloud/operator/crawls.py +++ b/backend/btrixcloud/operator/crawls.py @@ -1494,6 +1494,17 @@ async def update_crawl_state( # all expected pods are either done or failed all_completed = (num_done + num_failed) >= status.scale + # check paused + if not all_completed and crawl.paused_at and status.stopReason == "paused": + num_paused = status_count.get("interrupted", 0) + if (num_paused + num_failed) >= status.scale: + # now fully paused! + # remove pausing key and set state to paused + await redis.delete(f"{crawl.id}:paused") + await self.set_state( + "paused", status, crawl, allowed_from=RUNNING_AND_WAITING_STATES + ) + # if at least one is done according to redis, consider crawl successful # ensure pod successfully exited as well # pylint: disable=chained-comparison @@ -1526,17 +1537,6 @@ async def update_crawl_state( else: await self.fail_crawl(crawl, status, pods, stats) - # check paused - elif crawl.paused_at and status.stopReason == "paused": - num_paused = status_count.get("interrupted", 0) - if (num_paused + num_failed) >= status.scale: - # now fully paused! - # remove pausing key and set state to paused - await redis.delete(f"{crawl.id}:paused") - await self.set_state( - "paused", status, crawl, allowed_from=RUNNING_AND_WAITING_STATES - ) - # check for other statuses, default to "running" else: new_status: TYPE_RUNNING_STATES = "running" From 2903b92b2d3e6dcdbfe97d2f094431877388bde9 Mon Sep 17 00:00:00 2001 From: sua yoo Date: Tue, 27 May 2025 12:13:02 -0700 Subject: [PATCH 22/34] update labels --- .../src/features/org/usage-history-table.ts | 10 +- frontend/src/pages/org/workflow-detail.ts | 96 ++++++++++++------- frontend/src/types/crawler.ts | 2 + 3 files changed, 70 insertions(+), 38 deletions(-) diff --git a/frontend/src/features/org/usage-history-table.ts b/frontend/src/features/org/usage-history-table.ts index 35be985b4e..4b2db378e4 100644 --- a/frontend/src/features/org/usage-history-table.ts +++ b/frontend/src/features/org/usage-history-table.ts @@ -64,14 +64,14 @@ export class UsageHistoryTable extends BtrixElement { field: Field.ElapsedTime, label: msg("Elapsed Time"), description: msg( - "Total duration of crawls and QA analysis runs, from start to finish", + "Total duration of workflow and QA analysis runs, from start to finish.", ), }, { field: Field.ExecutionTime, label: msg("Execution Time"), description: msg( - "Aggregated time across all browser windows that the crawler was actively executing a crawl or QA analysis run, i.e. not waiting or paused", + "Aggregated time across all browser windows that the crawler was actively executing a crawl or QA analysis run, i.e. not waiting or paused.", ), }, ]; @@ -81,7 +81,7 @@ export class UsageHistoryTable extends BtrixElement { field: Field.BillableExecutionTime, label: msg("Billable Execution Time"), description: msg( - "Execution time used that is billable to the current month of the plan", + "Execution time used that is billable to the current month of the plan.", ), }); } @@ -90,7 +90,7 @@ export class UsageHistoryTable extends BtrixElement { field: Field.RolloverExecutionTime, label: msg("Rollover Execution Time"), description: msg( - "Additional execution time used, of which any extra minutes will roll over to next month as billable time", + "Additional execution time used, of which any extra minutes will roll over to next month as billable time.", ), }); } @@ -98,7 +98,7 @@ export class UsageHistoryTable extends BtrixElement { cols.push({ field: Field.GiftedExecutionTime, label: msg("Gifted Execution Time"), - description: msg("Execution time used that is free of charge"), + description: msg("Execution time used that is free of charge."), }); } diff --git a/frontend/src/pages/org/workflow-detail.ts b/frontend/src/pages/org/workflow-detail.ts index aa6c2090bc..5a651aa56e 100644 --- a/frontend/src/pages/org/workflow-detail.ts +++ b/frontend/src/pages/org/workflow-detail.ts @@ -648,6 +648,7 @@ export class WorkflowDetail extends BtrixElement { const latestCrawl = this.latestCrawlTask.value; if (this.groupedWorkflowTab === WorkflowTab.LatestCrawl && latestCrawl) { + const latestCrawlId = latestCrawl.id; const logTotals = this.logTotalsTask.value; const authToken = this.authState?.headers.Authorization.split(" ")[1]; const disableDownload = @@ -657,46 +658,78 @@ export class WorkflowDetail extends BtrixElement { return html` - - + - - ${msg("Download")} - - - - ${msg("Item")} - - + + + - - ${msg("Log")} - - - + ${msg("Download options")} + + + + + ${msg("Item")} + ${latestCrawl.fileSize + ? html` ${this.localize.bytes( + latestCrawl.fileSize, + )}` + : nothing} + + + + ${msg("Log")} + + + + `; } @@ -1384,19 +1417,16 @@ export class WorkflowDetail extends BtrixElement { - ${msg("Metadata")} - + ${msg("View Metadata")} - ${msg("Quality Assurance")} - + ${msg("View Quality Assurance")} - ${msg("WACZ Files")} - + ${msg("View WACZ Files")} diff --git a/frontend/src/types/crawler.ts b/frontend/src/types/crawler.ts index ac6e098fa2..f348bb5559 100644 --- a/frontend/src/types/crawler.ts +++ b/frontend/src/types/crawler.ts @@ -92,6 +92,8 @@ export type Workflow = CrawlConfig & { lastCrawlSize: number | null; lastStartedByName: string | null; lastCrawlStopping: boolean | null; + // User has requested pause, but actual state can be running or paused + // OR user has requested resume, but actual state is not running lastCrawlShouldPause: boolean | null; lastCrawlPausedAt: string | null; lastCrawlPausedExpiry: string | null; From 05d9271e364ad89fbfa961746c889928b28f9c8d Mon Sep 17 00:00:00 2001 From: sua yoo Date: Tue, 27 May 2025 13:23:35 -0700 Subject: [PATCH 23/34] update state renderer --- frontend/src/components/detail-page-title.ts | 2 +- .../features/archived-items/archived-item-list.ts | 6 ++++-- .../src/features/archived-items/crawl-status.ts | 13 ++++++++----- frontend/src/pages/crawls.ts | 2 +- frontend/src/pages/org/archived-items.ts | 2 +- frontend/src/pages/org/workflow-detail.ts | 2 +- 6 files changed, 16 insertions(+), 11 deletions(-) diff --git a/frontend/src/components/detail-page-title.ts b/frontend/src/components/detail-page-title.ts index 41018d6aa0..db9c34ee8e 100644 --- a/frontend/src/components/detail-page-title.ts +++ b/frontend/src/components/detail-page-title.ts @@ -70,7 +70,7 @@ export class DetailPageTitle extends TailwindElement { private renderIcon() { if (!this.item?.state) return; - const crawlStatus = CrawlStatus.getContent(this.item.state, this.item.type); + const crawlStatus = CrawlStatus.getContent(this.item); let icon = html` { - const { icon, label } = CrawlStatus.getContent(state); + const { icon, label } = CrawlStatus.getContent({ state }); return html`${icon}${label}`; }; diff --git a/frontend/src/pages/org/archived-items.ts b/frontend/src/pages/org/archived-items.ts index ca07b608e5..42e3bd06b5 100644 --- a/frontend/src/pages/org/archived-items.ts +++ b/frontend/src/pages/org/archived-items.ts @@ -669,7 +669,7 @@ export class CrawlsList extends BtrixElement { }; private readonly renderStatusMenuItem = (state: CrawlState) => { - const { icon, label } = CrawlStatus.getContent(state); + const { icon, label } = CrawlStatus.getContent({ state }); return html`${icon}${label}`; }; diff --git a/frontend/src/pages/org/workflow-detail.ts b/frontend/src/pages/org/workflow-detail.ts index 5a651aa56e..c3b72ced03 100644 --- a/frontend/src/pages/org/workflow-detail.ts +++ b/frontend/src/pages/org/workflow-detail.ts @@ -1353,7 +1353,7 @@ export class WorkflowDetail extends BtrixElement { } private readonly renderStatusMenuItem = (state: CrawlState) => { - const { icon, label } = CrawlStatus.getContent(state); + const { icon, label } = CrawlStatus.getContent({ state }); return html`${icon}${label}`; }; From 17e4af8d47f6eaf79619aac75bc411e2188692d5 Mon Sep 17 00:00:00 2001 From: sua yoo Date: Tue, 27 May 2025 13:50:25 -0700 Subject: [PATCH 24/34] update state --- .../features/archived-items/crawl-status.ts | 35 +++++++++++++++---- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/frontend/src/features/archived-items/crawl-status.ts b/frontend/src/features/archived-items/crawl-status.ts index 077bb559c7..ae0adce0a9 100644 --- a/frontend/src/features/archived-items/crawl-status.ts +++ b/frontend/src/features/archived-items/crawl-status.ts @@ -54,9 +54,12 @@ export class CrawlStatus extends TailwindElement { // instead of separate utility function? static getContent({ state, + originalState, type = "crawl", }: { state?: CrawlState | AnyString; + // `state` might be composed status + originalState?: CrawlState | AnyString; type?: CrawlType | undefined; }): { icon: TemplateResult; @@ -71,6 +74,9 @@ export class CrawlStatus extends TailwindElement { style="color: ${color}" >`; let label = ""; + let reason = ""; + + console.log(state, originalState); switch (state) { case "starting": @@ -94,10 +100,11 @@ export class CrawlStatus extends TailwindElement { slot="prefix" style="color: ${color}" >`; - label = - state === "waiting_capacity" - ? msg("Waiting (At Capacity)") - : msg("Waiting (Crawl Limit)"); + label = msg("Waiting"); + reason = + originalState === "waiting_capacity" + ? msg("At Capacity") + : msg("At Crawl Limit"); break; case "running": @@ -133,6 +140,12 @@ export class CrawlStatus extends TailwindElement { style="color: ${color}" >`; label = msg("Pausing"); + reason = + originalState === "pending-wait" + ? msg("Finishing Downloads") + : originalState?.endsWith("-wacz") + ? msg("Creating WACZ") + : ""; break; case "resuming": @@ -165,7 +178,7 @@ export class CrawlStatus extends TailwindElement { slot="prefix" style="color: ${color}" >`; - label = msg("Finishing Crawl"); + label = msg("Finishing Downloads"); break; case "generate-wacz": @@ -303,7 +316,11 @@ export class CrawlStatus extends TailwindElement { } break; } - return { icon, label, cssColor: color }; + return { + icon, + label: reason ? `${label} (${reason})` : label, + cssColor: color, + }; } filterState() { @@ -321,7 +338,11 @@ export class CrawlStatus extends TailwindElement { render() { const state = this.filterState(); - const { icon, label } = CrawlStatus.getContent({ state, type: this.type }); + const { icon, label } = CrawlStatus.getContent({ + state, + originalState: this.state, + type: this.type, + }); if (this.hideLabel) { return html`
Date: Tue, 27 May 2025 14:52:31 -0700 Subject: [PATCH 25/34] update exec seconds display --- frontend/src/utils/executionTimeFormatter.test.ts | 7 +++++-- frontend/src/utils/executionTimeFormatter.ts | 9 +++++---- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/frontend/src/utils/executionTimeFormatter.test.ts b/frontend/src/utils/executionTimeFormatter.test.ts index c5318b97cf..8aaaaf3093 100644 --- a/frontend/src/utils/executionTimeFormatter.test.ts +++ b/frontend/src/utils/executionTimeFormatter.test.ts @@ -12,6 +12,9 @@ describe("formatHours", () => { it("returns 1m when given a time under a minute", () => { expect(humanizeSeconds(24, "en-US")).to.equal("1m"); }); + it("returns seconds given a time under a minute when not rounding", () => { + expect(humanizeSeconds(24, "en-US")).to.equal("1m"); + }); it("returns 0m and seconds when given a time under a minute with seconds on", () => { expect(humanizeSeconds(24, "en-US", true)).to.equal("0m 24s"); }); @@ -105,7 +108,7 @@ describe("humanizeExecutionSeconds", () => { parentNode, }, ); - expect(el.textContent?.trim()).to.equal("1 minute\u00a0(0m 24s)"); - expect(parentNode.innerText).to.equal("1 minute\u00a0(0m 24s)"); + expect(el.textContent?.trim()).to.equal("<1 minute\u00a0(0m 24s)"); + expect(parentNode.innerText).to.equal("<1 minute\u00a0(0m 24s)"); }); }); diff --git a/frontend/src/utils/executionTimeFormatter.ts b/frontend/src/utils/executionTimeFormatter.ts index ee8e7d8a54..8f50f0089d 100644 --- a/frontend/src/utils/executionTimeFormatter.ts +++ b/frontend/src/utils/executionTimeFormatter.ts @@ -88,7 +88,7 @@ export const humanizeExecutionSeconds = ( ) => { const { style = "long", - displaySeconds = false, + displaySeconds = seconds < 60, round = "up", } = options || {}; const locale = localize.activeLanguage; @@ -110,7 +110,9 @@ export const humanizeExecutionSeconds = ( }); const details = humanizeSeconds(seconds, locale, displaySeconds); - const compactMinutes = compactMinuteFormatter.format(minutes); + const compactMinutes = + (displaySeconds && seconds < 60 ? "<" : "") + + compactMinuteFormatter.format(minutes); const fullMinutes = longMinuteFormatter.format(minutes); // if the time is less than an hour and lines up exactly on the minute, don't render the details. @@ -126,8 +128,7 @@ export const humanizeExecutionSeconds = ( title="${ifDefined( fullMinutes !== compactMinutes ? fullMinutes : undefined, )}" - > - ${compactMinutes}${formattedDetails}${compactMinutes}${formattedDetails}`; case "short": return html` Date: Tue, 27 May 2025 15:23:30 -0700 Subject: [PATCH 26/34] show exec time --- .../src/features/archived-items/crawl-list.ts | 17 +++- .../features/archived-items/crawl-status.ts | 2 - .../src/features/org/usage-history-table.ts | 4 +- .../archived-item-detail.ts | 2 +- frontend/src/pages/org/workflow-detail.ts | 81 +++++++++++++++---- 5 files changed, 82 insertions(+), 24 deletions(-) diff --git a/frontend/src/features/archived-items/crawl-list.ts b/frontend/src/features/archived-items/crawl-list.ts index 9b23456fbc..da5fe70f77 100644 --- a/frontend/src/features/archived-items/crawl-list.ts +++ b/frontend/src/features/archived-items/crawl-list.ts @@ -26,6 +26,7 @@ import { TailwindElement } from "@/classes/TailwindElement"; import type { OverflowDropdown } from "@/components/ui/overflow-dropdown"; import type { Crawl } from "@/types/crawler"; import { renderName } from "@/utils/crawler"; +import { humanizeExecutionSeconds } from "@/utils/executionTimeFormatter"; /** * @slot menu @@ -184,6 +185,11 @@ export class CrawlListItem extends BtrixElement { ), )} + + ${this.safeRender((crawl) => + humanizeExecutionSeconds(crawl.crawlExecSeconds), + )} + ${this.safeRender((crawl) => { const pagesFound = +(crawl.stats?.found || 0); @@ -284,8 +290,8 @@ export class CrawlList extends TailwindElement { return html` @@ -308,12 +314,15 @@ export class CrawlList extends TailwindElement { ${msg("Finished")} ${msg("Elapsed Time")}${msg("Run Duration")} + ${msg("Execution Time")} ${msg("Pages")} ${msg("Size")} - ${msg("Created By")} + ${msg("Run By")} ${msg("Row actions")} diff --git a/frontend/src/features/archived-items/crawl-status.ts b/frontend/src/features/archived-items/crawl-status.ts index ae0adce0a9..4ce68bdabf 100644 --- a/frontend/src/features/archived-items/crawl-status.ts +++ b/frontend/src/features/archived-items/crawl-status.ts @@ -76,8 +76,6 @@ export class CrawlStatus extends TailwindElement { let label = ""; let reason = ""; - console.log(state, originalState); - switch (state) { case "starting": color = "var(--sl-color-violet-600)"; diff --git a/frontend/src/features/org/usage-history-table.ts b/frontend/src/features/org/usage-history-table.ts index 4b2db378e4..45f76c35a7 100644 --- a/frontend/src/features/org/usage-history-table.ts +++ b/frontend/src/features/org/usage-history-table.ts @@ -63,9 +63,7 @@ export class UsageHistoryTable extends BtrixElement { { field: Field.ElapsedTime, label: msg("Elapsed Time"), - description: msg( - "Total duration of workflow and QA analysis runs, from start to finish.", - ), + description: msg("Total duration of workflow and QA analysis runs."), }, { field: Field.ExecutionTime, diff --git a/frontend/src/pages/org/archived-item-detail/archived-item-detail.ts b/frontend/src/pages/org/archived-item-detail/archived-item-detail.ts index a293a60266..f9826d6b5a 100644 --- a/frontend/src/pages/org/archived-item-detail/archived-item-detail.ts +++ b/frontend/src/pages/org/archived-item-detail/archived-item-detail.ts @@ -802,7 +802,7 @@ export class ArchivedItemDetail extends BtrixElement { ? this.formattedFinishedDate : html`${msg("Pending")}`} - + ${this.item!.finished ? html`${this.localize.humanizeDuration( new Date(this.item!.finished).valueOf() - diff --git a/frontend/src/pages/org/workflow-detail.ts b/frontend/src/pages/org/workflow-detail.ts index c3b72ced03..51820eae5f 100644 --- a/frontend/src/pages/org/workflow-detail.ts +++ b/frontend/src/pages/org/workflow-detail.ts @@ -37,6 +37,7 @@ import { isSuccessfullyFinished, } from "@/utils/crawler"; import { humanizeSchedule } from "@/utils/cron"; +import { humanizeExecutionSeconds } from "@/utils/executionTimeFormatter"; import { isArchivingDisabled } from "@/utils/orgs"; import { pluralOf } from "@/utils/pluralize"; import { tw } from "@/utils/tailwind"; @@ -1118,12 +1119,13 @@ export class WorkflowDetail extends BtrixElement { > `, )} - ${this.renderDetailItem( - msg("Total Size"), - (workflow) => - html` ${this.localize.bytes(Number(workflow.totalSize), { - unitDisplay: "narrow", - })}`, + ${this.renderDetailItem(msg("Last Run"), (workflow) => + workflow.lastRun + ? html`` + : html`${msg("Never")}`, )} ${this.renderDetailItem(msg("Schedule"), (workflow) => workflow.schedule @@ -1134,12 +1136,19 @@ export class WorkflowDetail extends BtrixElement { })}
` - : html`${msg("No Schedule")}`, + : html`${msg("None")}`, + )} + ${this.renderDetailItem(msg("Total Size"), (workflow) => + workflow.lastRun + ? html` ${this.localize.bytes(Number(workflow.totalSize), { + unitDisplay: "narrow", + })}` + : noData, )} - ${this.renderDetailItem(msg("Created By"), (workflow) => + ${this.renderDetailItem(msg("Last Modified By"), (workflow) => msg( - str`${workflow.createdByName} on ${this.localize.date( - new Date(workflow.created), + str`${workflow.modifiedByName} on ${this.localize.date( + new Date(workflow.modified), { year: "numeric", month: "numeric", @@ -1415,16 +1424,20 @@ export class WorkflowDetail extends BtrixElement { ${msg("More")} - + ${msg("View Metadata")} - + ${msg("View Quality Assurance")} - + ${msg("View WACZ Files")} @@ -1600,6 +1613,36 @@ export class WorkflowDetail extends BtrixElement { ); }; + const execTime = (workflow: Workflow) => { + if (!latestCrawl) return skeleton; + + if (workflow.isCrawlRunning && workflow.lastCrawlState !== "paused") { + return html` + ${noData} + + + + `; + } + + if (latestCrawl.crawlExecSeconds < 60) { + return this.localize.humanizeDuration( + latestCrawl.crawlExecSeconds * 1000, + ); + } + + return humanizeExecutionSeconds(latestCrawl.crawlExecSeconds, { + style: "short", + }); + }; + const pages = (workflow: Workflow) => { if (!latestCrawl) return skeleton; @@ -1650,7 +1693,7 @@ export class WorkflowDetail extends BtrixElement { return html` - ${this.renderDetailItem(msg("Elapsed Time"), (workflow) => + ${this.renderDetailItem(msg("Run Duration"), (workflow) => isLoading(this.runNowTask) ? html`${until( this.runNowTask.taskComplete.then((workflow) => @@ -1660,6 +1703,16 @@ export class WorkflowDetail extends BtrixElement { )}` : duration(workflow), )} + ${this.renderDetailItem(msg("Execution Time"), (workflow) => + isLoading(this.runNowTask) + ? html`${until( + this.runNowTask.taskComplete.then((workflow) => + workflow ? execTime(workflow) : noData, + ), + html``, + )}` + : execTime(workflow), + )} ${this.renderDetailItem(msg("Pages Crawled"), pages)} ${this.renderDetailItem(msg("Size"), (workflow) => this.localize.bytes(workflow.lastCrawlSize || 0, { From 1a564dbb2875978fe1fde89d371eeae777962987 Mon Sep 17 00:00:00 2001 From: sua yoo Date: Tue, 27 May 2025 15:36:12 -0700 Subject: [PATCH 27/34] update docs --- frontend/docs/docs/user-guide/running-crawl.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frontend/docs/docs/user-guide/running-crawl.md b/frontend/docs/docs/user-guide/running-crawl.md index 68583be68b..68f7422efe 100644 --- a/frontend/docs/docs/user-guide/running-crawl.md +++ b/frontend/docs/docs/user-guide/running-crawl.md @@ -12,7 +12,8 @@ A crawl workflow that is in progress can be in one of the following states: | :btrix-status-dot: Starting | New resources are starting up. Crawling should begin shortly.| | :btrix-status-dot: Running | The crawler is finding and capturing pages! | | :btrix-status-dot: Stopping | A user has instructed this workflow to stop. Finishing capture of the current pages.| -| :btrix-status-dot: Finishing Crawl | The workflow has finished crawling and data is being packaged into WACZ files.| +| :btrix-status-dot: Finishing Downloads | The workflow has finished crawling and is finalizing downloads.| +| :btrix-status-dot: Generating WACZ | Data is being packaged into WACZ files.| | :btrix-status-dot: Uploading WACZ | WACZ files have been created and are being transferred to storage.| ## Watch Crawl From 60bb539892c33ad3a28b5d684d99d5ebd91f8684 Mon Sep 17 00:00:00 2001 From: sua yoo Date: Tue, 27 May 2025 15:41:10 -0700 Subject: [PATCH 28/34] fix typo --- frontend/docs/docs/user-guide/running-crawl.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/docs/docs/user-guide/running-crawl.md b/frontend/docs/docs/user-guide/running-crawl.md index 68f7422efe..df68c37137 100644 --- a/frontend/docs/docs/user-guide/running-crawl.md +++ b/frontend/docs/docs/user-guide/running-crawl.md @@ -1,6 +1,6 @@ # Modifying Running Crawls -Running crawls can be modified from the crawl workflow **Latest Crawl** tab. You may want to modify a runnning crawl if you find that the workflow is crawling pages that you didn't intend to archive, or if you want a boost of speed. +Running crawls can be modified from the crawl workflow **Latest Crawl** tab. You may want to modify a running crawl if you find that the workflow is crawling pages that you didn't intend to archive, or if you want a boost of speed. ## Crawl Workflow Status From bc7e4211203033e26bc6c381f4aac3349022b7ed Mon Sep 17 00:00:00 2001 From: sua yoo Date: Tue, 27 May 2025 18:18:14 -0700 Subject: [PATCH 29/34] fix exec time prefix --- frontend/src/utils/executionTimeFormatter.test.ts | 13 +++++++++++++ frontend/src/utils/executionTimeFormatter.ts | 9 ++++----- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/frontend/src/utils/executionTimeFormatter.test.ts b/frontend/src/utils/executionTimeFormatter.test.ts index 8aaaaf3093..f4e178e07b 100644 --- a/frontend/src/utils/executionTimeFormatter.test.ts +++ b/frontend/src/utils/executionTimeFormatter.test.ts @@ -111,4 +111,17 @@ describe("humanizeExecutionSeconds", () => { expect(el.textContent?.trim()).to.equal("<1 minute\u00a0(0m 24s)"); expect(parentNode.innerText).to.equal("<1 minute\u00a0(0m 24s)"); }); + it("formats zero seconds", async () => { + const parentNode = document.createElement("div"); + const el = await fixture( + humanizeExecutionSeconds(0, { + displaySeconds: true, + }), + { + parentNode, + }, + ); + expect(el.textContent?.trim()).to.equal("0 minutes"); + expect(parentNode.innerText).to.equal("0 minutes"); + }); }); diff --git a/frontend/src/utils/executionTimeFormatter.ts b/frontend/src/utils/executionTimeFormatter.ts index 8f50f0089d..d80afdc783 100644 --- a/frontend/src/utils/executionTimeFormatter.ts +++ b/frontend/src/utils/executionTimeFormatter.ts @@ -110,9 +110,7 @@ export const humanizeExecutionSeconds = ( }); const details = humanizeSeconds(seconds, locale, displaySeconds); - const compactMinutes = - (displaySeconds && seconds < 60 ? "<" : "") + - compactMinuteFormatter.format(minutes); + const compactMinutes = compactMinuteFormatter.format(minutes); const fullMinutes = longMinuteFormatter.format(minutes); // if the time is less than an hour and lines up exactly on the minute, don't render the details. @@ -121,6 +119,7 @@ export const humanizeExecutionSeconds = ( : Math.floor(seconds / 60) === 0 && seconds % 60 !== 0; const formattedDetails = detailsRelevant || seconds > 3600 ? `\u00a0(${details})` : nothing; + const prefix = detailsRelevant && seconds < 60 ? "<" : ""; switch (style) { case "long": @@ -128,12 +127,12 @@ export const humanizeExecutionSeconds = ( title="${ifDefined( fullMinutes !== compactMinutes ? fullMinutes : undefined, )}" - >${compactMinutes}${formattedDetails}${prefix}${compactMinutes}${formattedDetails}`; case "short": return html`${compactMinutes}${prefix}${compactMinutes}`; } }; From 2d183273902e3e6294e52e7a2df225c125045ee5 Mon Sep 17 00:00:00 2001 From: sua yoo Date: Tue, 27 May 2025 18:33:52 -0700 Subject: [PATCH 30/34] replace icon for "More" --- frontend/src/pages/org/workflow-detail.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/src/pages/org/workflow-detail.ts b/frontend/src/pages/org/workflow-detail.ts index 51820eae5f..92f9b36874 100644 --- a/frontend/src/pages/org/workflow-detail.ts +++ b/frontend/src/pages/org/workflow-detail.ts @@ -1420,8 +1420,8 @@ export class WorkflowDetail extends BtrixElement { (id) => html` - - ${msg("More")} + + ${msg("More Info")} Date: Tue, 27 May 2025 19:37:04 -0700 Subject: [PATCH 31/34] return status when finally 'paused' --- backend/btrixcloud/operator/crawls.py | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/btrixcloud/operator/crawls.py b/backend/btrixcloud/operator/crawls.py index 7e1b68eac0..8a7b5e0995 100644 --- a/backend/btrixcloud/operator/crawls.py +++ b/backend/btrixcloud/operator/crawls.py @@ -1504,6 +1504,7 @@ async def update_crawl_state( await self.set_state( "paused", status, crawl, allowed_from=RUNNING_AND_WAITING_STATES ) + return status # if at least one is done according to redis, consider crawl successful # ensure pod successfully exited as well From 0808f06972ed8a692468507f67f81a358c992f84 Mon Sep 17 00:00:00 2001 From: sua yoo Date: Wed, 28 May 2025 12:40:42 -0700 Subject: [PATCH 32/34] update states --- .../features/archived-items/crawl-status.ts | 2 +- frontend/src/pages/org/workflow-detail.ts | 141 ++++++++++-------- 2 files changed, 81 insertions(+), 62 deletions(-) diff --git a/frontend/src/features/archived-items/crawl-status.ts b/frontend/src/features/archived-items/crawl-status.ts index 4ce68bdabf..ba0e485b87 100644 --- a/frontend/src/features/archived-items/crawl-status.ts +++ b/frontend/src/features/archived-items/crawl-status.ts @@ -325,7 +325,7 @@ export class CrawlStatus extends TailwindElement { if (this.stopping && this.state === "running") { return "stopping"; } - if (this.shouldPause && this.state === "running") { + if (this.shouldPause && this.state !== "paused") { return "pausing"; } if (!this.shouldPause && this.state === "paused") { diff --git a/frontend/src/pages/org/workflow-detail.ts b/frontend/src/pages/org/workflow-detail.ts index 92f9b36874..2f6b796122 100644 --- a/frontend/src/pages/org/workflow-detail.ts +++ b/frontend/src/pages/org/workflow-detail.ts @@ -26,7 +26,7 @@ import { ExclusionEditor } from "@/features/crawl-workflows/exclusion-editor"; import { pageError } from "@/layouts/pageError"; import { pageNav, type Breadcrumb } from "@/layouts/pageHeader"; import { WorkflowTab } from "@/routes"; -import { deleteConfirmation, noData } from "@/strings/ui"; +import { deleteConfirmation, noData, notApplicable } from "@/strings/ui"; import type { APIPaginatedList, APIPaginationQuery } from "@/types/api"; import { type CrawlState } from "@/types/crawlState"; import { isApiError } from "@/utils/api"; @@ -194,14 +194,10 @@ export class WorkflowDetail extends BtrixElement { return window.setTimeout(async () => { void this.workflowTask.run(); - const workflow = await this.workflowTask.taskComplete; + await this.workflowTask.taskComplete; // Retrieve additional data based on current tab - if ( - workflow.isCrawlRunning && - !workflow.lastCrawlShouldPause && - workflow.lastCrawlState !== "paused" - ) { + if (this.isRunning) { switch (this.groupedWorkflowTab) { case WorkflowTab.LatestCrawl: { void this.latestCrawlTask.run(); @@ -315,11 +311,13 @@ export class WorkflowDetail extends BtrixElement { return true; } - private get isPendingStopOrCancel() { - return isLoading(this.stopTask) || isLoading(this.cancelTask); + // Workflow is active and not paused + private get isRunning() { + return this.workflow?.isCrawlRunning && !this.isPaused; } - private get isExplicitRunning() { + // Crawl is explicitly running + private get isCrawling() { return ( this.workflow?.isCrawlRunning && !this.workflow.lastCrawlStopping && @@ -327,6 +325,14 @@ export class WorkflowDetail extends BtrixElement { ); } + private get isPaused() { + return this.workflow?.lastCrawlState === "paused"; + } + + private get isResuming() { + return this.workflow?.lastCrawlShouldPause === false && this.isPaused; + } + // Differentiate between archived item and crawl ID, since // non-successful crawls do not show up in the archived item list. private get archivedItemId() { @@ -652,9 +658,7 @@ export class WorkflowDetail extends BtrixElement { const latestCrawlId = latestCrawl.id; const logTotals = this.logTotalsTask.value; const authToken = this.authState?.headers.Authorization.split(" ")[1]; - const disableDownload = - this.workflow?.isCrawlRunning && - this.workflow.lastCrawlState !== "paused"; + const disableDownload = this.isRunning; return html` html` ${when(!hidePauseResume && !disablePauseResume, () => - workflow.lastCrawlState === "paused" + paused ? html` (this.openDialogName = "exclusions")} - ?disabled=${!this.isExplicitRunning} + ?disabled=${!this.isCrawling} > ${msg("Edit Exclusions")} @@ -1107,6 +1111,36 @@ export class WorkflowDetail extends BtrixElement { } private renderDetails() { + const relativeDate = (dateStr: string) => { + const date = new Date(dateStr); + const diff = new Date().valueOf() - date.valueOf(); + const seconds = diff / 1000; + const minutes = seconds / 60; + const hours = minutes / 60; + + return html` + + ${hours > 24 + ? this.localize.date(date, { + year: "numeric", + month: "short", + day: "numeric", + }) + : seconds > 60 + ? html`` + : msg("Now")} + + `; + }; return html` ${this.renderDetailItem( @@ -1121,10 +1155,8 @@ export class WorkflowDetail extends BtrixElement { )} ${this.renderDetailItem(msg("Last Run"), (workflow) => workflow.lastRun - ? html`` + ? // TODO Use `lastStartedByName` when it's updated to be null for scheduled runs + relativeDate(workflow.lastRun) : html`${msg("Never")}`, )} ${this.renderDetailItem(msg("Schedule"), (workflow) => @@ -1143,19 +1175,13 @@ export class WorkflowDetail extends BtrixElement { ? html` ${this.localize.bytes(Number(workflow.totalSize), { unitDisplay: "narrow", })}` - : noData, + : notApplicable, )} - ${this.renderDetailItem(msg("Last Modified By"), (workflow) => - msg( - str`${workflow.modifiedByName} on ${this.localize.date( - new Date(workflow.modified), - { - year: "numeric", - month: "numeric", - day: "numeric", - }, - )}`, - ), + ${this.renderDetailItem( + msg("Last Modified"), + (workflow) => + html`${relativeDate(workflow.modified)} ${msg("by")} + ${workflow.modifiedByName}`, )} `; @@ -1250,17 +1276,15 @@ export class WorkflowDetail extends BtrixElement { () => html`
- ${msg("A crawl is currently in progress.")} + ${this.isRunning + ? msg("Workflow crawl is currently in progress.") + : msg("This workflow has an active crawl.")} - ${this.workflow && - (this.workflow.lastCrawlShouldPause || - this.workflow.lastCrawlState === "paused") - ? msg("View Crawl") - : msg("Watch Crawl")} + ${this.isRunning ? msg("Watch Crawl") : msg("View Crawl")}
`, @@ -1373,10 +1397,7 @@ export class WorkflowDetail extends BtrixElement { } const logTotals = this.logTotalsTask.value; - const showReplay = - this.workflow && - (!this.workflow.isCrawlRunning || - this.workflow.lastCrawlState === "paused"); + const showReplay = !this.isRunning; return html`
@@ -1475,7 +1496,8 @@ export class WorkflowDetail extends BtrixElement { ) => { if ( !this.workflow || - this.workflow.lastCrawlState !== "paused" || + !this.isPaused || + this.isResuming || !this.workflow.lastCrawlPausedExpiry ) return; @@ -1563,10 +1585,7 @@ export class WorkflowDetail extends BtrixElement { private renderLatestCrawlAction() { if (!this.workflow || !this.lastCrawlId) return; - if ( - this.workflow.isCrawlRunning && - this.workflow.lastCrawlState !== "paused" - ) { + if (this.isRunning) { if (!this.isCrawler) return; const enableEditBrowserWindows = !this.workflow.lastCrawlStopping; @@ -1613,10 +1632,10 @@ export class WorkflowDetail extends BtrixElement { ); }; - const execTime = (workflow: Workflow) => { + const execTime = () => { if (!latestCrawl) return skeleton; - if (workflow.isCrawlRunning && workflow.lastCrawlState !== "paused") { + if (this.isRunning) { return html` ${noData} + ${this.renderDetailItem(msg("Execution Time"), () => isLoading(this.runNowTask) ? html`${until( this.runNowTask.taskComplete.then((workflow) => - workflow ? execTime(workflow) : noData, + workflow ? execTime() : noData, ), html``, )}` - : execTime(workflow), + : execTime(), )} ${this.renderDetailItem(msg("Pages Crawled"), pages)} ${this.renderDetailItem(msg("Size"), (workflow) => @@ -1730,7 +1749,7 @@ export class WorkflowDetail extends BtrixElement { // Show custom message if crawl is active but not explicitly running let waitingMsg: string | null = null; - if (!this.isExplicitRunning) { + if (!this.isCrawling) { switch (this.workflow.lastCrawlState) { case "starting": waitingMsg = msg("Crawl starting..."); @@ -1766,7 +1785,7 @@ export class WorkflowDetail extends BtrixElement { return html` ${when( - this.isExplicitRunning && this.workflow, + this.isCrawling && this.workflow, (workflow) => html`
`, () => this.renderNoCrawlLogs(), @@ -2242,11 +2261,11 @@ export class WorkflowDetail extends BtrixElement { private async pauseResume(signal: AbortSignal) { if (!this.lastCrawlId) return; - const pause = this.workflow?.lastCrawlState !== "paused"; + const shouldPause = !this.isPaused; try { const data = await this.api.fetch<{ success: boolean }>( - `/orgs/${this.orgId}/crawls/${this.lastCrawlId}/${pause ? "pause" : "resume"}`, + `/orgs/${this.orgId}/crawls/${this.lastCrawlId}/${shouldPause ? "pause" : "resume"}`, { method: "POST", signal, @@ -2254,7 +2273,7 @@ export class WorkflowDetail extends BtrixElement { ); if (data.success) { this.notify.toast({ - message: pause + message: shouldPause ? msg("Pausing crawl.") : msg("Resuming paused crawl."), variant: "success", @@ -2266,7 +2285,7 @@ export class WorkflowDetail extends BtrixElement { } } catch { this.notify.toast({ - message: pause + message: shouldPause ? msg("Something went wrong, couldn't pause crawl.") : msg("Something went wrong, couldn't resume paused crawl."), variant: "danger", From 757ea1257358ed3f9fddbd3641b0790737f13acf Mon Sep 17 00:00:00 2001 From: sua yoo Date: Wed, 28 May 2025 12:44:25 -0700 Subject: [PATCH 33/34] hoist tooltip --- frontend/src/pages/org/workflow-detail.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/frontend/src/pages/org/workflow-detail.ts b/frontend/src/pages/org/workflow-detail.ts index 2f6b796122..48b57811cd 100644 --- a/frontend/src/pages/org/workflow-detail.ts +++ b/frontend/src/pages/org/workflow-detail.ts @@ -1128,6 +1128,8 @@ export class WorkflowDetail extends BtrixElement { minute: "2-digit", timeZoneName: "short", })} + hoist + placement="bottom" > ${hours > 24 ? this.localize.date(date, { @@ -1141,6 +1143,7 @@ export class WorkflowDetail extends BtrixElement { `; }; + return html` ${this.renderDetailItem( From 2d676d486450e31bb6bd5e29a8cad10c22f9541c Mon Sep 17 00:00:00 2001 From: sua yoo Date: Wed, 28 May 2025 18:42:13 -0700 Subject: [PATCH 34/34] use new popover --- frontend/src/components/ui/popover.ts | 9 ++++++--- frontend/src/pages/org/workflow-detail.ts | 23 +++++++++-------------- 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/frontend/src/components/ui/popover.ts b/frontend/src/components/ui/popover.ts index f4ea9f153b..c1f97da134 100644 --- a/frontend/src/components/ui/popover.ts +++ b/frontend/src/components/ui/popover.ts @@ -13,8 +13,10 @@ import { customElement, property } from "lit/decorators.js"; * * @attr {String} content * @attr {String} placement + * @attr {String} distance * @attr {String} trigger * @attr {Boolean} open + * @attr {Boolean} disabled */ @customElement("btrix-popover") @localized() @@ -29,8 +31,9 @@ export class Popover extends SlTooltip { slTooltipStyles, css` :host { - --btrix-border: 1px solid var(--sl-panel-border-color); - --sl-tooltip-background-color: var(--sl-color-neutral-0); + --btrix-border: 1px solid var(--sl-color-neutral-300); + --sl-tooltip-border-radius: var(--sl-border-radius-large); + --sl-tooltip-background-color: var(--sl-color-neutral-50); --sl-tooltip-color: var(--sl-color-neutral-700); --sl-tooltip-font-size: var(--sl-font-size-x-small); --sl-tooltip-padding: var(--sl-spacing-small); @@ -39,7 +42,7 @@ export class Popover extends SlTooltip { ::part(body) { border: var(--btrix-border); - box-shadow: var(--sl-shadow-medium); + box-shadow: var(--sl-shadow-small), var(--sl-shadow-large); } ::part(arrow) { diff --git a/frontend/src/pages/org/workflow-detail.ts b/frontend/src/pages/org/workflow-detail.ts index 48b57811cd..f90d6f0330 100644 --- a/frontend/src/pages/org/workflow-detail.ts +++ b/frontend/src/pages/org/workflow-detail.ts @@ -667,13 +667,12 @@ export class WorkflowDetail extends BtrixElement { content=${msg("Copy Item ID")} hoist > - - + `; } @@ -1641,16 +1640,14 @@ export class WorkflowDetail extends BtrixElement { if (this.isRunning) { return html` ${noData} - - + `; } @@ -1684,14 +1681,12 @@ export class WorkflowDetail extends BtrixElement { if (workflow.isCrawlRunning) { return html` ${noData} - - + `; }