Allow to cancel running external service sync jobs (#41518)

This PR makes use of dbworker cancellation which we recently added as a feature to allow to cancel a running external service sync, so customers don't need to shoot down their instance in case of misconfiguration (eg when syncing the entirety of github.com).

It also contains a couple of tweaks to honor context cancellation better in the sync jobs, so that we bail out early. Worker cancellation is async, and the UI handles that with a canceling state.

Author: eseliger

client/web/src/components/externalServices/ExternalServicePage.module.scss

@@ -0,0 +1,5 @@
+.cancel-button {
+    margin-right: 0.5rem;
+    justify-content: flex-end;
+    align-self: flex-start;
+}

client/web/src/components/externalServices/ExternalServicePage.story.tsx

@@ -1,12 +1,13 @@
 import { DecoratorFn, Story, Meta } from '@storybook/react'
+import { subMinutes } from 'date-fns'
 import { of } from 'rxjs'
 import { MATCH_ANY_PARAMETERS, WildcardMockLink } from 'wildcard-mock-link'
 
 import { getDocumentNode } from '@sourcegraph/http-client'
 import { NOOP_TELEMETRY_SERVICE } from '@sourcegraph/shared/src/telemetry/telemetryService'
 import { MockedTestProvider } from '@sourcegraph/shared/src/testing/apollo'
 
-import { ExternalServiceFields, ExternalServiceKind } from '../../graphql-operations'
+import { ExternalServiceFields, ExternalServiceKind, ExternalServiceSyncJobState } from '../../graphql-operations'
 import { WebStory } from '../WebStory'
 
 import { FETCH_EXTERNAL_SERVICE, queryExternalServiceSyncJobs as _queryExternalServiceSyncJobs } from './backend'
@@ -51,9 +52,42 @@ const externalService = {
 
 const queryExternalServiceSyncJobs: typeof _queryExternalServiceSyncJobs = () =>
     of({
-        totalCount: 0,
+        totalCount: 4,
         pageInfo: { endCursor: null, hasNextPage: false },
-        nodes: [],
+        nodes: [
+            {
+                __typename: 'ExternalServiceSyncJob',
+                failureMessage: null,
+                startedAt: subMinutes(new Date(), 25).toISOString(),
+                finishedAt: null,
+                id: 'SYNCJOB1',
+                state: ExternalServiceSyncJobState.CANCELING,
+            },
+            {
+                __typename: 'ExternalServiceSyncJob',
+                failureMessage: null,
+                startedAt: subMinutes(new Date(), 25).toISOString(),
+                finishedAt: null,
+                id: 'SYNCJOB1',
+                state: ExternalServiceSyncJobState.PROCESSING,
+            },
+            {
+                __typename: 'ExternalServiceSyncJob',
+                failureMessage: 'Very bad error syncing with the code host.',
+                startedAt: subMinutes(new Date(), 25).toISOString(),
+                finishedAt: subMinutes(new Date(), 25).toISOString(),
+                id: 'SYNCJOB1',
+                state: ExternalServiceSyncJobState.FAILED,
+            },
+            {
+                __typename: 'ExternalServiceSyncJob',
+                failureMessage: null,
+                startedAt: subMinutes(new Date(), 25).toISOString(),
+                finishedAt: subMinutes(new Date(), 25).toISOString(),
+                id: 'SYNCJOB1',
+                state: ExternalServiceSyncJobState.COMPLETED,
+            },
+        ],
     })
 
 function newFetchMock(node: { __typename: 'ExternalService' } & ExternalServiceFields): WildcardMockLink {

client/web/src/components/externalServices/ExternalServicePage.tsx

@@ -3,7 +3,8 @@ import React, { useEffect, useState, useCallback, useMemo } from 'react'
 import * as H from 'history'
 import { parse as parseJSONC } from 'jsonc-parser'
 import { Redirect, useHistory } from 'react-router'
-import { Observable, Subject } from 'rxjs'
+import { Subject } from 'rxjs'
+import { delay, repeatWhen } from 'rxjs/operators'
 
 import { ErrorAlert } from '@sourcegraph/branded/src/components/alerts'
 import { hasProperty } from '@sourcegraph/common'
@@ -20,6 +21,7 @@ import {
     ExternalServiceSyncJobConnectionFields,
     ExternalServiceResult,
     ExternalServiceVariables,
+    ExternalServiceSyncJobState,
 } from '../../graphql-operations'
 import { FilteredConnection, FilteredConnectionQueryArguments } from '../FilteredConnection'
 import { LoaderButton } from '../LoaderButton'
@@ -32,12 +34,15 @@ import {
     queryExternalServiceSyncJobs as _queryExternalServiceSyncJobs,
     useUpdateExternalService,
     FETCH_EXTERNAL_SERVICE,
+    useCancelExternalServiceSync,
 } from './backend'
 import { ExternalServiceCard } from './ExternalServiceCard'
 import { ExternalServiceForm } from './ExternalServiceForm'
 import { defaultExternalServices, codeHostExternalServices } from './externalServices'
 import { ExternalServiceWebhook } from './ExternalServiceWebhook'
 
+import styles from './ExternalServicePage.module.scss'
+
 interface Props extends TelemetryProps {
     externalServiceID: Scalars['ID']
     isLightTheme: boolean
@@ -235,7 +240,7 @@ export const ExternalServicePage: React.FunctionComponent<React.PropsWithChildre
 
 interface ExternalServiceSyncJobsListProps {
     externalServiceID: Scalars['ID']
-    updates: Observable<void>
+    updates: Subject<void>
 
     /** For testing only. */
     queryExternalServiceSyncJobs?: typeof _queryExternalServiceSyncJobs
@@ -251,7 +256,7 @@ const ExternalServiceSyncJobsList: React.FunctionComponent<ExternalServiceSyncJo
             queryExternalServiceSyncJobs({
                 first: args.first ?? null,
                 externalService: externalServiceID,
-            }),
+            }).pipe(repeatWhen(obs => obs.pipe(delay(1500)))),
         [externalServiceID, queryExternalServiceSyncJobs]
     )
 
@@ -272,7 +277,7 @@ const ExternalServiceSyncJobsList: React.FunctionComponent<ExternalServiceSyncJo
                 pluralNoun="sync jobs"
                 queryConnection={queryConnection}
                 nodeComponent={ExternalServiceSyncJobNode}
-                nodeComponentProps={{}}
+                nodeComponentProps={{ onUpdate: updates }}
                 hideSearch={true}
                 noSummaryIfAllNodesVisible={true}
                 history={history}
@@ -285,47 +290,83 @@ const ExternalServiceSyncJobsList: React.FunctionComponent<ExternalServiceSyncJo
 
 interface ExternalServiceSyncJobNodeProps {
     node: ExternalServiceSyncJobListFields
+    onUpdate: Subject<void>
 }
 
-const ExternalServiceSyncJobNode: React.FunctionComponent<ExternalServiceSyncJobNodeProps> = ({ node }) => (
-    <li className="list-group-item py-3">
-        <div className="d-flex align-items-center justify-content-between">
-            <div className="flex-shrink-0 mr-2">
-                <Badge>{node.state}</Badge>
-            </div>
-            <div className="flex-shrink-0">
-                {node.startedAt && (
-                    <>
-                        {node.finishedAt === null && <>Running since </>}
-                        {node.finishedAt !== null && <>Ran for </>}
-                        <Duration
-                            start={node.startedAt}
-                            end={node.finishedAt ?? undefined}
-                            stableWidth={false}
-                            className="d-inline"
-                        />
-                    </>
-                )}
-            </div>
-            <div className="text-right flex-grow-1">
-                <div>
-                    {node.startedAt === null && 'Not started yet'}
-                    {node.startedAt !== null && (
-                        <>
-                            Started <Timestamp date={node.startedAt} />
-                        </>
-                    )}
+const ExternalServiceSyncJobNode: React.FunctionComponent<ExternalServiceSyncJobNodeProps> = ({ node, onUpdate }) => {
+    const [
+        cancelExternalServiceSync,
+        { error: cancelSyncJobError, loading: cancelSyncJobLoading },
+    ] = useCancelExternalServiceSync()
+
+    const cancelJob = useCallback(
+        () =>
+            cancelExternalServiceSync({ variables: { id: node.id } }).then(() => {
+                onUpdate.next()
+                // Optimistically set state.
+                node.state = ExternalServiceSyncJobState.CANCELING
+            }),
+        [cancelExternalServiceSync, node, onUpdate]
+    )
+
+    return (
+        <li className="list-group-item py-3">
+            <div className="d-flex align-items-center justify-content-between">
+                <div className="flex-shrink-0 mr-2">
+                    <Badge>{node.state}</Badge>
                 </div>
-                <div>
-                    {node.finishedAt === null && 'Not finished yet'}
-                    {node.finishedAt !== null && (
+                <div className="flex-shrink-0 flex-grow-1 mr-2">
+                    {node.startedAt && (
                         <>
-                            Finished <Timestamp date={node.finishedAt} />
+                            {node.finishedAt === null && <>Running since </>}
+                            {node.finishedAt !== null && <>Ran for </>}
+                            <Duration
+                                start={node.startedAt}
+                                end={node.finishedAt ?? undefined}
+                                stableWidth={false}
+                                className="d-inline"
+                            />
+                            {cancelSyncJobError && <ErrorAlert error={cancelSyncJobError} />}
                         </>
                     )}
                 </div>
+                {[
+                    ExternalServiceSyncJobState.QUEUED,
+                    ExternalServiceSyncJobState.PROCESSING,
+                    ExternalServiceSyncJobState.CANCELING,
+                ].includes(node.state) && (
+                    <LoaderButton
+                        label="Cancel"
+                        alwaysShowLabel={true}
+                        variant="danger"
+                        outline={true}
+                        size="sm"
+                        onClick={cancelJob}
+                        loading={cancelSyncJobLoading || node.state === ExternalServiceSyncJobState.CANCELING}
+                        disabled={cancelSyncJobLoading || node.state === ExternalServiceSyncJobState.CANCELING}
+                        className={styles.cancelButton}
+                    />
+                )}
+                <div className="text-right flex-shrink-0">
+                    <div>
+                        {node.startedAt === null && 'Not started yet'}
+                        {node.startedAt !== null && (
+                            <>
+                                Started <Timestamp date={node.startedAt} />
+                            </>
+                        )}
+                    </div>
+                    <div>
+                        {node.finishedAt === null && 'Not finished yet'}
+                        {node.finishedAt !== null && (
+                            <>
+                                Finished <Timestamp date={node.finishedAt} />
+                            </>
+                        )}
+                    </div>
+                </div>
             </div>
-        </div>
-        {node.failureMessage && <ErrorAlert error={node.failureMessage} className="mt-2 mb-0" />}
-    </li>
-)
+            {node.failureMessage && <ErrorAlert error={node.failureMessage} className="mt-2 mb-0" />}
+        </li>
+    )
+}

client/web/src/components/externalServices/backend.ts

@@ -25,6 +25,8 @@ import {
     ExternalServiceSyncJobsVariables,
     ExternalServiceSyncJobConnectionFields,
     ExternalServiceSyncJobsResult,
+    CancelExternalServiceSyncVariables,
+    CancelExternalServiceSyncResult,
 } from '../../graphql-operations'
 
 export const externalServiceFragment = gql`
@@ -262,6 +264,23 @@ export function useSyncExternalService(): MutationTuple<SyncExternalServiceResul
     return useMutation<SyncExternalServiceResult, SyncExternalServiceVariables>(SYNC_EXTERNAL_SERVICE)
 }
 
+export const CANCEL_EXTERNAL_SERVICE_SYNC = gql`
+    mutation CancelExternalServiceSync($id: ID!) {
+        cancelExternalServiceSync(id: $id) {
+            alwaysNil
+        }
+    }
+`
+
+export function useCancelExternalServiceSync(): MutationTuple<
+    CancelExternalServiceSyncResult,
+    CancelExternalServiceSyncVariables
+> {
+    return useMutation<CancelExternalServiceSyncResult, CancelExternalServiceSyncVariables>(
+        CANCEL_EXTERNAL_SERVICE_SYNC
+    )
+}
+
 export const EXTERNAL_SERVICE_SYNC_JOBS = gql`
     query ExternalServiceSyncJobs($first: Int, $externalService: ID!) {
         node(id: $externalService) {

client/web/src/components/time/Duration.tsx

@@ -57,7 +57,7 @@ export const Duration: React.FunctionComponent<React.PropsWithChildren<DurationP
     }, [end])
 
     return (
-        <div className={classNames({ [styles.stableWidth]: stableWidth }, className)} {...props}>
+        <div className={classNames('chromatic-ignore', { [styles.stableWidth]: stableWidth }, className)} {...props}>
             {stableWidth && (
                 // Set the width of the parent with a filler block of full-width digits,
                 // to prevent layout shift if the time changes.

cmd/frontend/graphqlbackend/external_service.go

@@ -324,11 +324,14 @@ func (r *externalServiceSyncJobResolver) ID() graphql.ID {
 }
 
 func (r *externalServiceSyncJobResolver) State() string {
+	if r.job.Cancel && r.job.State == "processing" {
+		return "CANCELING"
+	}
 	return strings.ToUpper(r.job.State)
 }
 
 func (r *externalServiceSyncJobResolver) FailureMessage() *string {
-	if r.job.FailureMessage == "" {
+	if r.job.FailureMessage == "" || r.job.Cancel {
 		return nil
 	}
 

cmd/frontend/graphqlbackend/external_services.go

@@ -473,3 +473,39 @@ func (r *schemaResolver) SyncExternalService(ctx context.Context, args *syncExte
 
 	return &EmptyResponse{}, nil
 }
+
+type cancelExternalServiceSyncArgs struct {
+	ID graphql.ID
+}
+
+func (r *schemaResolver) CancelExternalServiceSync(ctx context.Context, args *cancelExternalServiceSyncArgs) (*EmptyResponse, error) {
+	start := time.Now()
+	var err error
+	var namespaceUserID, namespaceOrgID int32
+	defer reportExternalServiceDuration(start, Update, &err, &namespaceUserID, &namespaceOrgID)
+
+	id, err := unmarshalExternalServiceSyncJobID(args.ID)
+	if err != nil {
+		return nil, err
+	}
+
+	esj, err := r.db.ExternalServices().GetSyncJobByID(ctx, id)
+	if err != nil {
+		return nil, err
+	}
+	es, err := r.db.ExternalServices().GetByID(ctx, esj.ExternalServiceID)
+	if err != nil {
+		return nil, err
+	}
+
+	// 🚨 SECURITY: check access to external service.
+	if err = backend.CheckExternalServiceAccess(ctx, r.db, es.NamespaceUserID, es.NamespaceOrgID); err != nil {
+		return nil, err
+	}
+
+	if err := r.db.ExternalServices().CancelSyncJob(ctx, esj.ID); err != nil {
+		return nil, err
+	}
+
+	return &EmptyResponse{}, nil
+}