Autocomplete: Don't delay completions for cached entries (#3138)

Right now, about 15% of all suggested completions originate from our
cache (actually quite nice considering we are aggressively invalidating
the cache e.g. when rejecting completions).

However, I noticed that these have a P75 duration of ~125ms which is
caused by the debounce time + retrieval time. We can check the cache
before we do any of this work though to remove the latency of cached
entries completely.

## Test plan

- Updated the unit tests
- Stepped into it with a debugger to see the right breakpoints are
triggered

---------

Co-authored-by: Valery Bugakov <skymk1@gmail.com>

Author: philipp-spiess

vscode/CHANGELOG.md

@@ -12,6 +12,7 @@ This is a log of all notable changes to Cody for VS Code. [Unreleased] changes a
 
 ### Changed
 
+- Autocomplete: Removes the latency for cached completions. [https://github.com/sourcegraph/cody/pull/3138](https://github.com/sourcegraph/cody/pull/3138)
 - Autocomplete: Enable the recent jaccard similarity improvements by default. [pull/3135](https://github.com/sourcegraph/cody/pull/3135)
 
 ## [1.4.3]

vscode/src/completions/get-inline-completions.ts

@@ -282,6 +282,30 @@ async function doGetInlineCompletions(
         traceId: getActiveTraceAndSpanId()?.traceId,
     })
 
+    const requestParams: RequestParams = {
+        document,
+        docContext,
+        position,
+        selectedCompletionInfo,
+        abortSignal,
+    }
+
+    const cachedResult = requestManager.checkCache({
+        requestParams,
+        isCacheEnabled: triggerKind !== TriggerKind.Manual,
+    })
+    if (cachedResult) {
+        const { completions, source } = cachedResult
+
+        CompletionLogger.loaded(logId, requestParams, completions, source, isDotComUser)
+
+        return {
+            logId,
+            items: completions,
+            source,
+        }
+    }
+
     // Debounce to avoid firing off too many network requests as the user is still typing.
     await wrapInActiveSpan('autocomplete.debounce', async () => {
         const interval =
@@ -334,14 +358,6 @@ async function doGetInlineCompletions(
 
     CompletionLogger.networkRequestStarted(logId, contextResult?.logSummary)
 
-    const requestParams: RequestParams = {
-        document,
-        docContext,
-        position,
-        selectedCompletionInfo,
-        abortSignal,
-    }
-
     // Get the processed completions from providers
     const { completions, source } = await requestManager.request({
         requestParams,

vscode/src/completions/request-manager.test.ts

@@ -99,6 +99,7 @@ describe('RequestManager', () => {
         provider: Provider,
         suffix?: string
     ) => Promise<RequestManagerResult>
+    let checkCache: (prefix: string, suffix?: string) => RequestManagerResult | null
     beforeEach(async () => {
         await initCompletionProviderConfig({})
         const requestManager = new RequestManager()
@@ -110,6 +111,8 @@ describe('RequestManager', () => {
                 context: [],
                 isCacheEnabled: true,
             })
+        checkCache = (prefix: string, suffix?: string) =>
+            requestManager.checkCache({ requestParams: docState(prefix, suffix), isCacheEnabled: true })
     })
 
     it('resolves a single request', async () => {
@@ -124,20 +127,6 @@ describe('RequestManager', () => {
         expect(source).toBe(InlineCompletionsResultSource.Network)
     })
 
-    it('resolves a single request', async () => {
-        const prefix = 'console.log('
-        const provider1 = createProvider(prefix)
-        setTimeout(() => provider1.yield(["'hello')"]), 0)
-        await createRequest(prefix, provider1)
-
-        const provider2 = createProvider(prefix)
-
-        const { completions, source } = await createRequest(prefix, provider2)
-
-        expect(source).toBe(InlineCompletionsResultSource.Cache)
-        expect(completions[0].insertText).toBe("'hello')")
-    })
-
     it('does not resolve from cache if the suffix has changed', async () => {
         const prefix = 'console.log('
         const suffix1 = ')\nconsole.log(1)'
@@ -205,6 +194,20 @@ describe('RequestManager', () => {
         provider2.yield(["'world')"])
     })
 
+    describe('cache', () => {
+        it('resolves a single request with a cached value without waiting for the debounce timeout', async () => {
+            const prefix = 'console.log('
+            const provider1 = createProvider(prefix)
+            setTimeout(() => provider1.yield(["'hello')"]), 0)
+            await createRequest(prefix, provider1)
+
+            const { completions, source } = checkCache(prefix)!
+
+            expect(source).toBe(InlineCompletionsResultSource.Cache)
+            expect(completions[0].insertText).toBe("'hello')")
+        })
+    })
+
     describe('abort logic', () => {
         it('aborts a newer request if a prior request resolves it', async () => {
             const prefix1 = 'console.'

vscode/src/completions/request-manager.ts

@@ -73,23 +73,28 @@ export class RequestManager {
     // the relevance of existing requests (i.e to find out if the generations are still relevant)
     private latestRequestParams: null | RequestsManagerParams = null
 
+    public checkCache(
+        params: Pick<RequestsManagerParams, 'requestParams' | 'isCacheEnabled'>
+    ): RequestManagerResult | null {
+        const { requestParams, isCacheEnabled } = params
+        const cachedCompletions = this.cache.get(requestParams)
+
+        if (isCacheEnabled && cachedCompletions) {
+            addAutocompleteDebugEvent('RequestManager.checkCache', { cachedCompletions })
+            return cachedCompletions
+        }
+        return null
+    }
+
     public async request(params: RequestsManagerParams): Promise<RequestManagerResult> {
         const eagerCancellation = completionProviderConfig.getPrefetchedFlag(
             FeatureFlag.CodyAutocompleteEagerCancellation
         )
         this.latestRequestParams = params
 
-        const { requestParams, provider, context, isCacheEnabled, tracer } = params
+        const { requestParams, provider, context, tracer } = params
 
-        const cachedCompletions = this.cache.get(requestParams)
-        addAutocompleteDebugEvent('RequestManager.request', {
-            cachedCompletions,
-            isCacheEnabled,
-        })
-
-        if (isCacheEnabled && cachedCompletions) {
-            return cachedCompletions
-        }
+        addAutocompleteDebugEvent('RequestManager.request')
 
         // When request recycling is enabled, we do not pass the original abort signal forward as to
         // not interrupt requests that are no longer relevant. Instead, we let all previous requests