Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 27 additions & 0 deletions apps/vscode-e2e/src/suite/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import * as vscode from "vscode"
import type { RooCodeAPI } from "@roo-code/types"

import { waitFor } from "./utils"
import * as fs from "fs"

export async function run() {
const extension = vscode.extensions.getExtension<RooCodeAPI>("RooVeterinaryInc.roo-cline")
Expand All @@ -16,6 +17,32 @@ export async function run() {

const api = extension.isActive ? extension.exports : await extension.activate()

// Ensure OPENROUTER_API_KEY is loaded from .env.local in CI (written by workflow)
// __dirname at runtime is apps/vscode-e2e/out/suite, so go up two levels
const envPath = path.resolve(__dirname, "..", "..", ".env.local")
if (!process.env.OPENROUTER_API_KEY && fs.existsSync(envPath)) {
try {
const content = fs.readFileSync(envPath, "utf8")
for (const rawLine of content.split("\n")) {
const line = rawLine.trim()
if (!line || line.startsWith("#")) continue
const m = /^\s*([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)\s*$/.exec(line)
if (!m) continue
const key: string = m[1] ?? ""
let val: string = m[2] ?? ""
// Strip surrounding quotes if present
if ((val.startsWith('"') && val.endsWith('"')) || (val.startsWith("'") && val.endsWith("'"))) {
val = val.slice(1, -1)
}
if (key && !(key in process.env)) {
;(process.env as Record<string, string>)[key] = val
}
}
} catch {
// ignore env load errors; tests may still pass without API calls
}
}

await api.setConfiguration({
apiProvider: "openrouter" as const,
openRouterApiKey: process.env.OPENROUTER_API_KEY!,
Expand Down
77 changes: 70 additions & 7 deletions apps/vscode-e2e/src/suite/tools/apply-diff.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,33 @@ import * as path from "path"
import * as vscode from "vscode"

import { RooCodeEventName, type ClineMessage } from "@roo-code/types"

import { waitFor, sleep } from "../utils"
import { setDefaultSuiteTimeout } from "../test-utils"

/**
* Local minimal typings to avoid module resolution issues in e2e package.
* We only use a subset of fields in these tests.
*/
// Local minimal type shapes to avoid ts-module resolution issues in e2e package
type ToolUsage = Record<string, { attempts: number; failures: number }>
type TokenUsage = {
totalTokensIn: number
totalTokensOut: number
totalCost: number
contextTokens: number
totalCacheWrites?: number
totalCacheReads?: number
}

// Global fallback storage for tool usage across handlers
let lastToolUsage: ToolUsage | undefined = undefined

// Helper to safely read apply_diff attempts without using `any`
function getApplyDiffAttempts() {
const lu = lastToolUsage as unknown as Record<string, { attempts?: number }>
return lu?.apply_diff?.attempts ?? 0
}

suite("Roo Code apply_diff Tool", function () {
setDefaultSuiteTimeout(this)

Expand Down Expand Up @@ -161,6 +184,7 @@ function validateInput(input) {
let taskCompleted = false
let errorOccurred: string | null = null
let applyDiffExecuted = false
const toolUsageRef: { current: ToolUsage | undefined } = { current: undefined }

// Listen for messages
const messageHandler = ({ message }: { message: ClineMessage }) => {
Expand Down Expand Up @@ -203,9 +227,11 @@ function validateInput(input) {
}
api.on(RooCodeEventName.TaskStarted, taskStartedHandler)

const taskCompletedHandler = (id: string) => {
const taskCompletedHandler = (id: string, _tokenUsage: TokenUsage, toolUsage: ToolUsage) => {
if (id === taskId) {
taskCompleted = true
toolUsageRef.current = toolUsage
lastToolUsage = toolUsage
console.log("Task completed:", id)
}
}
Expand Down Expand Up @@ -247,6 +273,11 @@ ${testFile.content}\nAssume the file exists and you can modify it directly.`,
const actualContent = await fs.readFile(testFile.path, "utf-8")
console.log("File content after modification:", actualContent)

// Fallback to toolUsage if api_req_started message is not emitted
if (!applyDiffExecuted) {
applyDiffExecuted = getApplyDiffAttempts() > 0
}

// Verify tool was executed
assert.strictEqual(applyDiffExecuted, true, "apply_diff tool should have been executed")

Expand Down Expand Up @@ -280,6 +311,7 @@ ${testFile.content}\nAssume the file exists and you can modify it directly.`,
let taskStarted = false
let taskCompleted = false
let applyDiffExecuted = false
const toolUsageRef: { current: ToolUsage | undefined } = { current: undefined }

// Listen for messages
const messageHandler = ({ message }: { message: ClineMessage }) => {
Expand Down Expand Up @@ -316,9 +348,11 @@ ${testFile.content}\nAssume the file exists and you can modify it directly.`,
}
api.on(RooCodeEventName.TaskStarted, taskStartedHandler)

const taskCompletedHandler = (id: string) => {
const taskCompletedHandler = (id: string, _tokenUsage: TokenUsage, toolUsage: ToolUsage) => {
if (id === taskId) {
taskCompleted = true
toolUsageRef.current = toolUsage
lastToolUsage = toolUsage
console.log("Task completed:", id)
}
}
Expand Down Expand Up @@ -362,6 +396,11 @@ ${testFile.content}\nAssume the file exists and you can modify it directly.`,
const actualContent = await fs.readFile(testFile.path, "utf-8")
console.log("File content after modification:", actualContent)

// Fallback to toolUsage if api_req_started message is not emitted
if (!applyDiffExecuted) {
applyDiffExecuted = getApplyDiffAttempts() > 0
}

// Verify tool was executed
assert.strictEqual(applyDiffExecuted, true, "apply_diff tool should have been executed")

Expand Down Expand Up @@ -402,6 +441,7 @@ function keepThis() {
let taskStarted = false
let taskCompleted = false
let applyDiffExecuted = false
const toolUsageRef: { current: ToolUsage | undefined } = { current: undefined }

// Listen for messages
const messageHandler = ({ message }: { message: ClineMessage }) => {
Expand Down Expand Up @@ -434,9 +474,11 @@ function keepThis() {
}
api.on(RooCodeEventName.TaskStarted, taskStartedHandler)

const taskCompletedHandler = (id: string) => {
const taskCompletedHandler = (id: string, _tokenUsage: TokenUsage, toolUsage: ToolUsage) => {
if (id === taskId) {
taskCompleted = true
toolUsageRef.current = toolUsage
lastToolUsage = toolUsage
}
}
api.on(RooCodeEventName.TaskCompleted, taskCompletedHandler)
Expand Down Expand Up @@ -474,6 +516,11 @@ ${testFile.content}\nAssume the file exists and you can modify it directly.`,
const actualContent = await fs.readFile(testFile.path, "utf-8")
console.log("File content after modification:", actualContent)

// Fallback to toolUsage if api_req_started message is not emitted
if (!applyDiffExecuted) {
applyDiffExecuted = getApplyDiffAttempts() > 0
}

// Verify tool was executed
assert.strictEqual(applyDiffExecuted, true, "apply_diff tool should have been executed")

Expand Down Expand Up @@ -501,6 +548,7 @@ ${testFile.content}\nAssume the file exists and you can modify it directly.`,
let taskCompleted = false
let errorDetected = false
let applyDiffAttempted = false
const toolUsageRef: { current: ToolUsage | undefined } = { current: undefined }

// Listen for messages
const messageHandler = ({ message }: { message: ClineMessage }) => {
Expand Down Expand Up @@ -542,9 +590,11 @@ ${testFile.content}\nAssume the file exists and you can modify it directly.`,
}
api.on(RooCodeEventName.TaskStarted, taskStartedHandler)

const taskCompletedHandler = (id: string) => {
const taskCompletedHandler = (id: string, _tokenUsage: TokenUsage, toolUsage: ToolUsage) => {
if (id === taskId) {
taskCompleted = true
toolUsageRef.current = toolUsage
lastToolUsage = toolUsage
}
}
api.on(RooCodeEventName.TaskCompleted, taskCompletedHandler)
Expand Down Expand Up @@ -585,7 +635,10 @@ Assume the file exists and you can modify it directly.`,
const actualContent = await fs.readFile(testFile.path, "utf-8")
console.log("File content after task:", actualContent)

// The AI should have attempted to use apply_diff
// The AI should have attempted to use apply_diff (fallback to toolUsage attempts)
if (!applyDiffAttempted) {
applyDiffAttempted = getApplyDiffAttempts() > 0
}
assert.strictEqual(applyDiffAttempted, true, "apply_diff tool should have been attempted")

// The content should remain unchanged since the search pattern wasn't found
Expand Down Expand Up @@ -631,6 +684,7 @@ function checkInput(input) {
let errorOccurred: string | null = null
let applyDiffExecuted = false
let applyDiffCount = 0
const toolUsageRef: { current: ToolUsage | undefined } = { current: undefined }

// Listen for messages
const messageHandler = ({ message }: { message: ClineMessage }) => {
Expand Down Expand Up @@ -674,9 +728,11 @@ function checkInput(input) {
}
api.on(RooCodeEventName.TaskStarted, taskStartedHandler)

const taskCompletedHandler = (id: string) => {
const taskCompletedHandler = (id: string, _tokenUsage: TokenUsage, toolUsage: ToolUsage) => {
if (id === taskId) {
taskCompleted = true
toolUsageRef.current = toolUsage
lastToolUsage = toolUsage
console.log("Task completed:", id)
}
}
Expand Down Expand Up @@ -728,6 +784,13 @@ Assume the file exists and you can modify it directly.`,
const actualContent = await fs.readFile(testFile.path, "utf-8")
console.log("File content after modification:", actualContent)

// Fallback to toolUsage counts if message-based detection didn't trigger
const attempts = getApplyDiffAttempts()
if (!applyDiffExecuted) {
applyDiffExecuted = attempts > 0
}
applyDiffCount = Math.max(applyDiffCount, attempts)

// Verify tool was executed
assert.strictEqual(applyDiffExecuted, true, "apply_diff tool should have been executed")
console.log(`apply_diff was executed ${applyDiffCount} time(s)`)
Expand Down
Loading
Loading