Suggestion: Refine search_files to improve output formatting and derisk large output

[nissa-seru]

The current search_files implementation can result in huge output if a model searches for a commonly used term. I have not seen Claude-Sonnet search for something trivially large like ".", but I commonly see it search for test|tests etc. The very first time I used Roo, actually, it errored itself very quickly with a search_files call yielded approx 206k tokens in response. Below is a tweaked implementation that has worked well for me - there is very likely room for improvement (for example, I thin the amount of context provided around each result, which may not be optimal wrt enabling accurate diffs.)

//src/services/ripgrep/index.ts
import * as vscode from "vscode"
import * as childProcess from "child_process"
import * as path from "path"
import * as fs from "fs"
import * as readline from "readline"

const isWindows = /^win/.test(process.platform)
const binName = isWindows ? "rg.exe" : "rg"

const MAX_FILES = 100

interface RipgrepMatch {
	file: string
	line: number
	column: number
	content: string
	lineNumber: number
	beforeContext: Array<{
		line: number
		content: string
	}>
	afterContext: Array<{
		line: number
		content: string
	}>
}

interface RipgrepResult {
	matches: Map<string, RipgrepMatch[]>
	totalMatches: number
	isComplete: boolean
}

async function getBinPath(vscodeAppRoot: string): Promise<string | undefined> {
	const checkPath = async (pkgFolder: string) => {
		const fullPath = path.join(vscodeAppRoot, pkgFolder, binName)
		return (await pathExists(fullPath)) ? fullPath : undefined
	}

	return (
		(await checkPath("node_modules/@vscode/ripgrep/bin/")) ||
		(await checkPath("node_modules/vscode-ripgrep/bin")) ||
		(await checkPath("node_modules.asar.unpacked/vscode-ripgrep/bin/")) ||
		(await checkPath("node_modules.asar.unpacked/@vscode/ripgrep/bin/"))
	)
}

async function pathExists(path: string): Promise<boolean> {
	return new Promise((resolve) => {
		fs.access(path, (err) => {
			resolve(err === null)
		})
	})
}

async function execRipgrep(bin: string, args: string[]): Promise<RipgrepResult> {
	return new Promise((resolve, reject) => {
		const rgProcess = childProcess.spawn(bin, args)
		const rl = readline.createInterface({
			input: rgProcess.stdout,
			crlfDelay: Infinity,
		})

		const matches = new Map<string, RipgrepMatch[]>()
		let totalMatches = 0
		let currentMatch: Partial<RipgrepMatch> | null = null
		let isComplete = true

		// First pass: collect first match from each file
		const processLine = (line: string) => {
			if (!line) return

			try {
				const parsed = JSON.parse(line)
				if (parsed.type === "match") {
					const file = parsed.data.path.text

					// Track total matches regardless of collection
					totalMatches++

					// First pass: only collect if we haven't seen this file
					if (!matches.has(file) && matches.size < MAX_FILES) {
						currentMatch = {
							file,
							line: parsed.data.line_number,
							column: parsed.data.submatches[0].start,
							content: parsed.data.lines.text,
							lineNumber: parsed.data.line_number,
							beforeContext: [],
							afterContext: [],
						}
					}
				} else if (parsed.type === "context" && currentMatch) {
					const contextEntry = {
						line: parsed.data.line_number,
						content: parsed.data.lines.text,
					}

					if (parsed.data.line_number < currentMatch.lineNumber!) {
						currentMatch.beforeContext!.push(contextEntry)
					} else {
						currentMatch.afterContext!.push(contextEntry)
					}
				}

				// If we've completed a match, store it
				if (currentMatch && currentMatch.file && !matches.has(currentMatch.file)) {
					matches.set(currentMatch.file, [currentMatch as RipgrepMatch])
					currentMatch = null

					// Check if we've hit our limits
					if (matches.size >= MAX_FILES) {
						isComplete = false
						rl.close()
						rgProcess.kill()
					}
				}
			} catch (error) {
				console.error("Error parsing ripgrep output:", error)
			}
		}

		rl.on("line", processLine)

		let errorOutput = ""
		rgProcess.stderr.on("data", (data) => {
			errorOutput += data.toString()
		})

		rl.on("close", () => {
			if (errorOutput) {
				reject(new Error(`ripgrep process error: ${errorOutput}`))
			} else {
				resolve({
					matches,
					totalMatches,
					isComplete,
				})
			}
		})

		rgProcess.on("error", (error) => {
			reject(new Error(`ripgrep process error: ${error.message}`))
		})
	})
}

interface SearchResult {
	matches: Array<{
		file: string
		line: number
		column: number
		content: string
	}>
	summary: {
		totalMatches: number
		matchedFiles: number
		isComplete: boolean
		truncated: boolean
	}
}

export async function regexSearchFiles(
	cwd: string,
	directoryPath: string,
	regex: string,
	filePattern?: string,
): Promise<SearchResult> {
	const rgPath = await getBinPath(vscode.env.appRoot)

	if (!rgPath) {
		throw new Error("Could not find ripgrep binary")
	}

	const args = ["--json", "-e", regex, "--glob", filePattern || "*", directoryPath]

	try {
		const { matches, totalMatches, isComplete: initialComplete } = await execRipgrep(rgPath, args)

		// Estimate tokens: roughly 4 chars per token
		let estimatedTokens = 0
		const results: SearchResult["matches"] = []
		let searchComplete = initialComplete

		for (const [filePath, fileMatches] of matches.entries()) {
			const relativePath = path.relative(cwd, filePath)

			// Estimate tokens for file path
			estimatedTokens += Math.ceil(relativePath.length / 4)

			for (const match of fileMatches) {
				// Estimate tokens for match content
				const matchTokens = Math.ceil(match.content.length / 4)

				// Check if adding this match would exceed token limit
				if (estimatedTokens + matchTokens > 8000) {
					// Conservative limit
					searchComplete = false
					break
				}

				estimatedTokens += matchTokens

				results.push({
					file: relativePath,
					line: match.line,
					column: match.column,
					content: match.content,
				})
			}

			if (!searchComplete) break
		}

		return {
			matches: results,
			summary: {
				totalMatches,
				matchedFiles: matches.size,
				isComplete: searchComplete,
				truncated: !searchComplete || matches.size >= MAX_FILES,
			},
		}
	} catch (error) {
		return {
			matches: [],
			summary: {
				totalMatches: 0,
				matchedFiles: 0,
				isComplete: true,
				truncated: false,
			},
		}
	}
}

export formatSearchResults(results: SearchResults): string {
		let output = ""

		// Add summary
		output += `Found ${results.summary.totalMatches} matches across ${results.summary.matchedFiles} files`
		if (results.summary.truncated) {
			output += " (results truncated due to size limits)"
		}
		output += "\n\n"

		// Format matches
		for (const match of results.matches) {
			output += `${match.file}:${match.line} - ${match.content}\n`
		}

		return output.trim()
	}
}

[nissa-seru]

Implemented in #709