diff --git a/docs/docs/test-runner/cli-and-configuration.md b/docs/docs/test-runner/cli-and-configuration.md index 7e718f000..822e574c9 100644 --- a/docs/docs/test-runner/cli-and-configuration.md +++ b/docs/docs/test-runner/cli-and-configuration.md @@ -104,6 +104,8 @@ interface CoverageConfig { report: boolean; reportDir: string; reporters?: ReportType[]; + // whether to measure coverage of untested files + all?: boolean; } type MimeTypeMappings = Record; diff --git a/packages/test-runner-core/src/config/TestRunnerCoreConfig.ts b/packages/test-runner-core/src/config/TestRunnerCoreConfig.ts index 6a17dacf8..b76a6e0f7 100644 --- a/packages/test-runner-core/src/config/TestRunnerCoreConfig.ts +++ b/packages/test-runner-core/src/config/TestRunnerCoreConfig.ts @@ -22,6 +22,7 @@ export interface CoverageConfig { report?: boolean; reportDir?: string; reporters?: ReportType[]; + all?: boolean; } export interface TestRunnerCoreConfig { diff --git a/packages/test-runner-core/src/coverage/getTestCoverage.ts b/packages/test-runner-core/src/coverage/getTestCoverage.ts index 040d0b558..fea9196b9 100644 --- a/packages/test-runner-core/src/coverage/getTestCoverage.ts +++ b/packages/test-runner-core/src/coverage/getTestCoverage.ts @@ -178,11 +178,15 @@ function addingMissingCoverageItems(coverages: CoverageMapData[]) { export function getTestCoverage( sessions: Iterable, config?: CoverageConfig, + allFilesCoverage?: CoverageMapData ): TestCoverage { const coverageMap = createCoverageMap(); let coverages = Array.from(sessions) .map(s => s.testCoverage) .filter(c => c) as CoverageMapData[]; + if (allFilesCoverage) { + coverages.unshift(allFilesCoverage); + } // istanbul mutates the coverage objects, which pollutes coverage in watch mode // cloning prevents this. JSON stringify -> parse is faster than a fancy library // because we're only working with objects and arrays diff --git a/packages/test-runner-core/src/runner/TestRunner.ts b/packages/test-runner-core/src/runner/TestRunner.ts index f2b486244..f674676f1 100644 --- a/packages/test-runner-core/src/runner/TestRunner.ts +++ b/packages/test-runner-core/src/runner/TestRunner.ts @@ -11,6 +11,7 @@ import { createDebugSessions } from './createDebugSessions.js'; import { TestRunnerServer } from '../server/TestRunnerServer.js'; import { BrowserLauncher } from '../browser-launcher/BrowserLauncher.js'; import { TestRunnerGroupConfig } from '../config/TestRunnerGroupConfig.js'; +import { generateEmptyReportsForUntouchedFiles } from '@web/test-runner-coverage-v8'; interface EventMap { 'test-run-started': { testRun: number }; @@ -205,7 +206,11 @@ export class TestRunner extends EventEmitter { let passedCoverage = true; let testCoverage: TestCoverage | undefined = undefined; if (this.config.coverage) { - testCoverage = getTestCoverage(this.sessions.all(), this.config.coverageConfig); + let allFilesCoverage; + if (this.config.coverageConfig?.all) { + allFilesCoverage = await generateEmptyReportsForUntouchedFiles(this.config, this.testFiles); + } + testCoverage = getTestCoverage(this.sessions.all(), this.config.coverageConfig, allFilesCoverage); passedCoverage = testCoverage.passed; } diff --git a/packages/test-runner-coverage-v8/src/index.ts b/packages/test-runner-coverage-v8/src/index.ts index 306f53086..6c8986f9f 100644 --- a/packages/test-runner-coverage-v8/src/index.ts +++ b/packages/test-runner-coverage-v8/src/index.ts @@ -1,13 +1,13 @@ -import { extname, join, isAbsolute, sep, posix } from 'path'; +import { extname, join, isAbsolute, sep, posix } from 'node:path'; import { CoverageMapData } from 'istanbul-lib-coverage'; import v8toIstanbulLib from 'v8-to-istanbul'; import { TestRunnerCoreConfig, fetchSourceMap } from '@web/test-runner-core'; import { Profiler } from 'inspector'; import picoMatch from 'picomatch'; import LruCache from 'lru-cache'; -import { readFile } from 'node:fs/promises'; - -import { toFilePath } from './utils.js'; +import { readFile, readdir, stat } from 'node:fs/promises'; +import { Stats } from 'node:fs'; +import { toFilePath, toBrowserPath } from './utils.js'; type V8Coverage = Profiler.ScriptCoverage; type Matcher = (test: string) => boolean; @@ -32,11 +32,10 @@ function hasOriginalSource(source: IstanbulSource): boolean { typeof source.sourceMap.sourcemap === 'object' && source.sourceMap.sourcemap !== null && Array.isArray(source.sourceMap.sourcemap.sourcesContent) && - source.sourceMap.sourcemap.sourcesContent.length > 0 - ); + source.sourceMap.sourcemap.sourcesContent.length > 0); } -function getMatcher(patterns?: string[]) { +function getMatcher(patterns?: string[]): picoMatch.Matcher { if (!patterns || patterns.length === 0) { return () => true; } @@ -60,63 +59,154 @@ export async function v8ToIstanbul( testFiles: string[], coverage: V8Coverage[], userAgent?: string, -) { +): Promise { const included = getMatcher(config?.coverageConfig?.include); const excluded = getMatcher(config?.coverageConfig?.exclude); const istanbulCoverage: CoverageMapData = {}; for (const entry of coverage) { - const url = new URL(entry.url); - const path = url.pathname; - if ( - // ignore non-http protocols (for exmaple webpack://) - url.protocol.startsWith('http') && - // ignore external urls - url.hostname === config.hostname && - url.port === `${config.port}` && - // ignore non-files - !!extname(path) && - // ignore virtual files - !path.startsWith('/__web-test-runner') && - !path.startsWith('/__web-dev-server') - ) { - try { + try { + const url = new URL(entry.url); + const path = url.pathname; + if ( + // ignore non-http protocols (for exmaple webpack://) + url.protocol.startsWith('http') && + // ignore external urls + url.hostname === config.hostname && + url.port === `${config.port}` && + // ignore non-files + !!extname(path) && + // ignore virtual files + !path.startsWith('/__web-test-runner') && + !path.startsWith('/__web-dev-server') + ) { const filePath = join(config.rootDir, toFilePath(path)); - if (!testFiles.includes(filePath) && included(filePath) && !excluded(filePath)) { const browserUrl = `${url.pathname}${url.search}${url.hash}`; - const cachedSource = cachedSources.get(browserUrl); - const sources = - cachedSource ?? - ((await fetchSourceMap({ - protocol: config.protocol, - host: config.hostname, - port: config.port, - browserUrl, - userAgent, - })) as IstanbulSource); - - if (!cachedSource) { - if (!hasOriginalSource(sources)) { - const contents = await readFile(filePath, 'utf8'); - (sources as IstanbulSource & { originalSource: string }).originalSource = contents; - } - cachedSources.set(browserUrl, sources); - } - - const converter = v8toIstanbulLib(filePath, 0, sources); - await converter.load(); - - converter.applyCoverage(entry.functions); - Object.assign(istanbulCoverage, converter.toIstanbul()); + const sources = await getIstanbulSource(config, filePath, browserUrl, userAgent); + await addCoverageForFilePath(sources, filePath, entry, istanbulCoverage); } - } catch (error) { - console.error(`Error while generating code coverage for ${entry.url}.`); - console.error(error); + } + } catch (error) { + console.error(`Error while generating code coverage for ${entry.url}.`); + console.error(error); + } + } + + return istanbulCoverage; +} + +async function addCoverageForFilePath( + sources: IstanbulSource, + filePath: string, + entry: V8Coverage, + istanbulCoverage: CoverageMapData, +): Promise { + const converter = v8toIstanbulLib(filePath, 0, sources); + await converter.load(); + + converter.applyCoverage(entry.functions); + Object.assign(istanbulCoverage, converter.toIstanbul()); +} + +async function getIstanbulSource( + config: TestRunnerCoreConfig, + filePath: string, + browserUrl: string, + userAgent?: string, + doNotAddToCache?: boolean, +): Promise { + const cachedSource = cachedSources.get(browserUrl); + const sources = + cachedSource ?? + ((await fetchSourceMap({ + protocol: config.protocol, + host: config.hostname, + port: config.port, + browserUrl, + userAgent, + })) as IstanbulSource); + + if (!cachedSource) { + if (!hasOriginalSource(sources)) { + const contents = await readFile(filePath, 'utf8'); + (sources as IstanbulSource & { originalSource: string }).originalSource = contents; + } + !doNotAddToCache && cachedSources.set(browserUrl, sources); + } + return sources; +} + + +async function recursivelyAddEmptyReports( + config: TestRunnerCoreConfig, + testFiles: string[], + include: picoMatch.Matcher, + exclude: picoMatch.Matcher, + istanbulCoverage: CoverageMapData, + dir = '', +): Promise { + const contents = await readdir(join(coverageBaseDir, dir)); + for (const file of contents) { + const filePath = join(coverageBaseDir, dir, file); + if (!exclude(filePath)) { + const stats = await stat(filePath); + const relativePath = join(dir, file); + if (stats.isDirectory()) { + await recursivelyAddEmptyReports(config, testFiles, include, exclude, istanbulCoverage, relativePath); + } else if (!testFiles.includes(filePath) && include(filePath)) { + await addEmptyReportIfFileUntouched(config, istanbulCoverage, filePath, stats, relativePath); } } } +} +async function addEmptyReportIfFileUntouched( + config: TestRunnerCoreConfig, + istanbulCoverage: CoverageMapData, + filePath: string, + stats: Stats, + relativePath: string, +): Promise { + try { + const browserUrl = toBrowserPath(relativePath); + const fileHasBeenTouched = cachedSources.find((_, key) => { + return key === browserUrl || key.startsWith(browserUrl+'?') || key.startsWith(browserUrl+'#'); + }); + if (fileHasBeenTouched) { + return; + } + const sources = await getIstanbulSource(config, filePath, browserUrl, undefined, true); + const entry = { + scriptId: browserUrl, + url: browserUrl, + functions: [{ + functionName: '(empty-report)', + isBlockCoverage: true, + ranges: [{ + startOffset: 0, + endOffset: stats.size, + count: 0 + }] + }] + } as V8Coverage; + await addCoverageForFilePath(sources, filePath, entry, istanbulCoverage); + } catch (error) { + console.error(`Error while generating empty code coverage for ${filePath}.`); + console.error(error); + } +} + +export async function generateEmptyReportsForUntouchedFiles( + config: TestRunnerCoreConfig, + testFiles: string[], +): Promise { + const istanbulCoverage: CoverageMapData = {}; + if (config?.coverageConfig) { + const include = getMatcher(config.coverageConfig.include); + const exclude = getMatcher(config.coverageConfig.exclude); + await recursivelyAddEmptyReports(config, testFiles, include, exclude, istanbulCoverage); + } return istanbulCoverage; } diff --git a/packages/test-runner-coverage-v8/src/utils.ts b/packages/test-runner-coverage-v8/src/utils.ts index 925271265..021d0e271 100644 --- a/packages/test-runner-coverage-v8/src/utils.ts +++ b/packages/test-runner-coverage-v8/src/utils.ts @@ -1,7 +1,16 @@ -import path from 'path'; +import path from 'node:path'; const REGEXP_TO_FILE_PATH = new RegExp('/', 'g'); +const REGEXP_TO_BROWSER_PATH = new RegExp('\\\\', 'g'); -export function toFilePath(browserPath: string) { +export function toFilePath(browserPath: string): string { return browserPath.replace(REGEXP_TO_FILE_PATH, path.sep); } + +export function toBrowserPath(filePath: string): string { + const replaced = filePath.replace(REGEXP_TO_BROWSER_PATH, '/'); + if (replaced[0] !== '/') { + return '/' + replaced; + } + return replaced; +}