Add support for collecting coverage data and exporting LCOV reports

Src/java/engine/src/main/kotlin/org/opencds/cqf/cql/engine/execution/Coverage.kt

@@ -0,0 +1,193 @@
+package org.opencds.cqf.cql.engine.execution
+
+import org.cqframework.cql.elm.visiting.BaseElmLibraryVisitor
+import org.hl7.elm.r1.Element
+import org.hl7.elm.r1.Expression
+import org.hl7.elm.r1.ExpressionDef
+import org.hl7.elm.r1.Library
+import org.hl7.elm.r1.VersionedIdentifier
+import org.opencds.cqf.cql.engine.debug.Location
+
+/**
+ * Represents the CQL/ELM coverage information for all libraries involved in execution. An instance
+ * of this class is kept in the engine state during execution.
+ */
+class GlobalCoverage {
+    private val libraryCoverages = mutableMapOf<VersionedIdentifier, LibraryCoverage>()
+
+    /** Called during ELM evaluation to mark an element as visited for coverage reporting. */
+    fun markElementAsVisitedForCoverageReport(elm: Element, library: Library) {
+        val libraryIdentifier =
+            checkNotNull(library.identifier) {
+                "Current library has null identifier when marking element for coverage report"
+            }
+        val coverage = libraryCoverages.getOrPut(libraryIdentifier) { LibraryCoverage(library) }
+        coverage.markVisited(elm)
+    }
+
+    /** Exports coverage information in LCOV format (lcov.info) for multiple libraries. */
+    fun exportLcovInfo(libraryIdentifiers: List<VersionedIdentifier>): String {
+        return buildString {
+            for (libraryIdentifier in libraryIdentifiers) {
+                val libraryCoverage = libraryCoverages[libraryIdentifier]
+                if (libraryCoverage != null) {
+                    append(libraryCoverage.toLcovInfo())
+                }
+            }
+        }
+    }
+}
+
+/** Represents coverage information for a single ELM library. */
+internal class LibraryCoverage(val library: Library) {
+    /** Keeps track of how many times each element was visited. */
+    private val elementVisitCounts = mutableMapOf<Element, Int>()
+
+    /** Marks an ELM element as visited. */
+    fun markVisited(elm: Element) {
+        elementVisitCounts[elm] = (elementVisitCounts[elm] ?: 0) + 1
+    }
+
+    /** Returns the visit count for a branch. */
+    fun getBranchVisitCount(branch: Branch): Int {
+        // ExpressionDefs (including FunctionDefs) aren't directly marked as visited. When they are
+        // called or evaluated, the evaluation visitor instead visits the expression inside the
+        // definition.
+        if (branch.elm is ExpressionDef) {
+            return branch.children.sumOf { getBranchVisitCount(it) }
+        }
+
+        return elementVisitCounts[branch.elm] ?: 0
+    }
+
+    /**
+     * Calculates line coverage results from branches.
+     *
+     * @param branches The branches representing an ELM library.
+     * @return A map of line numbers to their coverage information.
+     */
+    fun calculateLineCoverages(branches: List<Branch>): Map<Int, LineCoverage> {
+
+        /** Maps line numbers to their coverage information. */
+        val lineCoverages = mutableMapOf<Int, LineCoverage>()
+
+        /** Recursively calculates line coverages for branches. */
+        fun calculateLineCoveragesInner(branches: List<Branch>) {
+
+            /** Maps line numbers to branches that cover that line. */
+            val branchBlocks = mutableMapOf<Int, MutableSet<Branch>>()
+
+            for (branch in branches) {
+                val branchLocation = branch.location
+                if (branchLocation != null) {
+                    val visitCountForBranch = getBranchVisitCount(branch)
+                    for (lineNumber in branchLocation.startLine..branchLocation.endLine) {
+                        lineCoverages.getOrPut(lineNumber) { LineCoverage() }.visitCount +=
+                            visitCountForBranch
+                        branchBlocks.getOrPut(lineNumber) { mutableSetOf() }.add(branch)
+                    }
+                }
+                calculateLineCoveragesInner(branch.children)
+            }
+
+            for ((lineNumber, branchBlock) in branchBlocks) {
+                lineCoverages.getOrPut(lineNumber) { LineCoverage() }.branchBlocks.add(branchBlock)
+            }
+        }
+
+        calculateLineCoveragesInner(branches)
+
+        return lineCoverages
+    }
+
+    /** Exports coverage information in LCOV format (lcov.info). */
+    fun toLcovInfo(): String {
+        val branches = collectBranches(library)
+        val lineCoverages = calculateLineCoverages(branches)
+        return buildString {
+            append("TN:\n")
+            val libraryIdentifier = library.identifier!!
+            val libraryName = buildString {
+                append(libraryIdentifier.id ?: "unknown")
+                if (libraryIdentifier.version != null) {
+                    append("-${libraryIdentifier.version}")
+                }
+                append(".cql")
+            }
+            append("SF:$libraryName\n")
+            for ((lineNumber, lineCoverage) in lineCoverages.toSortedMap()) {
+                append("DA:$lineNumber,${lineCoverage.visitCount}\n")
+                for ((branchBlockIndex, branchBlock) in lineCoverage.branchBlocks.withIndex()) {
+                    for ((branchIndex, branch) in branchBlock.withIndex()) {
+                        val branchVisitCount = this@LibraryCoverage.getBranchVisitCount(branch)
+                        append(
+                            "BRDA:$lineNumber,$branchBlockIndex,$branchIndex,$branchVisitCount\n"
+                        )
+                    }
+                }
+            }
+            val totalBranches = lineCoverages.values.sumOf { it.branchBlocks.sumOf { it.size } }
+            append("BRF:$totalBranches\n")
+            val coveredBranches =
+                lineCoverages.values.sumOf { lineCoverage ->
+                    lineCoverage.branchBlocks.sumOf { branchBlock ->
+                        branchBlock.count { branch ->
+                            val visitCount = this@LibraryCoverage.getBranchVisitCount(branch)
+                            visitCount > 0
+                        }
+                    }
+                }
+            append("BRH:$coveredBranches\n")
+            append("end_of_record\n")
+        }
+    }
+}
+
+/** Represents coverage information for a single line in a CQL source file. */
+internal class LineCoverage {
+    /** How many times an ELM element on this line was visited. */
+    var visitCount = 0
+
+    /** Each branch block is a collection of sibling (same-level) branches that cover this line. */
+    val branchBlocks = mutableListOf<Set<Branch>>()
+}
+
+/**
+ * Represents an ELM node as an execution branch. A branch can have child branches, e.g. in the case
+ * of If, List, In nodes.
+ */
+internal class Branch(val elm: Element, val children: List<Branch>) {
+    @Suppress("VariableNaming") var _location: Location? = null
+    val location: Location?
+        get() {
+            if (_location == null) {
+                val locator = elm.locator
+                if (locator != null) {
+                    _location = Location.fromLocator(locator)
+                }
+            }
+            return _location
+        }
+}
+
+/** Used to collect the branches of an ELM tree. */
+internal class BranchCollectionVisitor : BaseElmLibraryVisitor<List<Branch>, Unit>() {
+    override fun visitExpression(elm: Expression, context: Unit): List<Branch> {
+        return listOf(Branch(elm, super.visitExpression(elm, context)))
+    }
+
+    override fun aggregateResult(aggregate: List<Branch>, nextResult: List<Branch>): List<Branch> {
+        return aggregate + nextResult
+    }
+
+    override fun defaultResult(elm: Element, context: Unit): List<Branch> {
+        return listOf()
+    }
+}
+
+/** Converts an ELM tree to a [Branch] tree using [BranchCollectionVisitor]. */
+internal fun collectBranches(library: Library): List<Branch> {
+    return library.statements?.def?.map {
+        Branch(it, BranchCollectionVisitor().visitExpression(it.expression!!, Unit))
+    } ?: emptyList()
+}

Src/java/engine/src/main/kotlin/org/opencds/cqf/cql/engine/execution/CqlEngine.kt

@@ -49,6 +49,10 @@ constructor(val environment: Environment, engineOptions: MutableSet<Options>? =
         // ratio for certain elements such as expression and function
         // definitions and retrieves.
         EnableProfiling,
+
+        // Collect coverage information during execution. Coverage
+        // data can be exported in LCOV format after execution.
+        EnableCoverageCollection,
     }
 
     val state: State

Src/java/engine/src/main/kotlin/org/opencds/cqf/cql/engine/execution/EvaluationVisitor.kt

@@ -174,6 +174,8 @@ import org.opencds.cqf.cql.engine.runtime.TemporalHelper
 
 class EvaluationVisitor : BaseElmLibraryVisitor<Any?, State?>() {
     override fun visitExpression(elm: Expression, context: State?): Any? {
+        context?.markElementAsVisitedForCoverageReport(elm)
+
         try {
             return super.visitExpression(elm, context)
         } catch (e: CqlException) {

Src/java/engine/src/main/kotlin/org/opencds/cqf/cql/engine/execution/State.kt

@@ -21,6 +21,7 @@ import org.opencds.cqf.cql.engine.debug.DebugResult
 import org.opencds.cqf.cql.engine.debug.SourceLocator
 import org.opencds.cqf.cql.engine.exception.CqlException
 import org.opencds.cqf.cql.engine.exception.Severity
+import org.opencds.cqf.cql.engine.execution.CqlEngine.Options
 import org.opencds.cqf.cql.engine.runtime.DateTime
 import org.opencds.cqf.cql.engine.runtime.Tuple
 
@@ -99,6 +100,8 @@ constructor(
 
     var debugMap: DebugMap? = null
 
+    val globalCoverage by lazy { GlobalCoverage() }
+
     fun getCurrentLibrary(): Library? {
         return currentLibrary.peek()
     }
@@ -471,4 +474,14 @@ constructor(
         ensureDebugResult()
         debugResult!!.logDebugError(e)
     }
+
+    fun markElementAsVisitedForCoverageReport(elm: Element) {
+        if (engineOptions.contains(Options.EnableCoverageCollection)) {
+            val library =
+                checkNotNull(getCurrentLibrary()) {
+                    "No current library available when marking element for coverage report"
+                }
+            globalCoverage.markElementAsVisitedForCoverageReport(elm, library)
+        }
+    }
 }

Src/java/engine/src/test/kotlin/org/opencds/cqf/cql/engine/execution/CoverageTest.kt

@@ -0,0 +1,63 @@
+package org.opencds.cqf.cql.engine.execution
+
+import kotlin.test.assertEquals
+import kotlin.test.assertSame
+import org.hl7.elm.r1.If
+import org.hl7.elm.r1.Library
+import org.hl7.elm.r1.Literal
+import org.hl7.elm.r1.VersionedIdentifier
+import org.junit.jupiter.api.Test
+
+class CoverageTest : CqlTestBase() {
+    override val cqlSubdirectory = "CoverageTest"
+
+    @Test
+    fun exportLcovInfoTest() {
+        engine.state.engineOptions.add(CqlEngine.Options.EnableCoverageCollection)
+        engine.evaluate("Tests")
+        val actual =
+            engine.state.globalCoverage.exportLcovInfo(
+                listOf(
+                    VersionedIdentifier().withId("Library1"),
+                    VersionedIdentifier().withId("Library2"),
+                )
+            )
+        // Git converts \n to \r\n on checkout on Windows
+        val expected = this::class.java.getResource("CoverageTest/lcov.info").readText()
+        assertEqualsIgnoringLineEndings(expected, actual)
+    }
+
+    @Test
+    fun branchVisitCountTest() {
+        val lib = Library()
+        val coverage = LibraryCoverage(lib)
+        val elm = Literal()
+        coverage.markVisited(elm)
+        coverage.markVisited(elm)
+        val branch = Branch(elm, emptyList())
+
+        assertEquals(2, coverage.getBranchVisitCount(branch))
+    }
+
+    @Test
+    fun branchCollectionVisitorTest() {
+        val conditionNode = Literal()
+        val thenNode = Literal()
+        val elseNode = Literal()
+        val ifNode = If().withCondition(conditionNode).withThen(thenNode).withElse(elseNode)
+        val visitor = BranchCollectionVisitor()
+        val branches = visitor.visitExpression(ifNode, Unit)
+
+        assertEquals(1, branches.size)
+        assertSame(branches[0].elm, ifNode)
+        assertEquals(3, branches[0].children.size)
+        assertSame(conditionNode, branches[0].children[0].elm)
+        assertSame(thenNode, branches[0].children[1].elm)
+        assertSame(elseNode, branches[0].children[2].elm)
+        assertEquals(Literal(), Literal())
+    }
+
+    private fun assertEqualsIgnoringLineEndings(expected: String, actual: String) {
+        assertEquals(expected.replace("\r\n", "\n"), actual.replace("\r\n", "\n"))
+    }
+}

Src/java/engine/src/test/resources/org/opencds/cqf/cql/engine/execution/CoverageTest/Library1.cql

@@ -0,0 +1,30 @@
+library Library1
+
+include Library2
+
+define expr1: 1 + 2
+
+define expr2: Library2.expr1 + 3
+
+define expr3: Library2.func1(1, 2, true)
+
+define expr4: if true then 1 else
+  0
+
+define expr5: if true then 1
+  else 0
+
+define expr6: if
+  true then 1 else 0
+
+define expr7: if false then 1 else
+  0
+
+define expr8: if false then 1
+  else 0
+
+define expr9: AnyTrue({
+    false,
+    true,
+    false
+ })

Src/java/engine/src/test/resources/org/opencds/cqf/cql/engine/execution/CoverageTest/Library2.cql

@@ -0,0 +1,8 @@
+library Library2
+
+define expr1: if true then 1 else 0
+
+define expr2: 2
+
+define function func1(x Integer, y Integer, z Boolean):
+  if z then x + y else x - y

Src/java/engine/src/test/resources/org/opencds/cqf/cql/engine/execution/CoverageTest/Tests.cql

@@ -0,0 +1,18 @@
+library Tests
+
+include Library1
+
+define function assertEquals(expected Integer, actual Integer):
+  Message('Assertion failed', not (expected = actual), 'Undefined', 'Error',  'Expected ' + ToString(expected) + ' Found ' + ToString(actual))
+define function assertEquals(expected Boolean, actual Boolean):
+  Message('Assertion failed', not (expected = actual), 'Undefined', 'Error',  'Expected ' + ToString(expected) + ' Found ' + ToString(actual))
+
+define test1: assertEquals(Library1.expr1, 3)
+define test2: assertEquals(Library1.expr2, 4)
+define test3: assertEquals(Library1.expr3, 3)
+define test4: assertEquals(Library1.expr4, 1)
+define test5: assertEquals(Library1.expr5, 1)
+define test6: assertEquals(Library1.expr6, 1)
+define test7: assertEquals(Library1.expr7, 0)
+define test8: assertEquals(Library1.expr8, 0)
+define test9: assertEquals(Library1.expr9, true)