Test example notebooks against latest code

.github/workflows/pr.yml

@@ -20,6 +20,7 @@ jobs:
           args: >-
             test
             compileIntegrationTestKotlin
+            compileExamplesTestKotlin
             :build-logic:check
             :library:apiCheck
 

build-logic/src/main/kotlin/com/gabrielfeo/examples-test-suite.gradle.kts

@@ -0,0 +1,61 @@
+@file:Suppress("UnstableApiUsage")
+
+package com.gabrielfeo
+
+plugins {
+    id("org.jetbrains.kotlin.jvm")
+}
+
+testing {
+    suites {
+        register<JvmTestSuite>("examplesTest") {
+            useKotlinTest()
+        }
+    }
+}
+
+kotlin {
+    target {
+        val main by compilations.getting
+        val examplesTest by compilations.getting
+        examplesTest.associateWith(main)
+    }
+}
+
+val examples = fileTree(rootDir) {
+    include("examples/**")
+    exclude("**/build", "**/.*")
+}
+
+tasks.named("processExamplesTestResources", ProcessResources::class) {
+    from(examples)
+}
+
+val downloadPipRequirements by tasks.registering(Exec::class) {
+    val requirementsFiles = examples.filter { it.name == "requirements.txt" }
+    inputs.files(requirementsFiles)
+        .withPropertyName("requirementsFiles")
+        .withPathSensitivity(PathSensitivity.NONE)
+        .skipWhenEmpty()
+    val downloadDir = layout.buildDirectory.dir("pip-requirements")
+    outputs.dir(downloadDir)
+    commandLine("pip3", "download")
+    workingDir(downloadDir)
+    argumentProviders += CommandLineArgumentProvider {
+        requirementsFiles.files.flatMap { listOf("-r", it.absolutePath) }
+    }
+}
+
+tasks.named<Test>("examplesTest") {
+    inputs.files(downloadPipRequirements)
+        .withPropertyName("downloadedPipRequirements")
+        .withPathSensitivity(PathSensitivity.NONE)
+    systemProperty(
+        "downloaded-requirements-path",
+        downloadPipRequirements.map { it.outputs.files.singleFile }.get().relativeTo(workingDir).path,
+    )
+}
+
+tasks.named("check") {
+    dependsOn("examplesTest")
+}

build-logic/src/main/kotlin/com/gabrielfeo/test-suites.gradle.kts → build-logic/src/main/kotlin/com/gabrielfeo/integration-test-suite.gradle.kts

@@ -7,9 +7,7 @@ plugins {
 
 testing {
     suites {
-        // 'test' is registered by default
-        register<JvmTestSuite>("integrationTest")
-        withType<JvmTestSuite>().configureEach {
+        register<JvmTestSuite>("integrationTest") {
             useKotlinTest()
         }
     }

build-logic/src/main/kotlin/com/gabrielfeo/kotlin-jvm-library.gradle.kts

@@ -1,3 +1,5 @@
+@file:Suppress("UnstableApiUsage")
+
 package com.gabrielfeo
 
 plugins {
@@ -14,3 +16,11 @@ java {
         useRuntimeClasspathVersions()
     }
 }
+
+testing {
+    suites {
+        named<JvmTestSuite>("test") {
+            useKotlinTest()
+        }
+    }
+}

examples/build.gradle.kts

@@ -30,40 +30,9 @@ exampleTestTasks += tasks.register("runExampleProject") {
     dependsOn(":examples:example-project:run")
 }
 
-val notebooksDir = file("example-notebooks")
-val notebooks = fileTree(notebooksDir) { include("*.ipynb") }
-val venvDir = project.layout.buildDirectory.asFile.map { File(it, "venv") }
-
-val createPythonVenv by tasks.registering(Exec::class) {
-    val requirements = File(notebooksDir, "requirements.txt")
-    val venv = venvDir.get()
-    commandLine(
-        "bash", "-c",
-        "python3 -m venv --upgrade-deps $venv "
-            + "&& source $venv/bin/activate "
-            + "&& pip install --upgrade pip"
-            + "&& pip install -r $requirements"
-    )
-}
-
-exampleTestTasks += notebooks.map { notebook ->
-    val buildDir = project.layout.buildDirectory.asFile.get()
-    tasks.register<Exec>("run${notebook.nameWithoutExtension}Notebook") {
-        group = "Application"
-        description = "Runs the '${notebook.name}' notebook with 'jupyter nbconvert --execute'"
-        val venv = venvDir.get()
-        dependsOn(createPythonVenv)
-        commandLine(
-            "bash", "-c",
-            "source $venv/bin/activate "
-                + "&& jupyter nbconvert --execute --to ipynb --output-dir='$buildDir' '$notebook'"
-        )
-    }
-}
-
 val runAll = tasks.register("runAll") {
     group = "Application"
-    description = "Runs everything in 'examples' directory"
+    description = "Runs everything in 'examples' directory, except for notebooks (moved to exampleTests suite)"
     dependsOn(exampleTestTasks)
 }
 

examples/example-notebooks/MostFrequentBuilds.ipynb

@@ -121,7 +121,7 @@
     "    }.toList(LinkedList())\n",
     "}\n",
     "\n",
-    "println(\"${builds.size} builds\")\n",
+    "println(\"Collected ${builds.size} builds from the API\")\n",
     "check(builds.isNotEmpty()) { \"No builds found. Adjust query and try again.\" }"
    ]
   },
@@ -840,9 +840,7 @@
     }
    },
    "outputs": [],
-   "source": [
-    "%use kandy(v=0.6.0)"
-   ]
+   "source": "%use kandy(v=0.6.0)"
   },
   {
    "cell_type": "code",

library/build.gradle.kts

@@ -1,10 +1,12 @@
+import org.jetbrains.kotlin.gradle.dsl.JvmTarget
 import org.jetbrains.kotlin.gradle.dsl.KotlinVersion
 import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
 
 plugins {
     id("com.gabrielfeo.published-kotlin-jvm-library")
     id("com.gabrielfeo.develocity-api-code-generation")
-    id("com.gabrielfeo.test-suites")
+    id("com.gabrielfeo.integration-test-suite")
+    id("com.gabrielfeo.examples-test-suite")
     alias(libs.plugins.kotlin.jupyter)
 }
 
@@ -14,10 +16,6 @@ tasks.processJupyterApiResources {
     )
 }
 
-tasks.named<Test>("integrationTest") {
-    environment("DEVELOCITY_API_LOG_LEVEL", "DEBUG")
-}
-
 dependencies {
     constraints {
         implementation(libs.okio)
@@ -81,6 +79,12 @@ publishing {
             from(components["java"])
             pom(libraryPom)
         }
+        register<MavenPublication>("unsignedSnapshotDevelocityApiKotlin") {
+            artifactId = "develocity-api-kotlin"
+            version = "SNAPSHOT"
+            from(components["java"])
+            pom(libraryPom)
+        }
         register<MavenPublication>("relocation") {
             artifactId = "gradle-enterprise-api-kotlin"
             pom {
@@ -102,3 +106,18 @@ tasks.named("compileKotlin", KotlinCompile::class) {
         languageVersion = KotlinVersion.KOTLIN_1_8
     }
 }
+
+tasks.withType<Test>().configureEach {
+    environment("DEVELOCITY_API_LOG_LEVEL", "DEBUG")
+    providers.environmentVariablesPrefixedBy("DEVELOCITY_API_").get().forEach { (name, value) ->
+        inputs.property("${name}.hashCode", value.hashCode())
+    }
+}
+
+val publishUnsignedSnapshotDevelocityApiKotlinPublicationToMavenLocal by tasks.getting
+
+tasks.named<Test>("examplesTest") {
+    inputs.files(files(publishUnsignedSnapshotDevelocityApiKotlinPublicationToMavenLocal))
+        .withPropertyName("snapshotPublicationArtifacts")
+        .withNormalizer(ClasspathNormalizer::class)
+}

library/src/examplesTest/kotlin/com/gabrielfeo/develocity/api/example/JsonParser.kt

@@ -0,0 +1,20 @@
+package com.gabrielfeo.develocity.api.example
+
+import com.squareup.moshi.Moshi
+import okio.buffer
+import okio.source
+import java.nio.file.Path
+
+object JsonAdapter {
+
+    private val jsonAdapter = Moshi.Builder().build().adapter(Map::class.java)
+
+    fun fromJson(path: Path): Map<*, *>? =
+        jsonAdapter.fromJson(path.source().buffer())
+
+    fun toJson(map: Map<*, *>?): String =
+        jsonAdapter.toJson(map)
+
+    fun toPrettyJson(map: Map<*, *>?): String =
+        jsonAdapter.indent("  ").toJson(map)
+}

library/src/examplesTest/kotlin/com/gabrielfeo/develocity/api/example/Resources.kt

@@ -0,0 +1,17 @@
+package com.gabrielfeo.develocity.api.example
+
+import java.nio.file.Path
+import kotlin.io.path.ExperimentalPathApi
+import kotlin.io.path.copyToRecursively
+import kotlin.io.path.createParentDirectories
+import kotlin.io.path.div
+
+
+@OptIn(ExperimentalPathApi::class)
+fun Any.copyFromResources(path: String, targetDir: Path) {
+    val examples = requireNotNull(this::class.java.getResource(path))
+    val sourcePath = Path.of(examples.toURI())
+    val destPath = targetDir / path.removePrefix("/")
+    destPath.createParentDirectories()
+    sourcePath.copyToRecursively(destPath, followLinks = false, overwrite = true)
+}

library/src/examplesTest/kotlin/com/gabrielfeo/develocity/api/example/Shell.kt

@@ -0,0 +1,33 @@
+package com.gabrielfeo.develocity.api.example
+
+import kotlinx.coroutines.CoroutineStart.UNDISPATCHED
+import kotlinx.coroutines.async
+import kotlinx.coroutines.launch
+import kotlinx.coroutines.runBlocking
+import java.nio.file.Path
+
+fun runInShell(workDir: Path, vararg command: String) =
+    runInShell(workDir, command.joinToString(" "))
+
+fun runInShell(workDir: Path, command: String): String {
+    val process = ProcessBuilder("bash", "-c", command).apply {
+        directory(workDir.toFile())
+        // Ensure the test's build toolchain is used (not whatever JAVA_HOME is set to)
+        environment()["JAVA_HOME"] = System.getProperty("java.home")
+    }.start()
+    val stdout = runBlocking {
+        launch(start = UNDISPATCHED) {
+            process.errorStream.bufferedReader().lineSequence()
+                .onEach(System.err::println)
+                .joinToString("\n")
+        }
+        async(start = UNDISPATCHED) {
+            process.inputStream.bufferedReader().lineSequence()
+                .onEach(System.out::println)
+                .joinToString("\n")
+        }.await()
+    }
+    val exitCode = process.waitFor()
+    check(exitCode == 0) { "Exit code '$exitCode' for command: $command" }
+    return stdout
+}

library/src/examplesTest/kotlin/com/gabrielfeo/develocity/api/example/notebook/Jupyter.kt

@@ -0,0 +1,49 @@
+package com.gabrielfeo.develocity.api.example.notebook
+
+import com.gabrielfeo.develocity.api.example.copyFromResources
+import com.gabrielfeo.develocity.api.example.runInShell
+import java.nio.file.Path
+import kotlin.io.path.div
+import kotlin.io.path.nameWithoutExtension
+import kotlin.io.path.notExists
+
+class Jupyter(
+    val workDir: Path,
+    val venv: Path,
+) {
+
+    fun executeNotebook(path: Path): Path {
+        val outputPath = path.parent / "${path.nameWithoutExtension}-executed.ipynb"
+        runInShell(
+            workDir,
+            "source '${venv / "bin/activate"}' &&",
+            "jupyter nbconvert '$path'",
+            "--to ipynb",
+            "--execute",
+            "--output='$outputPath'",
+        )
+        return outputPath
+    }
+
+    fun replaceMagics(
+        path: Path,
+        replacePattern: Regex,
+        replacement: String
+    ): Path {
+        if ((workDir / "preprocessors.py").notExists()) {
+            copyFromResources("/preprocessors.py", workDir)
+        }
+        val outputPath = path.parent / "${path.nameWithoutExtension}-processed.ipynb"
+        runInShell(
+            workDir,
+            "source '${venv / "bin/activate"}' &&",
+            "jupyter nbconvert '$path'",
+            "--to ipynb",
+            "--output='$outputPath'",
+            "--NotebookExporter.preprocessors=preprocessors.ReplaceMagicsPreprocessor",
+            "--ReplaceMagicsPreprocessor.pattern='$replacePattern'",
+            "--ReplaceMagicsPreprocessor.replacement='$replacement'",
+        )
+        return outputPath
+    }
+}

library/src/examplesTest/kotlin/com/gabrielfeo/develocity/api/example/notebook/NotebookJson.kt

@@ -0,0 +1,38 @@
+package com.gabrielfeo.develocity.api.example.notebook
+
+class NotebookJson(
+    val properties: Map<String, Any?>,
+) {
+
+    val cells = properties["cells"] as List<Map<String, Any>>
+
+    val allOutputs by lazy {
+        cells.flatMap { (it["outputs"] as? List<Map<String, Any>>).orEmpty() }
+    }
+
+    val textOutputLines by lazy {
+        allOutputs
+            .filter { it["output_type"] == "stream" }
+            .flatMap { it["text"] as List<String> }
+    }
+
+    val dataframeOutputs by lazy {
+        executeOutputsByMimeType
+            .filter { (mimeType, _) -> mimeType == "application/kotlindataframe+json" }
+            .map { it.value as String }
+    }
+
+    private val executeOutputsByMimeType by lazy {
+        allOutputs
+            .filter { it["output_type"] == "execute_result" }
+            .flatMap { (it["data"] as Map<String, Any>).entries }
+    }
+
+    val kandyOutputs by lazy {
+        executeOutputsByMimeType
+            .filter { (mimeType, _) -> mimeType == "application/plot+json" }
+            .map { it.value as Map<String, Any> }
+    }
+}
+
+fun Map<*, *>?.asNotebookJson() = NotebookJson(this as Map<String, Any?>)