From 2d7c2c339115884ee32192fd0eb4bf7c81de759b Mon Sep 17 00:00:00 2001 From: helmetcrest Date: Mon, 28 Oct 2024 21:54:59 -0700 Subject: [PATCH 1/5] Add placeholder ordo-benchmark and JMH benchmarks --- .github/workflows/pull_request.yml | 4 +- .../JavaApiCallBenchmarks.swift | 34 +++++++++ Makefile | 4 +- Package.swift | 49 +++++++++---- Samples/SwiftKitSampleApp/build.gradle | 70 +++++++------------ .../swift/swiftkit/JavaToSwiftBenchmark.java | 53 ++++++++++++++ .../org/swift/swiftkit/MySwiftClassTest.java | 7 +- .../ExampleSwiftLibrary/MySwiftLibrary.swift | 4 ++ 8 files changed, 158 insertions(+), 67 deletions(-) create mode 100644 Benchmarks/JavaApiCallBenchmarks/JavaApiCallBenchmarks.swift create mode 100644 Samples/SwiftKitSampleApp/src/jmh/java/org/swift/swiftkit/JavaToSwiftBenchmark.java diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 0005f1c4..6fb3a0ff 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -29,7 +29,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: Install System Dependencies - run: apt-get -qq update && apt-get -qq install -y make curl wget + run: apt-get -qq update && apt-get -qq install -y make curl wget libjemalloc2 libjemalloc-dev - name: Cache JDK id: cache-jdk uses: actions/cache@v4 @@ -86,7 +86,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: Install System Dependencies - run: apt-get -qq update && apt-get -qq install -y make curl wget + run: apt-get -qq update && apt-get -qq install -y make curl wget libjemalloc2 libjemalloc-dev - name: Cache JDK id: cache-jdk uses: actions/cache@v4 diff --git a/Benchmarks/JavaApiCallBenchmarks/JavaApiCallBenchmarks.swift b/Benchmarks/JavaApiCallBenchmarks/JavaApiCallBenchmarks.swift new file mode 100644 index 00000000..25c67074 --- /dev/null +++ b/Benchmarks/JavaApiCallBenchmarks/JavaApiCallBenchmarks.swift @@ -0,0 +1,34 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift.org project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of Swift.org project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import Benchmark +import Foundation +import JavaKit +import JavaKitNetwork + +@MainActor let benchmarks = { + var jvm: JavaVirtualMachine { + get throws { + try .shared() + } + } + Benchmark("Simple call to Java library") { benchmark in + for _ in benchmark.scaledIterations { + let environment = try jvm.environment() + + let urlConnectionClass = try JavaClass(environment: environment) + blackHole(urlConnectionClass.getDefaultAllowUserInteraction()) + } + } +} diff --git a/Makefile b/Makefile index 4aee0561..aeb3eedd 100644 --- a/Makefile +++ b/Makefile @@ -133,12 +133,12 @@ jextract-generate: jextract-swift generate-JExtract-interface-files swift run jextract-swift \ --package-name com.example.swift.generated \ --swift-module ExampleSwiftLibrary \ - --output-directory ${SAMPLES_DIR}/SwiftKitSampleApp/src/generated/java \ + --output-directory ${SAMPLES_DIR}/SwiftKitSampleApp/build/generated/sources/jextract/main \ $(BUILD_DIR)/jextract/ExampleSwiftLibrary/MySwiftLibrary.swiftinterface; \ swift run jextract-swift \ --package-name org.swift.swiftkit.generated \ --swift-module SwiftKitSwift \ - --output-directory ${SAMPLES_DIR}/SwiftKitSampleApp/src/generated/java \ + --output-directory ${SAMPLES_DIR}/SwiftKitSampleApp/build/generated/sources/jextract/main \ $(BUILD_DIR)/jextract/SwiftKitSwift/SwiftKit.swiftinterface diff --git a/Package.swift b/Package.swift index 006562b2..d4078695 100644 --- a/Package.swift +++ b/Package.swift @@ -94,18 +94,18 @@ let package = Package( // ==== Plugin for building Java code .plugin( - name: "JavaCompilerPlugin", - targets: [ - "JavaCompilerPlugin" - ] + name: "JavaCompilerPlugin", + targets: [ + "JavaCompilerPlugin" + ] ), // ==== Plugin for wrapping Java classes in Swift .plugin( - name: "Java2SwiftPlugin", - targets: [ - "Java2SwiftPlugin" - ] + name: "Java2SwiftPlugin", + targets: [ + "Java2SwiftPlugin" + ] ), // ==== jextract-swift (extract Java accessors from Swift interface files) @@ -140,6 +140,7 @@ let package = Package( .package(url: "https://github.com/swiftlang/swift-syntax.git", branch: "main"), .package(url: "https://github.com/apple/swift-argument-parser", from: "1.5.0"), .package(url: "https://github.com/apple/swift-collections.git", .upToNextMinor(from: "1.1.0")), + .package(url: "https://github.com/ordo-one/package-benchmark", .upToNextMajor(from: "1.4.0")), ], targets: [ .macro( @@ -225,16 +226,16 @@ let package = Package( ] ), .plugin( - name: "JavaCompilerPlugin", - capability: .buildTool() + name: "JavaCompilerPlugin", + capability: .buildTool() ), .plugin( - name: "Java2SwiftPlugin", - capability: .buildTool(), - dependencies: [ - "Java2Swift" - ] + name: "Java2SwiftPlugin", + capability: .buildTool(), + dependencies: [ + "Java2Swift" + ] ), .target( @@ -374,5 +375,23 @@ let package = Package( .unsafeFlags(["-I\(javaIncludePath)", "-I\(javaPlatformIncludePath)"]) ] ), + + .executableTarget( + name: "JavaApiCallBenchmarks", + dependencies: [ + "JavaRuntime", + "JavaKit", + "JavaKitNetwork", + .product(name: "Benchmark", package: "package-benchmark"), + ], + path: "Benchmarks/JavaApiCallBenchmarks", + swiftSettings: [ + .unsafeFlags(["-I\(javaIncludePath)", "-I\(javaPlatformIncludePath)"]), + .swiftLanguageMode(.v5), + ], + plugins: [ + .plugin(name: "BenchmarkPlugin", package: "package-benchmark") + ], + ), ] ) diff --git a/Samples/SwiftKitSampleApp/build.gradle b/Samples/SwiftKitSampleApp/build.gradle index e1b3d78d..7522d182 100644 --- a/Samples/SwiftKitSampleApp/build.gradle +++ b/Samples/SwiftKitSampleApp/build.gradle @@ -16,6 +16,7 @@ import org.swift.swiftkit.gradle.BuildUtils plugins { id("build-logic.java-application-conventions") + id "me.champeau.jmh" version "0.7.2" } group = "org.swift.swiftkit" @@ -31,43 +32,38 @@ java { } } -sourceSets { - generated { - java.srcDir "${buildDir}/generated/src/java/" - } - // Make the 'main' and 'test' source sets depend on the generated sources - main { - compileClasspath += sourceSets.generated.output - runtimeClasspath += sourceSets.generated.output - } - test { - compileClasspath += sourceSets.main.output - runtimeClasspath += sourceSets.main.output +def jextract = tasks.register("jextract", Exec) { + description = "Extracts Java accessor sources using jextract" + + outputs.dir(layout.buildDirectory.dir("generated/sources/jextract/main")) + inputs.dir("$rootDir/Sources/ExampleSwiftLibrary") // monitored library + + // any changes in the source generator sources also mean the resulting output might change + inputs.dir("$rootDir/Sources/JExtractSwift") + inputs.dir("$rootDir/Sources/JExtractSwiftTool") - compileClasspath += sourceSets.generated.output - runtimeClasspath += sourceSets.generated.output + workingDir = rootDir + commandLine "make" + args "jextract-generate" +} + +sourceSets { + main { + java { + srcDir(jextract) + } } } dependencies { implementation(project(':SwiftKit')) - generatedImplementation(project(':SwiftKit')) testImplementation(platform("org.junit:junit-bom:5.10.0")) testImplementation("org.junit.jupiter:junit-jupiter") } -configurations { - generatedImplementation.extendsFrom(mainImplementation) - generatedRuntimeOnly.extendsFrom(mainRuntimeOnly) -} - -tasks.named("compileJava").configure { - dependsOn("jextract") -} - -tasks.test { +tasks.named('test', Test) { useJUnitPlatform() } @@ -84,27 +80,15 @@ application { "--enable-native-access=ALL-UNNAMED", // Include the library paths where our dylibs are that we want to load and call - "-Djava.library.path=" + BuildUtils.javaLibraryPaths().join(":"), + "-Djava.library.path=" + BuildUtils.javaLibraryPaths(rootDir).join(":"), // Enable tracing downcalls (to Swift) "-Djextract.trace.downcalls=true" ] } -task jextract(type: Exec) { - description = "Extracts Java accessor sources using jextract" - outputs.dir(layout.buildDirectory.dir("generated")) - inputs.dir("$rootDir/Sources/ExampleSwiftLibrary") // monitored library - - // any changes in the source generator sources also mean the resulting output might change - inputs.dir("$rootDir/Sources/JExtractSwift") - inputs.dir("$rootDir/Sources/JExtractSwiftTool") - - workingDir = rootDir - commandLine "make" - args "jextract-generate" -} - -tasks.named("compileGeneratedJava").configure { - dependsOn jextract -} +jmh { + jvmArgsAppend = [ + "-Djava.library.path=" + BuildUtils.javaLibraryPaths(rootDir).join(":"), + ] +} \ No newline at end of file diff --git a/Samples/SwiftKitSampleApp/src/jmh/java/org/swift/swiftkit/JavaToSwiftBenchmark.java b/Samples/SwiftKitSampleApp/src/jmh/java/org/swift/swiftkit/JavaToSwiftBenchmark.java new file mode 100644 index 00000000..5ce9ee6b --- /dev/null +++ b/Samples/SwiftKitSampleApp/src/jmh/java/org/swift/swiftkit/JavaToSwiftBenchmark.java @@ -0,0 +1,53 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift.org project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of Swift.org project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +package org.swift.swiftkit; + +import java.util.concurrent.TimeUnit; + +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.infra.Blackhole; + +import com.example.swift.generated.MySwiftClass; + +public class JavaToSwiftBenchmark { + + @State(Scope.Benchmark) + public static class BenchmarkState { + MySwiftClass obj; + + @Setup(Level.Trial) + public void beforeALl() { + System.loadLibrary("swiftCore"); + System.loadLibrary("ExampleSwiftLibrary"); + + // Tune down debug statements so they don't fill up stdout + System.setProperty("jextract.trace.downcalls", "false"); + + obj = new MySwiftClass(1, 2); + } + } + + @Benchmark @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.NANOSECONDS) + public void simpleSwiftApiCall(BenchmarkState state, Blackhole blackhole) { + blackhole.consume(state.obj.makeRandomIntMethod()); + } +} diff --git a/Samples/SwiftKitSampleApp/src/test/java/org/swift/swiftkit/MySwiftClassTest.java b/Samples/SwiftKitSampleApp/src/test/java/org/swift/swiftkit/MySwiftClassTest.java index 1b98d1e0..069a4e41 100644 --- a/Samples/SwiftKitSampleApp/src/test/java/org/swift/swiftkit/MySwiftClassTest.java +++ b/Samples/SwiftKitSampleApp/src/test/java/org/swift/swiftkit/MySwiftClassTest.java @@ -14,14 +14,11 @@ package org.swift.swiftkit; -import com.example.swift.generated.MySwiftClass; +import static org.junit.jupiter.api.Assertions.assertEquals; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.condition.DisabledOnOs; -import org.junit.jupiter.api.condition.OS; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotNull; +import com.example.swift.generated.MySwiftClass; public class MySwiftClassTest { diff --git a/Sources/ExampleSwiftLibrary/MySwiftLibrary.swift b/Sources/ExampleSwiftLibrary/MySwiftLibrary.swift index 146601d0..b5c8f5fd 100644 --- a/Sources/ExampleSwiftLibrary/MySwiftLibrary.swift +++ b/Sources/ExampleSwiftLibrary/MySwiftLibrary.swift @@ -78,6 +78,10 @@ public class MySwiftClass { p("make int -> 12") return 12 } + + public func makeRandomIntMethod() -> Int { + return Int.random(in: 1..<256) + } } @_silgen_name("swift_getTypeByMangledNameInEnvironment") From a96d07761042e99d8f44a13f4446fc8928ab8f2e Mon Sep 17 00:00:00 2001 From: helmetcrest Date: Wed, 30 Oct 2024 03:47:35 +0000 Subject: [PATCH 2/5] Fix build error --- .../JavaApiCallBenchmarks.swift | 0 Benchmarks/Package.swift | 69 +++++++++++++++++++ Package.swift | 20 +----- Samples/SwiftKitSampleApp/build.gradle | 4 +- docker/Dockerfile | 4 +- 5 files changed, 75 insertions(+), 22 deletions(-) rename Benchmarks/{ => Benchmarks}/JavaApiCallBenchmarks/JavaApiCallBenchmarks.swift (100%) create mode 100644 Benchmarks/Package.swift diff --git a/Benchmarks/JavaApiCallBenchmarks/JavaApiCallBenchmarks.swift b/Benchmarks/Benchmarks/JavaApiCallBenchmarks/JavaApiCallBenchmarks.swift similarity index 100% rename from Benchmarks/JavaApiCallBenchmarks/JavaApiCallBenchmarks.swift rename to Benchmarks/Benchmarks/JavaApiCallBenchmarks/JavaApiCallBenchmarks.swift diff --git a/Benchmarks/Package.swift b/Benchmarks/Package.swift new file mode 100644 index 00000000..b8fbe580 --- /dev/null +++ b/Benchmarks/Package.swift @@ -0,0 +1,69 @@ +// swift-tools-version: 6.0 + +import PackageDescription + +import class Foundation.FileManager +import class Foundation.ProcessInfo + +// Note: the JAVA_HOME environment variable must be set to point to where +// Java is installed, e.g., +// Library/Java/JavaVirtualMachines/openjdk-21.jdk/Contents/Home. +func findJavaHome() -> String { + if let home = ProcessInfo.processInfo.environment["JAVA_HOME"] { + return home + } + + // This is a workaround for envs (some IDEs) which have trouble with + // picking up env variables during the build process + let path = "\(FileManager.default.homeDirectoryForCurrentUser.path()).java_home" + if let home = try? String(contentsOfFile: path, encoding: .utf8) { + if let lastChar = home.last, lastChar.isNewline { + return String(home.dropLast()) + } + + return home + } + + fatalError("Please set the JAVA_HOME environment variable to point to where Java is installed.") +} +let javaHome = findJavaHome() + +let javaIncludePath = "\(javaHome)/include" +#if os(Linux) + let javaPlatformIncludePath = "\(javaIncludePath)/linux" +#elseif os(macOS) + let javaPlatformIncludePath = "\(javaIncludePath)/darwin" +#else + // TODO: Handle windows as well + #error("Currently only macOS and Linux platforms are supported, this may change in the future.") +#endif + +let package = Package( + name: "benchmarks", + platforms: [ + .macOS("15.03") + ], + dependencies: [ + .package(path: "../"), + .package(url: "https://github.com/ordo-one/package-benchmark", .upToNextMajor(from: "1.4.0")), + ], + targets: [ + .executableTarget( + name: "JavaApiCallBenchmarks", + dependencies: [ + .product(name: "JavaRuntime", package: "swift-java"), + .product(name: "JavaKit", package: "swift-java"), + .product(name: "JavaKitNetwork", package: "swift-java"), + .product(name: "Benchmark", package: "package-benchmark"), + ], + path: "Benchmarks/JavaApiCallBenchmarks", + swiftSettings: [ + .unsafeFlags(["-I\(javaIncludePath)", "-I\(javaPlatformIncludePath)"]), + .swiftLanguageMode(.v5), + ], + plugins: [ + .plugin(name: "BenchmarkPlugin", package: "package-benchmark") + ] + ) + ] +) diff --git a/Package.swift b/Package.swift index d4078695..ac1d6842 100644 --- a/Package.swift +++ b/Package.swift @@ -374,24 +374,6 @@ let package = Package( .swiftLanguageMode(.v5), .unsafeFlags(["-I\(javaIncludePath)", "-I\(javaPlatformIncludePath)"]) ] - ), - - .executableTarget( - name: "JavaApiCallBenchmarks", - dependencies: [ - "JavaRuntime", - "JavaKit", - "JavaKitNetwork", - .product(name: "Benchmark", package: "package-benchmark"), - ], - path: "Benchmarks/JavaApiCallBenchmarks", - swiftSettings: [ - .unsafeFlags(["-I\(javaIncludePath)", "-I\(javaPlatformIncludePath)"]), - .swiftLanguageMode(.v5), - ], - plugins: [ - .plugin(name: "BenchmarkPlugin", package: "package-benchmark") - ], - ), + ) ] ) diff --git a/Samples/SwiftKitSampleApp/build.gradle b/Samples/SwiftKitSampleApp/build.gradle index 7522d182..3b79bd72 100644 --- a/Samples/SwiftKitSampleApp/build.gradle +++ b/Samples/SwiftKitSampleApp/build.gradle @@ -16,7 +16,7 @@ import org.swift.swiftkit.gradle.BuildUtils plugins { id("build-logic.java-application-conventions") - id "me.champeau.jmh" version "0.7.2" + id("me.champeau.jmh") version "0.7.2" } group = "org.swift.swiftkit" @@ -91,4 +91,4 @@ jmh { jvmArgsAppend = [ "-Djava.library.path=" + BuildUtils.javaLibraryPaths(rootDir).join(":"), ] -} \ No newline at end of file +} diff --git a/docker/Dockerfile b/docker/Dockerfile index 0660f117..c4711253 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -11,7 +11,9 @@ RUN apt-get update && apt-get install -y \ locales locales-all \ make \ libc6-dev \ - curl + curl \ + libjemalloc2 \ + libjemalloc-dev ENV LC_ALL=en_US.UTF-8 ENV LANG=en_US.UTF-8 ENV LANGUAGE=en_US.UTF-8 From 0a1b74252aeca1643cffcbed37b3d760b1ac9e9c Mon Sep 17 00:00:00 2001 From: Konrad `ktoso` Malawski Date: Thu, 31 Oct 2024 18:21:22 +0900 Subject: [PATCH 3/5] add benchmarking readme --- README.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/README.md b/README.md index 6bf9df2f..aae218ba 100644 --- a/README.md +++ b/README.md @@ -111,6 +111,28 @@ To run a simple example app showcasing the jextract (Java calling Swift) approac This will also generate the necessary sources (by invoking jextract, extracting the `Sources/ExampleSwiftLibrary`) and generating Java sources in `src/generated/java`. +## Benchmarks + +You can run Swift [ordo-one/package-benchmark](https://github.com/ordo-one/package-benchmark) and OpenJDK [JMH](https://github.com/openjdk/jmh) benchmarks in this project. + +Swift benchmarks are located under `Benchmarks/` and JMH benchmarks are currently part of the SwiftKit sample project: `Samples/SwiftKitSampleApp/src/jmh` because they depend on generated sources from the sample. + +To run **Swift benchmarks** you can: + +```bash +cd Benchmarks +swift package benchmark +``` + +In order to run JMH benchmarks you can: + +```bash +cd Samples/SwiftKitSampleApp +gradle jmh +``` + +Please read documentation of both performance testing tools and understand that results must be interpreted and not just taken at face value. Benchmarking is tricky and environment sensitive task, so please be careful when constructing and reading benchmarks and their results. If in doubt, please reach out on the forums. + ## User Guide More details about the project and how it can be used are available in [USER_GUIDE.md](USER_GUIDE.md) From 1d231bc21a78282e2d5c9070148d06a7557ce8e7 Mon Sep 17 00:00:00 2001 From: Konrad `ktoso` Malawski Date: Thu, 31 Oct 2024 18:31:15 +0900 Subject: [PATCH 4/5] Include benchmarks in PR validation: both JMH and Swift --- .github/workflows/pull_request.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 6fb3a0ff..7635abf2 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -69,6 +69,9 @@ jobs: run: | ./gradlew build -x test --no-daemon # just build ./gradlew build --info --no-daemon + - name: Gradle build (benchmarks) + run: | + ./gradlew compileJmh --info --no-daemon test-swift: name: Swift tests (swift:${{ matrix.swift_version }} jdk:${{matrix.jdk_vendor}} os:${{ matrix.os_version }}) @@ -130,3 +133,5 @@ jobs: - name: Build (Swift) Sample Apps run: | find Samples/ -name Package.swift -maxdepth 2 -exec swift build --package-path $(dirname {}) \;; + - name: Build (Swift) Benchmarks + run: "swift package --package-path Benchmarks/ benchmark list" From 80f91a0f3dfe5bb87f512ca48a8404a6a2328ab8 Mon Sep 17 00:00:00 2001 From: Konrad `ktoso` Malawski Date: Thu, 31 Oct 2024 18:47:07 +0900 Subject: [PATCH 5/5] disable Swift benchmark building in CI --- .github/workflows/pull_request.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 7635abf2..eea13444 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -133,5 +133,6 @@ jobs: - name: Build (Swift) Sample Apps run: | find Samples/ -name Package.swift -maxdepth 2 -exec swift build --package-path $(dirname {}) \;; - - name: Build (Swift) Benchmarks - run: "swift package --package-path Benchmarks/ benchmark list" + # TODO: Benchmark compile crashes in CI, enable when nightly toolchains in better shape. + # - name: Build (Swift) Benchmarks + # run: "swift package --package-path Benchmarks/ benchmark list"