Skip to content

Commit 4a47ce0

Browse files
PerkseyPerksey
committed
Some cleanup, and add docs
1 parent 8cf95c2 commit 4a47ce0

File tree

4 files changed

+201
-9
lines changed

4 files changed

+201
-9
lines changed

‎Directory.Build.props

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
3131
<ManagePackageVersionsCentrally>true</ManagePackageVersionsCentrally>
3232
<CentralPackageTransitivePinningEnabled>true</CentralPackageTransitivePinningEnabled>
33+
<ImplicitUsings>enable</ImplicitUsings>
3334
</PropertyGroup>
3435

3536
<PropertyGroup Condition="'$(Configuration)' == 'Debug'">

‎Directory.Build.targets

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -147,18 +147,24 @@
147147
<IncludeSymbols>true</IncludeSymbols>
148148
<SymbolPackageFormat>snupkg</SymbolPackageFormat>
149149
</PropertyGroup>
150+
150151
<!-- Non-C# packages (i.e. metapackages, natives) -->
151152
<PropertyGroup Condition="'$(SilkNativePackage)' == 'true' or '$(SilkMetapackage)' == 'true'">
152153
<IncludeBuildOutput Condition="'$(SilkNativeHasAndroidJars)' != 'true'">false</IncludeBuildOutput>
153-
<NoWarn>NU5128;$(NoWarn)</NoWarn>
154+
<IncludeBuildOutput Condition="'$(TargetFramework)' != ''">false</IncludeBuildOutput>
155+
<IncludeBuildOutput Condition="'$(SilkNativeHasAndroidJars)' == 'true' and $([MSBuild]::GetTargetPlatformIdentifier('$(TargetFramework)')) == 'android'">true</IncludeBuildOutput>
156+
<NoWarn>NU5128;1591;$(NoWarn)</NoWarn>
157+
<GenerateDocumentationFile>false</GenerateDocumentationFile>
154158
</PropertyGroup>
159+
155160
<!-- Public API -->
156161
<PropertyGroup Condition="'$(SilkTrackPublicAPI)' != 'true' or '$(TargetFramework)' == ''">
157162
<SilkTrackPublicAPI>false</SilkTrackPublicAPI>
158163
</PropertyGroup>
159164
<ItemGroup Condition="'$(SilkTrackPublicAPI)' == 'true'">
160165
<PackageReference Include="Microsoft.CodeAnalysis.PublicApiAnalyzers" PrivateAssets="All" />
161166
</ItemGroup>
167+
162168
<!-- SourceLink -->
163169
<ItemGroup Condition="'$(SilkSourceLinkExempt)' == '' and '$(SilkNativePackage)' != 'true' and '$(SilkMetapackage)' != 'true'">
164170
<PackageReference Include="DotNet.ReproducibleBuilds" PrivateAssets="All"/>
@@ -175,6 +181,7 @@
175181
<WriteLinesToFile File="PublicAPI/$(TargetFramework)/PublicAPI.Unshipped.txt"
176182
Lines="@(SilkNewPublicAPILines)"
177183
Condition="'$(SilkTrackPublicAPI)' == 'true' and !Exists('PublicAPI/$(TargetFramework)/PublicAPI.Unshipped.txt')" />
184+
178185
<!-- Versioning -->
179186
<PropertyGroup>
180187
<SilkChangelog>$([System.IO.File]::ReadAllText("$(MSBuildThisFileDirectory)docs/CHANGELOG.md"))</SilkChangelog>
@@ -185,17 +192,23 @@
185192
<Output PropertyName="SilkReleaseNotes" TaskParameter="SilkReleaseNotes" />
186193
</SilkGetVersionInfoTask>
187194
<PropertyGroup>
195+
<SilkOriginalVersionSuffix>$(VersionSuffix)</SilkOriginalVersionSuffix>
188196
<Version>$(SilkVersion)</Version>
189197
<VersionSuffix Condition="'$(VersionSuffix)' == ''">$(SilkVersionSuffix)</VersionSuffix>
190198
<PackageReleaseNotes>$(SilkReleaseNotes)</PackageReleaseNotes>
191199
<PackageVersion Condition="'$(VersionSuffix)' == ''">$(SilkVersion)</PackageVersion>
192200
<PackageVersion Condition="'$(VersionSuffix)' != ''">$(SilkVersion)-$(VersionSuffix)</PackageVersion>
193201
</PropertyGroup>
202+
194203
<!-- Native Packaging -->
195204
<PropertyGroup>
196205
<SilkVersionTxtPath>$(MSBuildProjectDirectory)/version.txt</SilkVersionTxtPath>
197-
<PackageVersion Condition="Exists('$(SilkVersionTxtPath)')">$([System.IO.File]::ReadAllText("$(SilkVersionTxtPath)"))</PackageVersion>
206+
<PackageVersion Condition="Exists('$(SilkVersionTxtPath)')">$([System.IO.File]::ReadAllText("$(SilkVersionTxtPath)").Trim())</PackageVersion>
207+
<PackageVersion Condition="'$(SilkOriginalVersionSuffix)' != '' and $(PackageVersion.Contains('-'))">$(PackageVersion)$(SilkOriginalVersionSuffix)</PackageVersion>
208+
<PackageVersion Condition="'$(SilkOriginalVersionSuffix)' != '' and !$(PackageVersion.Contains('-'))">$(PackageVersion)-$(SilkOriginalVersionSuffix)</PackageVersion>
198209
</PropertyGroup>
210+
<Error Text="Native packages should have a version.txt file! Read documentation/for-contributors/build-system.md for more info."
211+
Condition="'$(SilkNativePackage)' == 'true' and !Exists('$(SilkVersionTxtPath)')" />
199212
<ItemGroup Condition="'$(SilkNativePackage)' == 'true'">
200213
<None Include="$(MSBuildProjectDirectory)/runtimes/**/*" Pack="true" PackagePath="runtimes" />
201214
<!-- bundled in the aar instead -->
@@ -207,6 +220,7 @@
207220
<AndroidLibrary Include="$(MSBuildProjectDirectory)/android/*.jar" Bind="true" Pack="true" />
208221
<TransformFile Include="$(MSBuildProjectDirectory)/android/*.xml" />
209222
</ItemGroup>
223+
210224
<!-- Trimming -->
211225
<PropertyGroup Condition="'$(SilkTrimmingExempt)' == ''">
212226
<IsTrimmable>true</IsTrimmable>

‎docs/for-contributors/build-system.md

Lines changed: 184 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,190 @@ top-level directory in `sources`. It is highly likely that we'll want to amend t
8383
SilkTouch from this as this is unlikely to be useful to an end user. This is done using cursed MSBuild patterns that I
8484
recommend just squinting at for ages until they click.
8585

86+
### Native Packaging
87+
88+
Native packaging is one of the most frustratingly difficult aspect of packaging a library such as Silk.NET in an
89+
effective, elegant way. Silk.NET 2.X had a fairly easy-to-maintain system, but there were some small issues we decided
90+
to factor into the rewrite.
91+
92+
The primary goals in the native packaging system used for Silk.NET 3.0 are:
93+
- As in 2.X, ensuring our native binaries come from a trusted source (CI), and that executing these builds is as simple
94+
as possible.
95+
- Making it as easy as possible to add a new native binary build.
96+
- Ensuring that the native binary builds are maximally low-maintenance, in-keeping with the native library author's
97+
intent, and resistant to breakage when we update the native library version used.
98+
99+
There are two parts to this: the GitHub Actions workflow and the MSBuild Usage. Obviously as this is the MSBuild Usage
100+
section, we'll discuss the latter here.
101+
102+
NuGet's native packaging scheme in the simple case is fairly obvious - the native binaries are placed into the
103+
`runtimes/rid/native` directory of the package where `rid` is replaced with the appropriate runtime identifier. For fat
104+
binaries, we omit the architecture suffix. We automatically do this in the `SilkShippingControl` target (which is really
105+
just a kitchen sink of MSBuild fluff) in `Directory.Build.targets`, where all of the binaries are added as `None` items
106+
that are packed into the `runtimes` directory. The binaries are picked up from the project directory in the same
107+
`runtimes` directory structure as that which is added to the package.
108+
109+
To create a new native package, first create a csproj with the following contents:
110+
```xml
111+
<Project Sdk="Microsoft.NET.Sdk">
112+
<PropertyGroup>
113+
<TargetFrameworks>net8.0</TargetFrameworks>
114+
<SilkDescription>Native binaries for LIBRARY_NAME_HERE.</SilkDescription>
115+
<SilkNativePackage>true</SilkNativePackage>
116+
</PropertyGroup>
117+
</Project>
118+
```
119+
120+
After that, create a `version.txt` that contains the `PackageVersion` for the native binary. Ideally you should create
121+
an `update.sh` script that will automatically update the submodule to the latest upstream release, and update the
122+
`version.txt` to contain that version. If your native library doesn't really have a versioning scheme (we've experienced
123+
this with some of the Google libraries e.g. ANGLE, SwiftShader, etc), then it's recommended that the version be set to
124+
a date-style version `YYYY.MM.DD` where the date used is the date of the commit the submodule is currently checked out
125+
to.
126+
127+
Android is a bit of a unique case, as we not only have native binaries, we also have Java JARs in some cases. These need
128+
to be exposed to the .NET for Android toolchain to ensure these JARs are accessible. This toolchain produces an `aar`
129+
file which is added to the NuGet package, which incidentally includes both the JAR and the native binaries. Therefore,
130+
we exclude the android binaries from the `runtimes` directory in this case. If you have JARs, add the following to the
131+
native package csproj:
132+
```diff
133+
<Project Sdk="Microsoft.NET.Sdk">
134+
<PropertyGroup>
135+
- <TargetFrameworks>net8.0</TargetFrameworks>
136+
+ <TargetFrameworks>net8.0;net8.0-android</TargetFrameworks>
137+
+ <SilkNativeHasAndroidJars>true</SilkNativeHasAndroidJars>
138+
+ <EnableDefaultAndroidItems>false</EnableDefaultAndroidItems>
139+
</PropertyGroup>
140+
</Project>
141+
```
142+
143+
The JARs, Proguard configurations, and XML transforms (for the .NET for Android generator) will be picked up from an
144+
`android` subdirectory of the project folder. Native binaries will be picked up from `runtimes/android*/native` as
145+
usual, but obviously merged into the `aar` as above.
146+
147+
iOS on the other hand is a lot simpler, however we still have some iOS-specific handling. Specifically, we inject a
148+
`targets` file that is pulled in by downstream packages to add the `NativeReference` with the appropriate flags. We have
149+
seen anecdotal evidence that modern .NET for iOS toolchains pull in `runtimes` `.a` files as `NativeReference`s
150+
automatically, however in some cases there are specific linker flags required which are not picked up automatically. The
151+
`.targets` file adds this. If these linker flags are required, add something similar to the following:
152+
```xml
153+
<Project Sdk="Microsoft.NET.Sdk">
154+
<PropertyGroup>
155+
<SilkNativeiOSLinkerFlags>-framework AudioToolbox -framework AVFoundation -framework CoreAudio -framework CoreBluetooth -framework CoreFoundation -framework CoreGraphics -framework CoreHaptics -framework CoreMotion -framework CoreVideo -framework GameController -framework IOKit -framework Metal -framework OpenGLES -framework QuartzCore -framework UIKit -framework Foundation</SilkNativeiOSLinkerFlags>
156+
</PropertyGroup>
157+
</Project>
158+
```
159+
160+
The `.targets` injected can be seen at `eng/native/nuget/NativeNuGetPackage.targets` with the `TO_BE_REPLACED`
161+
placeholders replaced in `Directory.Build.targets`.
162+
163+
## Native Build Workflow
164+
165+
As mentioned previously, we build all of our native binaries in CI to ensure they come from a trusted source and also to
166+
ensure silly mistakes like forgetting to update the binaries when we update the bindings don't happen. We check every
167+
single PR for changes to the native build and, if we detect any, tell the PR author that they need to declare those
168+
changes in their PR description. This is done by simply adding `/build-native sdl` for example in the description. This
169+
is to ensure an unrelated change or merge doesn't result in a rebuild of an unnecessary amount of native binaries, as
170+
was an issue with 2.X's build system. Also unlike 2.X, the binaries are committed straight to the PR rather than having
171+
a PR into that PR (as this was very annoying), in a single commit aggregating all the outputs from all of the builds.
172+
173+
The workflow is split into three jobs:
174+
1. **PR Check** - runs on every PR, evaluates what binaries the author has indicated should be rebuilt.
175+
2. **Native Build** - a matrix job that uses a strategy determined dynamically as part of the PR Check to run all of the
176+
required native builds on the appropriate runners.
177+
3. **Commit Native Binaries** - all the outputs from the matrix jobs are downloaded, aggregated, and then committed to
178+
the PR.
179+
180+
To add a new native build to this workflow, modify the `env` at the top of the `native.yml` GitHub Actions workflow:
181+
```yaml
182+
env:
183+
# A space-separated list of paths to native libraries to build.
184+
NATIVE_LIBRARY_PATHS: "sources/SDL/Native"
185+
# A space-separated list of submodule paths for each native library path. Use _ if a submodule is not used - this must
186+
# match the number of spaces in NATIVE_LIBRARY_PATHS.
187+
NATIVE_LIBRARY_SUBMODULE_PATHS: "eng/submodules/sdl"
188+
# A space-separated list of shorthands to the native library paths that will build the native library for each native
189+
# library path. This must match the number of spaces in NATIVE_LIBRARY_PATHS. If a shorthand builds multiple native
190+
# binary paths, these will be deduplicated.
191+
NATIVE_LIBRARY_SHORTHANDS: "SDL"
192+
```
193+
194+
This is obviously assuming the native library path is valid. After this, any PR that contains `/build-native whatever`
195+
where `whatever` is replaced with the "shorthand" added to `NATIVE_LIBRARY_SHORTHANDS` will run a native binary build on
196+
each PR change.
197+
198+
After that, create the native package csproj as described in the MSBuild Usage section and add `build-rid.ext` scripts
199+
where `rid` is replaced with a runtime identifier and `ext` is replaced with `sh` or `cmd` if the build is running on
200+
Windows. All `osx`/`ios`/`tvos` prefixed RID builds run on macOS, all `win` prefixed RID builds run on Windows, and all
201+
other builds run on Linux. All of this again in the project directory/the directory added to `NATIVE_LIBRARY_PATHS`.
202+
203+
Note that for Linux we strive to have compatibility with glibc 2.17 and above, which in our experience from 2.X is a
204+
happy medium in terms of wide compatibility and feature set in most cases. It's not easy to build for a specific glibc
205+
target on Linux, which is why we use `zig cc` for these targets. For CMake targets, this is easy as we include the
206+
relevant toolchain files all ready to use at `eng/native/cmake`. For the build scripts themselves, we include a
207+
`eng/native/buildsystem/download-zig.py` script which will download the zig toolchain to `eng/native/buildsystem/zig`,
208+
which should then be added to the `PATH`. This often looks similar to the following:
209+
```bash
210+
if [[ ! -z ${GITHUB_ACTIONS+x} ]]; then
211+
../../../eng/native/buildsystem/download-zig.py
212+
export PATH="$PATH:$(readlink -f "../../../eng/native/buildsystem/zig")"
213+
fi
214+
```
215+
216+
Note that there are no prerequisite actions run before the native build occurs in the Build job, so these need to be
217+
integrated into the build scripts, using the `GITHUB_ACTIONS` environment variable as appropriate. Other cases where
218+
this is used beyond downloading Zig is installing `apt` dependencies, installing Android toolchains using `sdkmanager`,
219+
etc.
220+
221+
### PR Check
222+
223+
First, the script located at `eng/native/buildsystem/workflow-stage1.sh` is run. This script outputs something similar
224+
to the following to `GITHUB_OUTPUT`:
225+
```
226+
workflow_filters<<EOF
227+
SDL: ["sources/SDL/Native/*", "eng/submodules/sdl"]
228+
EOF
229+
targets_referenced=SDL
230+
```
231+
232+
The `targets_referenced` is one of the main exports from this job. The `workflow_filters` is used as an input to the
233+
`dorny/paths-filter@v3` action which will determine which targets had any changes that match the patterns given in the
234+
output array. This is so we can tell the user off for missing out `/build-native`s. Note that we try to edit an existing
235+
comment so we're not constantly spamming the PR, which is why we use `peter-evans/find-comment@v3` and
236+
`peter-evans/create-or-update-comment@v4`.
237+
238+
After this Stage 1 script has run, and we've tried to locate an existing comment, we run the Stage 2 script to determine
239+
the matrix strategy and the contents of the comment we should add/update. This outputs something like the following to
240+
`GITHUB_OUTPUT`:
241+
```
242+
matrix_strategy<<EOF
243+
[
244+
{
245+
"target": "SDL",
246+
"runtime": "osx",
247+
"exec": "build-osx.sh",
248+
"dir": "sources/SDL/Native"
249+
}
250+
]
251+
EOF
252+
comment_to_write=Some of the native library builds modified in this PR were not referenced in the PR description. Please ensure that the PR description contains \`/build-native SDL\`. These libraries won't be rebuilt without this being specified. If you believe this is in error, then please write a comment explaining why and ignore this suggestion. This comment will be automatically updated if rectified.
253+
```
254+
255+
`matrix_strategy` is the actual JSON representation of what would be in the `matrix: strategy:` section of the GitHub
256+
Workflow for the Build job. This is determined by the `build-*.{sh,cmd}` scripts added to the `NATIVE_LIBRARY_PATHS`
257+
directories. `comment_to_write` is the... comment to write... Note that this Stage 2 script receives the output from the
258+
`find-comment` action and this will indicate whether the issues have been resolved as appropriate (i.e. if we've already
259+
told the PR author off and they've listened). If no comment should be written and we don't have a comment to update,
260+
this is simply omitted.
261+
262+
The rest of the workflow continues as expected - the Build job essentially does exactly what is expected from the
263+
`matrix_strategy` outputs and uploads the binaries as an artifact, which are all pulled down in the Commit Native
264+
Binaries job and aggregated into a single commit.
265+
266+
To better understand the workflow scripts, it's probably better to just read through the
267+
`eng/native/buildsystem/test-workflow-stages.sh` script as this has a number of different test cases, but the scripts
268+
aren't too hard to understand if you're familiar with Bash.
269+
86270
## NUKE
87271

88272
NUKE is used to provide an easy interface into both MSBuild and our other non-C# or otherwise auxiliary build tasks.

‎sources/SDL/Native/Silk.NET.SDL.Native.csproj

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2,18 +2,11 @@
22

33
<PropertyGroup>
44
<TargetFrameworks>net8.0;net8.0-android</TargetFrameworks>
5-
<TargetFrameworks Condition="$([MSBuild]::IsOSPlatform('Linux')) == false">$(TargetFrameworks);net8.0-ios</TargetFrameworks>
6-
<ImplicitUsings>enable</ImplicitUsings>
7-
<Nullable>enable</Nullable>
85
<SilkDescription>Native binaries for SDL3.</SilkDescription>
96
<SilkNativePackage>true</SilkNativePackage>
107
<SilkNativeHasAndroidJars>true</SilkNativeHasAndroidJars>
118
<SilkNativeiOSLinkerFlags>-framework AudioToolbox -framework AVFoundation -framework CoreAudio -framework CoreBluetooth -framework CoreFoundation -framework CoreGraphics -framework CoreHaptics -framework CoreMotion -framework CoreVideo -framework GameController -framework IOKit -framework Metal -framework OpenGLES -framework QuartzCore -framework UIKit -framework Foundation</SilkNativeiOSLinkerFlags>
129
<EnableDefaultAndroidItems>false</EnableDefaultAndroidItems>
13-
<GenerateDocumentationFile>false</GenerateDocumentationFile>
14-
<NoWarn>1591</NoWarn>
15-
<IncludeBuildOutput Condition="'$(TargetFramework)' != ''">false</IncludeBuildOutput>
16-
<IncludeBuildOutput Condition="'$(SilkNativeHasAndroidJars)' == 'true' and $([MSBuild]::GetTargetPlatformIdentifier('$(TargetFramework)')) == 'android'">true</IncludeBuildOutput>
1710
</PropertyGroup>
1811

1912
</Project>

0 commit comments

Comments
 (0)