From d7ac805ed28eacbef19e9346fb2cad74e47f5410 Mon Sep 17 00:00:00 2001 From: Ivan Savenko Date: Thu, 29 Aug 2024 19:10:36 +0000 Subject: [PATCH 01/14] Install java for Android CI --- .github/workflows/rebuildDependencies.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/rebuildDependencies.yml b/.github/workflows/rebuildDependencies.yml index e86b3c6..d26beb2 100644 --- a/.github/workflows/rebuildDependencies.yml +++ b/.github/workflows/rebuildDependencies.yml @@ -62,6 +62,12 @@ jobs: repository: 'vcmi/vcmi' ref: 'update_prebuilts' + - uses: actions/setup-java@v4 + if: ${{ startsWith(matrix.platform, 'android') }} + with: + distribution: 'temurin' + java-version: '11' + - name: Install dependencies run: source '${{github.workspace}}/CI/${{matrix.platform}}/before_install.sh' env: From 637200d587453e33d4efb59a3defc5cb1c2e4994 Mon Sep 17 00:00:00 2001 From: Ivan Savenko Date: Thu, 29 Aug 2024 19:12:13 +0000 Subject: [PATCH 02/14] Use Ubuntu 24.04 for mingw builds --- .github/workflows/rebuildDependencies.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/rebuildDependencies.yml b/.github/workflows/rebuildDependencies.yml index d26beb2..6b43f20 100644 --- a/.github/workflows/rebuildDependencies.yml +++ b/.github/workflows/rebuildDependencies.yml @@ -31,11 +31,11 @@ jobs: conan_profile: ios-arm64 conan_options: --options with_apple_system_libs=True - platform: mingw - os: ubuntu-22.04 + os: ubuntu-24.04 preset: windows-mingw-conan-linux conan_profile: mingw64-linux.jinja - platform: mingw-32 - os: ubuntu-22.04 + os: ubuntu-24.04 preset: windows-mingw-conan-linux conan_profile: mingw32-linux.jinja - platform: android-32 From e10129507728f5625c2f2e34353fea77b1fa029c Mon Sep 17 00:00:00 2001 From: Ivan Savenko Date: Thu, 29 Aug 2024 19:13:21 +0000 Subject: [PATCH 03/14] Install dependencies as separate CI step --- .github/workflows/rebuildDependencies.yml | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/.github/workflows/rebuildDependencies.yml b/.github/workflows/rebuildDependencies.yml index 6b43f20..93a5330 100644 --- a/.github/workflows/rebuildDependencies.yml +++ b/.github/workflows/rebuildDependencies.yml @@ -17,37 +17,44 @@ jobs: os: macos-13 preset: macos-conan-ninja-release conan_profile: macos-intel + conan_prebuilts: dependencies-mac-intel conan_options: --options with_apple_system_libs=True artifact_platform: intel - platform: mac-arm os: macos-13 preset: macos-arm-conan-ninja-release conan_profile: macos-arm + conan_prebuilts: dependencies-mac-arm conan_options: --options with_apple_system_libs=True artifact_platform: arm - platform: ios os: macos-13 preset: ios-release-conan-ccache conan_profile: ios-arm64 + conan_prebuilts: dependencies-ios conan_options: --options with_apple_system_libs=True - platform: mingw os: ubuntu-24.04 preset: windows-mingw-conan-linux conan_profile: mingw64-linux.jinja + conan_prebuilts: dependencies-mingw - platform: mingw-32 os: ubuntu-24.04 preset: windows-mingw-conan-linux conan_profile: mingw32-linux.jinja + conan_prebuilts: dependencies-mingw-32 - platform: android-32 os: macos-14 preset: android-conan-ninja-release conan_profile: android-32-ndk + conan_prebuilts: dependencies-android-32 conan_options: --conf tools.android:ndk_path=$ANDROID_NDK_ROOT artifact_platform: armeabi-v7a - platform: android-64 os: macos-14 preset: android-conan-ninja-release conan_profile: android-64-ndk + conan_prebuilts: dependencies-android-64 conan_options: --conf tools.android:ndk_path=$ANDROID_NDK_ROOT artifact_platform: arm64-v8a runs-on: ${{ matrix.os }} @@ -68,10 +75,13 @@ jobs: distribution: 'temurin' java-version: '11' - - name: Install dependencies + - name: Prepare CI + if: ${{ !startsWith(matrix.platform, 'android-linux') }} run: source '${{github.workspace}}/CI/${{matrix.platform}}/before_install.sh' - env: - VCMI_BUILD_PLATFORM: x64 + + - name: Install Conan Dependencies + if: "${{ matrix.conan_prebuilts != '' }}" + run: source '${{github.workspace}}/CI/install_conan_dependencies.sh' '${{matrix.conan_prebuilts}}' - name: Remove old packages run: rm -rf ~/.conan/data/ffmpeg ~/.conan/data/yasm ~/.conan/data/pkgconfig ~/.conan/data/xz_utils From 096974f3701965e14944c9d23ba69ae97b6f23f7 Mon Sep 17 00:00:00 2001 From: Ivan Savenko Date: Thu, 29 Aug 2024 19:17:35 +0000 Subject: [PATCH 04/14] Try to rebuild package from scratch to drop old binaries where possible --- .github/workflows/rebuildDependencies.yml | 30 +++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/.github/workflows/rebuildDependencies.yml b/.github/workflows/rebuildDependencies.yml index 93a5330..7efcdaf 100644 --- a/.github/workflows/rebuildDependencies.yml +++ b/.github/workflows/rebuildDependencies.yml @@ -83,8 +83,34 @@ jobs: if: "${{ matrix.conan_prebuilts != '' }}" run: source '${{github.workspace}}/CI/install_conan_dependencies.sh' '${{matrix.conan_prebuilts}}' - - name: Remove old packages - run: rm -rf ~/.conan/data/ffmpeg ~/.conan/data/yasm ~/.conan/data/pkgconfig ~/.conan/data/xz_utils + - name: Remove old binary packages + if: ${{ !startsWith(matrix.platform, 'android') }} + run: rm -rf ~/.conan/data/*/*/_/_/package + + # TODO: fix libiconv - fails to build on android (both macos and linux host) + - name: Remove old binary packages + if: ${{ startsWith(matrix.platform, 'android') }} + run: | + mv ~/.conan/data/libiconv ~/ + rm -rf ~/.conan/data/*/*/_/_/package + mv ~/libiconv ~/.conan/data + + # Completely remove packages that were confirmed to be rebuildable using upstream recipe/sources + # TODO: generate entire package from scratch instead of such cleanup + - name: Remove old recipes + run: | + rm -rf ~/.conan/data/ffmpeg + rm -rf ~/.conan/data/pkgconfig + rm -rf ~/.conan/data/xz_utils + rm -rf ~/.conan/data/sdl_mixer + rm -rf ~/.conan/data/sdl_image + rm -rf ~/.conan/data/sdl_ttf + rm -rf ~/.conan/data/sdl + + - name: Remove old recipes (non-apple) + if: ${{ matrix.platform != 'ios' && matrix.platform != 'mac-intel' && matrix.platform != 'mac-arm' }} + run: | + rm -rf ~/.conan/data/sqlite3 - name: Setup Python uses: actions/setup-python@v5 From 9d72d4b08a4c89d801b56e55abbd489a354ce6ae Mon Sep 17 00:00:00 2001 From: Ivan Savenko Date: Thu, 29 Aug 2024 19:54:03 +0000 Subject: [PATCH 05/14] Drop build tools like cmake from output package --- .github/workflows/rebuildDependencies.yml | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/.github/workflows/rebuildDependencies.yml b/.github/workflows/rebuildDependencies.yml index 7efcdaf..903b111 100644 --- a/.github/workflows/rebuildDependencies.yml +++ b/.github/workflows/rebuildDependencies.yml @@ -136,9 +136,18 @@ jobs: - name: Remove builds and source code run: "conan remove --builds --src --force '*'" - - name: Remove Android SDK - if: ${{ startsWith(matrix.platform, 'android') }} - run: rm -rf ~/.conan/data/android-ndk + - name: Remove build requirements + run: | + rm -rf ~/.conan/data/android-ndk + rm -rf ~/.conan/data/autoconf + rm -rf ~/.conan/data/automake + rm -rf ~/.conan/data/b2 + rm -rf ~/.conan/data/cmake + rm -rf ~/.conan/data/gnu-config + rm -rf ~/.conan/data/libtool + rm -rf ~/.conan/data/m4 + rm -rf ~/.conan/data/pkgconf + rm -rf ~/.conan/data/yasm - name: Create dependencies archive run: "tar --create --xz --file dependencies-${{matrix.platform}}.txz -C ~/.conan data" From a4e5f2ee14127bf33bac9c00147d3ca79c67207b Mon Sep 17 00:00:00 2001 From: Ivan Savenko Date: Thu, 29 Aug 2024 19:57:37 +0000 Subject: [PATCH 06/14] Try to use linux runner for Android build --- .github/workflows/rebuildDependencies.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/rebuildDependencies.yml b/.github/workflows/rebuildDependencies.yml index 903b111..62ddea9 100644 --- a/.github/workflows/rebuildDependencies.yml +++ b/.github/workflows/rebuildDependencies.yml @@ -44,18 +44,16 @@ jobs: conan_profile: mingw32-linux.jinja conan_prebuilts: dependencies-mingw-32 - platform: android-32 - os: macos-14 + os: ubuntu-24.04 preset: android-conan-ninja-release conan_profile: android-32-ndk conan_prebuilts: dependencies-android-32 - conan_options: --conf tools.android:ndk_path=$ANDROID_NDK_ROOT artifact_platform: armeabi-v7a - platform: android-64 - os: macos-14 + os: ubuntu-24.04 preset: android-conan-ninja-release conan_profile: android-64-ndk conan_prebuilts: dependencies-android-64 - conan_options: --conf tools.android:ndk_path=$ANDROID_NDK_ROOT artifact_platform: arm64-v8a runs-on: ${{ matrix.os }} defaults: @@ -76,7 +74,7 @@ jobs: java-version: '11' - name: Prepare CI - if: ${{ !startsWith(matrix.platform, 'android-linux') }} + if: ${{ !startsWith(matrix.platform, 'android') }} run: source '${{github.workspace}}/CI/${{matrix.platform}}/before_install.sh' - name: Install Conan Dependencies From 09fb828d90970e350ffd630a2002128ac5ca8af1 Mon Sep 17 00:00:00 2001 From: Ivan Savenko Date: Sat, 31 Aug 2024 11:26:11 +0000 Subject: [PATCH 07/14] Startup script is now parameter in matrix --- .github/workflows/rebuildDependencies.yml | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/.github/workflows/rebuildDependencies.yml b/.github/workflows/rebuildDependencies.yml index 62ddea9..f9002dc 100644 --- a/.github/workflows/rebuildDependencies.yml +++ b/.github/workflows/rebuildDependencies.yml @@ -15,6 +15,7 @@ jobs: include: - platform: mac-intel os: macos-13 + before_install: macos.sh preset: macos-conan-ninja-release conan_profile: macos-intel conan_prebuilts: dependencies-mac-intel @@ -22,6 +23,7 @@ jobs: artifact_platform: intel - platform: mac-arm os: macos-13 + before_install: macos.sh preset: macos-arm-conan-ninja-release conan_profile: macos-arm conan_prebuilts: dependencies-mac-arm @@ -29,27 +31,30 @@ jobs: artifact_platform: arm - platform: ios os: macos-13 + before_install: macos.sh preset: ios-release-conan-ccache conan_profile: ios-arm64 conan_prebuilts: dependencies-ios conan_options: --options with_apple_system_libs=True - - platform: mingw + - platform: mingw-x86-64 os: ubuntu-24.04 + before_install: mingw_x86_64.sh preset: windows-mingw-conan-linux conan_profile: mingw64-linux.jinja conan_prebuilts: dependencies-mingw - - platform: mingw-32 + - platform: mingw-x86 os: ubuntu-24.04 + before_install: mingw_x86.sh preset: windows-mingw-conan-linux conan_profile: mingw32-linux.jinja conan_prebuilts: dependencies-mingw-32 - - platform: android-32 + - platform: android-armeabi-v7a os: ubuntu-24.04 preset: android-conan-ninja-release conan_profile: android-32-ndk conan_prebuilts: dependencies-android-32 artifact_platform: armeabi-v7a - - platform: android-64 + - platform: android-arm64-v8a os: ubuntu-24.04 preset: android-conan-ninja-release conan_profile: android-64-ndk @@ -74,8 +79,8 @@ jobs: java-version: '11' - name: Prepare CI - if: ${{ !startsWith(matrix.platform, 'android') }} - run: source '${{github.workspace}}/CI/${{matrix.platform}}/before_install.sh' + if: "${{ matrix.before_install != '' }}" + run: source '${{github.workspace}}/CI/before_install/${{matrix.before_install}}' - name: Install Conan Dependencies if: "${{ matrix.conan_prebuilts != '' }}" From 16e80117be5909c70a8a4b29bdde645edf351136 Mon Sep 17 00:00:00 2001 From: Ivan Savenko Date: Sat, 31 Aug 2024 11:27:48 +0000 Subject: [PATCH 08/14] Remove one more build tool --- .github/workflows/rebuildDependencies.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/rebuildDependencies.yml b/.github/workflows/rebuildDependencies.yml index f9002dc..695ba8e 100644 --- a/.github/workflows/rebuildDependencies.yml +++ b/.github/workflows/rebuildDependencies.yml @@ -149,6 +149,7 @@ jobs: rm -rf ~/.conan/data/gnu-config rm -rf ~/.conan/data/libtool rm -rf ~/.conan/data/m4 + rm -rf ~/.conan/data/nasm rm -rf ~/.conan/data/pkgconf rm -rf ~/.conan/data/yasm From f3230101c2444ddbd75b4bf63dd88d2180922663 Mon Sep 17 00:00:00 2001 From: Ivan Savenko Date: Sat, 31 Aug 2024 12:27:06 +0000 Subject: [PATCH 09/14] Cleaned up unused parameters, clarified steps names --- .github/workflows/rebuildDependencies.yml | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/.github/workflows/rebuildDependencies.yml b/.github/workflows/rebuildDependencies.yml index 695ba8e..eee899f 100644 --- a/.github/workflows/rebuildDependencies.yml +++ b/.github/workflows/rebuildDependencies.yml @@ -16,50 +16,39 @@ jobs: - platform: mac-intel os: macos-13 before_install: macos.sh - preset: macos-conan-ninja-release conan_profile: macos-intel conan_prebuilts: dependencies-mac-intel conan_options: --options with_apple_system_libs=True - artifact_platform: intel - platform: mac-arm os: macos-13 before_install: macos.sh - preset: macos-arm-conan-ninja-release conan_profile: macos-arm conan_prebuilts: dependencies-mac-arm conan_options: --options with_apple_system_libs=True - artifact_platform: arm - platform: ios os: macos-13 before_install: macos.sh - preset: ios-release-conan-ccache conan_profile: ios-arm64 conan_prebuilts: dependencies-ios conan_options: --options with_apple_system_libs=True - platform: mingw-x86-64 os: ubuntu-24.04 before_install: mingw_x86_64.sh - preset: windows-mingw-conan-linux conan_profile: mingw64-linux.jinja conan_prebuilts: dependencies-mingw - platform: mingw-x86 os: ubuntu-24.04 before_install: mingw_x86.sh - preset: windows-mingw-conan-linux conan_profile: mingw32-linux.jinja conan_prebuilts: dependencies-mingw-32 - platform: android-armeabi-v7a os: ubuntu-24.04 - preset: android-conan-ninja-release conan_profile: android-32-ndk conan_prebuilts: dependencies-android-32 - artifact_platform: armeabi-v7a - platform: android-arm64-v8a os: ubuntu-24.04 - preset: android-conan-ninja-release conan_profile: android-64-ndk conan_prebuilts: dependencies-android-64 - artifact_platform: arm64-v8a runs-on: ${{ matrix.os }} defaults: run: @@ -86,12 +75,12 @@ jobs: if: "${{ matrix.conan_prebuilts != '' }}" run: source '${{github.workspace}}/CI/install_conan_dependencies.sh' '${{matrix.conan_prebuilts}}' - - name: Remove old binary packages + - name: Remove old binary packages (non-android) if: ${{ !startsWith(matrix.platform, 'android') }} run: rm -rf ~/.conan/data/*/*/_/_/package # TODO: fix libiconv - fails to build on android (both macos and linux host) - - name: Remove old binary packages + - name: Remove old binary packages (android) if: ${{ startsWith(matrix.platform, 'android') }} run: | mv ~/.conan/data/libiconv ~/ @@ -103,7 +92,6 @@ jobs: - name: Remove old recipes run: | rm -rf ~/.conan/data/ffmpeg - rm -rf ~/.conan/data/pkgconfig rm -rf ~/.conan/data/xz_utils rm -rf ~/.conan/data/sdl_mixer rm -rf ~/.conan/data/sdl_image From 4193f6628f7482712fce8e5da00fcdf57d45060d Mon Sep 17 00:00:00 2001 From: Ivan Savenko Date: Sat, 31 Aug 2024 12:30:41 +0000 Subject: [PATCH 10/14] Update todo list in readme --- README.md | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 069c7bb..1aadfd4 100644 --- a/README.md +++ b/README.md @@ -22,21 +22,13 @@ Current flow to update dependencies: - ffmpeg fails to find its dependencies when building with conan 1 + msvc 2019. Might be fixed in conan 2. - Qt fails to build due to broken string escaping in a path (conan 1 + msvc 2019) -- Switch Android CI to use Linux runners instead of macOS runners (both for prebuilts and for vcmi itself) +- Rebuild SDL_mixer and try to enable support for opus and flac. Needs investigation as to why libopus / libflac fail to build -- Upgrade ubuntu runner to ubuntu-24.04 and rebuild packages using newer mingw - -- Rebuild boost and disable boost_url which we don't use - -- Rebuild SDL (including SDL_mixer and SDL_image). - - Consider updating packages. - - Enable support for opus and flac. - - Remove unnecessary image formats such as gif and pcx - - Ensure that vcmi can load ogg/opus and flac as 'sounds' and not only as music +- Consider removing pcx support from SDL_image - Rebuild ffmpeg with libdav1d and av1 support enabled. Needs investigation as to why dav1d fails to build on mingw and on android. -- Rebuild all binaries in prebuilts package to ensure that everything is configured correctly and to replace any locally-built binaries with binaries from CI +- Find out why libiconv fails to rebuild on Android - Rebuild entire package from scratch using latest recipes from conan, to test current version of recipes From 58aeda12ffb21f7460341bed6e59aa74341da03e Mon Sep 17 00:00:00 2001 From: Ivan Savenko Date: Fri, 6 Sep 2024 17:01:56 +0000 Subject: [PATCH 11/14] Use new path to dependencies archive --- .github/workflows/rebuildDependencies.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/rebuildDependencies.yml b/.github/workflows/rebuildDependencies.yml index eee899f..9572880 100644 --- a/.github/workflows/rebuildDependencies.yml +++ b/.github/workflows/rebuildDependencies.yml @@ -35,20 +35,20 @@ jobs: os: ubuntu-24.04 before_install: mingw_x86_64.sh conan_profile: mingw64-linux.jinja - conan_prebuilts: dependencies-mingw + conan_prebuilts: dependencies-mingw-x86-64 - platform: mingw-x86 os: ubuntu-24.04 before_install: mingw_x86.sh conan_profile: mingw32-linux.jinja - conan_prebuilts: dependencies-mingw-32 + conan_prebuilts: dependencies-mingw-x86 - platform: android-armeabi-v7a os: ubuntu-24.04 conan_profile: android-32-ndk - conan_prebuilts: dependencies-android-32 + conan_prebuilts: dependencies-android-armeabi-v7a - platform: android-arm64-v8a os: ubuntu-24.04 conan_profile: android-64-ndk - conan_prebuilts: dependencies-android-64 + conan_prebuilts: dependencies-android-arm64-v8a runs-on: ${{ matrix.os }} defaults: run: From 312e65da277c5d8c336df3a3a6fc05b8f05b03f2 Mon Sep 17 00:00:00 2001 From: Ivan Savenko Date: Fri, 6 Sep 2024 17:09:38 +0000 Subject: [PATCH 12/14] Use newest boost recipe --- .github/workflows/rebuildDependencies.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/rebuildDependencies.yml b/.github/workflows/rebuildDependencies.yml index 9572880..85bfc96 100644 --- a/.github/workflows/rebuildDependencies.yml +++ b/.github/workflows/rebuildDependencies.yml @@ -91,11 +91,12 @@ jobs: # TODO: generate entire package from scratch instead of such cleanup - name: Remove old recipes run: | - rm -rf ~/.conan/data/ffmpeg + rm -rf ~/.conan/data/boost + rm -rf ~/.conan/data/ffmpeg rm -rf ~/.conan/data/xz_utils rm -rf ~/.conan/data/sdl_mixer rm -rf ~/.conan/data/sdl_image - rm -rf ~/.conan/data/sdl_ttf + rm -rf ~/.conan/data/sdl_ttf rm -rf ~/.conan/data/sdl - name: Remove old recipes (non-apple) From 44bec85a530f9b2bc1c69053380d342d88527549 Mon Sep 17 00:00:00 2001 From: Ivan Savenko Date: Fri, 20 Sep 2024 12:22:27 +0000 Subject: [PATCH 13/14] Use preinstalled python with pipx --- .github/workflows/rebuildDependencies.yml | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/.github/workflows/rebuildDependencies.yml b/.github/workflows/rebuildDependencies.yml index 85bfc96..8d048c6 100644 --- a/.github/workflows/rebuildDependencies.yml +++ b/.github/workflows/rebuildDependencies.yml @@ -104,13 +104,8 @@ jobs: run: | rm -rf ~/.conan/data/sqlite3 - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: '3.10' - - - name: Setup Conan - run: pip3 install 'conan<2.0' + - name: Install Conan + run: pipx install 'conan<2.0' - name: Generate conan profile run: | From e8fb9c41a790a36f18e8610ce9f0a0955866a3a5 Mon Sep 17 00:00:00 2001 From: Ivan Savenko Date: Fri, 20 Sep 2024 12:42:39 +0000 Subject: [PATCH 14/14] Adjust paths of mingw before_install file --- .github/workflows/rebuildDependencies.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/rebuildDependencies.yml b/.github/workflows/rebuildDependencies.yml index 8d048c6..76dd7f9 100644 --- a/.github/workflows/rebuildDependencies.yml +++ b/.github/workflows/rebuildDependencies.yml @@ -33,12 +33,12 @@ jobs: conan_options: --options with_apple_system_libs=True - platform: mingw-x86-64 os: ubuntu-24.04 - before_install: mingw_x86_64.sh + before_install: mingw.sh conan_profile: mingw64-linux.jinja conan_prebuilts: dependencies-mingw-x86-64 - platform: mingw-x86 os: ubuntu-24.04 - before_install: mingw_x86.sh + before_install: mingw.sh conan_profile: mingw32-linux.jinja conan_prebuilts: dependencies-mingw-x86 - platform: android-armeabi-v7a