conan v2 wip

.github/workflows/rebuildDependencies.yml

@@ -8,141 +8,271 @@ on:
   workflow_dispatch:
 
 jobs:
+  prepare:
+    runs-on: ubuntu-latest
+    outputs:
+      RELEASE_TAG: ${{ steps.release_tag.outputs.RELEASE_TAG }}
+    steps:
+    - name: Name release tag after today's date
+      id: release_tag
+      run: echo "RELEASE_TAG=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT
+
+    - name: Create draft release
+      uses: softprops/action-gh-release@v2
+      with:
+        tag_name: ${{ steps.release_tag.outputs.RELEASE_TAG }}
+        draft: true
+        prerelease: true
+
   build:
+    needs: prepare
     strategy:
       fail-fast: false
       matrix:
         include:
           - platform: mac-intel
-            os: macos-13
+            os: macos-15
             before_install: macos.sh
             conan_profile: macos-intel
-            conan_prebuilts: dependencies-mac-intel
-            conan_options: --options with_apple_system_libs=True
+            conan_system_libs: bzip2 libiconv sqlite3 zlib
           - platform: mac-arm
-            os: macos-13
+            os: macos-15
             before_install: macos.sh
             conan_profile: macos-arm
-            conan_prebuilts: dependencies-mac-arm
-            conan_options: --options with_apple_system_libs=True
+            conan_system_libs: bzip2 libiconv sqlite3 zlib
           - platform: ios
-            os: macos-13
+            os: macos-15
             before_install: macos.sh
             conan_profile: ios-arm64
-            conan_prebuilts: dependencies-ios
-            conan_options: --options with_apple_system_libs=True
-          - platform: mingw-x86-64
-            os: ubuntu-24.04
-            before_install: mingw.sh
-            conan_profile: mingw64-linux.jinja
-            conan_prebuilts: dependencies-mingw-x86-64
-          - platform: mingw-x86
-            os: ubuntu-24.04
-            before_install: mingw.sh
-            conan_profile: mingw32-linux.jinja
-            conan_prebuilts: dependencies-mingw-x86
+            conan_system_libs: bzip2 libiconv sqlite3 zlib
           - platform: android-armeabi-v7a
-            os: ubuntu-24.04
+            os: ubuntu-latest
+            before_install: android-32.sh
             conan_profile: android-32-ndk
-            conan_prebuilts: dependencies-android-armeabi-v7a
+            conan_system_libs: zlib
           - platform: android-arm64-v8a
-            os: ubuntu-24.04
+            os: ubuntu-latest
             conan_profile: android-64-ndk
-            conan_prebuilts: dependencies-android-arm64-v8a
+            conan_system_libs: zlib
+          - platform: windows-x64
+            os: windows-2022
+            conan_profile: msvc-x64
+            conan_options: -o "&:target_pre_windows10=True"
+          - platform: windows-x86
+            os: windows-2022
+            conan_profile: msvc-x86
+            conan_options: -o "&:target_pre_windows10=True"
     runs-on: ${{ matrix.os }}
     defaults:
       run:
         shell: bash
-
     steps:
-    - name: Checkout repository
+    - name: Checkout current
       uses: actions/checkout@v4
-      with:
-        repository: 'vcmi/vcmi'
-        ref: 'update_prebuilts'
+
+    - name: Prepare CI
+      run: |
+        echo CUSTOM_PATCHES_PATH="$(pwd)/conan_patches" >> $GITHUB_ENV
+        echo DEPS_FILE="dependencies-${{ matrix.platform }}.tgz" >> $GITHUB_ENV
+        [ -z '${{ matrix.before_install }}' ] || 'ci/${{ matrix.before_install }}'
 
     - uses: actions/setup-java@v4
       if: ${{ startsWith(matrix.platform, 'android') }}
       with:
         distribution: 'temurin'
-        java-version: '11'
+        java-version: '17'
 
-    - name: Prepare CI
-      if: "${{ matrix.before_install != '' }}"
-      run: source '${{github.workspace}}/CI/before_install/${{matrix.before_install}}'
-
-    - name: Install Conan Dependencies
-      if: "${{ matrix.conan_prebuilts != '' }}"
-      run: source '${{github.workspace}}/CI/install_conan_dependencies.sh' '${{matrix.conan_prebuilts}}'
-
-    - name: Remove old binary packages (non-android)
-      if: ${{ !startsWith(matrix.platform, 'android') }}
-      run: rm -rf ~/.conan/data/*/*/_/_/package
-
-    # TODO: fix libiconv - fails to build on android (both macos and linux host)
-    - name: Remove old binary packages (android)
-      if: ${{ startsWith(matrix.platform, 'android') }}
+    - name: Setup Conan Client
+      run: |
+        pipx install conan
+        conan profile detect
+
+    # CMake/Ninja version should be synced with runners
+    # https://github.com/actions/runner-images/tree/main/images
+    - name: Prepare platform tools
+      run: |
+        echo "
+        [platform_tool_requires]
+        cmake/3.31.6
+        ninja/1.12.1
+
+        [conf]
+        tools.cmake.cmaketoolchain:generator=Ninja" >> $(conan profile path default)
+
+    - name: Install system libs recipes
+      if: ${{ matrix.conan_system_libs }}
       run: |
-        mv ~/.conan/data/libiconv ~/
-        rm -rf ~/.conan/data/*/*/_/_/package
-        mv ~/libiconv ~/.conan/data
+        systemLibsRepo='conan-system-libs'
+        git clone "https://github.com/kambala-decapitator/$systemLibsRepo.git" \
+          --depth 1 \
+          --no-tags \
+          --single-branch
+        cd "$systemLibsRepo"
+        for p in ${{ matrix.conan_system_libs }} ; do
+          conan create "$p" --user system
+        done
 
-    # Completely remove packages that were confirmed to be rebuildable using upstream recipe/sources
-    # TODO: generate entire package from scratch instead of such cleanup
-    - name: Remove old recipes
+    - name: Build recipes with our patches
       run: |
-        rm -rf ~/.conan/data/boost
-        rm -rf ~/.conan/data/ffmpeg
-        rm -rf ~/.conan/data/xz_utils
-        rm -rf ~/.conan/data/sdl_mixer
-        rm -rf ~/.conan/data/sdl_image
-        rm -rf ~/.conan/data/sdl_ttf
-        rm -rf ~/.conan/data/sdl
-
-    - name: Remove old recipes (non-apple)
-      if: ${{ matrix.platform != 'ios' && matrix.platform != 'mac-intel' && matrix.platform != 'mac-arm' }}
+        cciRepo='conan-center-index'
+        branchName='master'
+        recipePathQt='recipes/qt'
+
+        git clone "https://github.com/conan-io/$cciRepo.git" \
+          --branch "$branchName" \
+          --no-checkout \
+          --depth 1 \
+          --no-tags \
+          --single-branch \
+          --sparse
+        cd "$cciRepo"
+        git sparse-checkout set \
+          recipes/minizip \
+          recipes/flac \
+
+        git checkout
+
+        # versions must be synced with: conan_patches/<package>/conandata.yml
+        # if no custom patches are required for a package, it should be removed from here
+        for p in minizip/1.3.1 flac/1.4.2 ; do
+          IFS_OLD="$IFS"
+          IFS=/
+          read package version <<<"$p"
+          IFS="$IFS_OLD"
+
+          if [[ $package == qt ]] ; then
+            packagePath="$recipePathQt/5.x.x"
+          else
+            packagePath="recipes/$package/all"
+          fi
+
+          conan create $packagePath \
+            --version=$version \
+            --profile=../conan_profiles/${{ matrix.conan_profile }} \
+            --build=missing \
+            --test-folder= \
+            --core-conf core.sources.patch:extra_path=$CUSTOM_PATCHES_PATH
+        done
+
+    # TODO: remove LuaJIT when https://github.com/conan-io/conan-center-index/pull/26577 is merged
+    - name: Build LuaJIT & Qt from PR changes
       run: |
-        rm -rf ~/.conan/data/sqlite3
+        cciForkRepo='cci-fork'
+        branchName='vcmi'
+        recipePathQt='recipes/qt'
+
+        git clone "https://github.com/kambala-decapitator/conan-center-index.git" "$cciForkRepo" \
+          --branch "$branchName" \
+          --no-checkout \
+          --depth 1 \
+          --no-tags \
+          --single-branch \
+          --sparse
+        cd "$cciForkRepo"
+        git sparse-checkout set \
+          recipes/luajit \
+          $recipePathQt \
+
+        git checkout
+
+        for p in luajit/2.1.0-beta3 qt/5.15.16 ; do
+          IFS_OLD="$IFS"
+          IFS=/
+          read package version <<<"$p"
+          IFS="$IFS_OLD"
 
-    - name: Install Conan
-      run: pipx install 'conan<2.0'
+          if [[ $package == qt ]] ; then
+            packagePath="$recipePathQt/5.x.x"
+          else
+            packagePath="recipes/$package/all"
+          fi
+
+          # Windows workaround for https://bugreports.qt.io/browse/QTBUG-84543
+          PATH="/c/Strawberry/perl/bin:$PATH" conan create $packagePath \
+            --version=$version \
+            --profile=../conan_profiles/${{ matrix.conan_profile }} \
+            --build=missing \
+            --test-folder= \
+            --core-conf core.sources.patch:extra_path=$CUSTOM_PATCHES_PATH \
+            ${{ startsWith(matrix.platform, 'android') && '-o "qt/*:android_sdk=$ANDROID_HOME"' || '' }}
+        done
 
     - name: Generate conan profile
       run: |
-        conan profile new default --detect
         conan install . \
-          --install-folder=conan-generated \
-          --no-imports \
+          --output-folder=conan-generated \
           --build=missing \
-          --profile:build=default \
-          --profile:host=CI/conan/${{ matrix.conan_profile }} \
+          --profile=conan_profiles/${{ matrix.conan_profile }} \
           ${{ matrix.conan_options }}
-      env:
-        GENERATE_ONLY_BUILT_CONFIG: 1
 
     - name: Remove builds and source code
-      run: "conan remove --builds --src --force '*'"
-
-    - name: Remove build requirements
+      run: conan cache clean
+
+    - name: Get NDK path
+      if: ${{ startsWith(matrix.platform, 'android') }}
+      run: |
+        ndkPackage='android-ndk'
+        hexRegex='[[:xdigit:]]+'
+
+        ndkPackageRevision=$(conan list --format=compact "$ndkPackage/*:*" \
+          | egrep --only-matching "$ndkPackage/\\w+#$hexRegex:$hexRegex")
+        ndkPackagePath=$(conan cache path "$ndkPackageRevision")
+        ndkPath="$ndkPackagePath/bin"
+        echo "NDK directory: $ndkPath"
+
+    - name: Remove build requirements' binaries
       run: |
-        rm -rf ~/.conan/data/android-ndk
-        rm -rf ~/.conan/data/autoconf
-        rm -rf ~/.conan/data/automake
-        rm -rf ~/.conan/data/b2
-        rm -rf ~/.conan/data/cmake
-        rm -rf ~/.conan/data/gnu-config
-        rm -rf ~/.conan/data/libtool
-        rm -rf ~/.conan/data/m4
-        rm -rf ~/.conan/data/nasm
-        rm -rf ~/.conan/data/pkgconf
-        rm -rf ~/.conan/data/yasm
+        graphFile='graph.json'
+        packageListFile='pkglist.json'
+
+        conan graph info . \
+          --profile=conan_profiles/${{ matrix.conan_profile }} \
+          --format=json \
+          --build=never \
+          --no-remote \
+          > "$graphFile"
+        conan list \
+          --graph "$graphFile" \
+          --graph-context=build-only \
+          --format=json \
+          > "$packageListFile"
+        conan remove --list "$packageListFile" --confirm
+
+    - name: Create list of built packages
+      run: |
+        packageList="$(conan list --format=compact | tail -n +2)"
+        echo "CONAN_PACKAGES<<EOF
+        $packageList
+        EOF" >> $GITHUB_ENV
 
     - name: Create dependencies archive
-      run: "tar --create --xz --file dependencies-${{matrix.platform}}.txz -C ~/.conan data"
+      run: conan cache save --file "$DEPS_FILE" "*:*"
 
-    - name: Upload artifacts
+    - name: Upload artifact
       uses: actions/upload-artifact@v4
       with:
         name: dependencies-${{ matrix.platform }}
         compression-level: 0
-        path: 'dependencies-${{matrix.platform}}.txz'
+        path: ${{ env.DEPS_FILE }}
+
+    - name: Update release with artifact and text
+      uses: softprops/action-gh-release@v2
+      with:
+        tag_name: ${{ needs.prepare.outputs.RELEASE_TAG }}
+        files: ${{ env.DEPS_FILE }}
+        append_body: true
+        body: |
+          <details><summary><b>${{ matrix.platform }}</b> packages</summary>
+
+          ${{ env.CONAN_PACKAGES }}
+          </details>
+
+  release:
+    needs: [prepare, build]
+    runs-on: ubuntu-latest
+    steps:
+    - name: Publish release
+      uses: softprops/action-gh-release@v2
+      with:
+        tag_name: ${{ needs.prepare.outputs.RELEASE_TAG }}
+        draft: false

README.md

@@ -15,23 +15,8 @@ Current flow to update dependencies:
   - https://docs.conan.io/2/reference/conan_server.html (not recommended by Conan)
   - https://jfrog.com/
 
-- Switch to conan 2. Incomplete (and potentially outdated) PR can be found here: https://github.com/vcmi/vcmi/pull/1603 Will also require changing how we create final package - instead of archiving `~/.conan/data` we'll need to use `conan cache` command
-
-- Use Conan for msvc builds. Currently blocked by several issues, namely:
-  - Conan 1 does not works with latest Visual Studio 2022. We need to either use msvc 2019 or upgrade to conan 2.
-  - ffmpeg fails to find its dependencies when building with conan 1 + msvc 2019. Might be fixed in conan 2.
-  - Qt fails to build due to broken string escaping in a path (conan 1 + msvc 2019)
-
-- Rebuild SDL_mixer and try to enable support for opus and flac. Needs investigation as to why libopus / libflac fail to build
-
-- Consider removing pcx support from SDL_image
-
 - Rebuild ffmpeg with libdav1d and av1 support enabled. Needs investigation as to why dav1d fails to build on mingw and on android.
 
-- Find out why libiconv fails to rebuild on Android
-
-- Rebuild entire package from scratch using latest recipes from conan, to test current version of recipes
-
 - Run CI with full package rebuild on schedule (weekly? monthly?) to detect any regressions or breaking changes in CI or in used recipes
 
 - Automatically generate Github release with updated packages as part of CI. Should probably be done only for changes in main branch and/or for manually triggered workflows
@@ -49,4 +34,4 @@ With this approach we will be able to use following flow for new dependencies:
 
 After merging PR in vcmi/vcmi repository vcmi will use new dependencies.
 
-If vcmi/vcmi PR is discarded for one reason or another, changes in this repository will have to be discarded as well
+If vcmi/vcmi PR is discarded for one reason or another, changes in this repository will have to be discarded as well

ci/android-32.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+
+# for LuaJIT
+sudo apt install libc6-dev-i386

ci/macos.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+echo DEVELOPER_DIR=/Applications/Xcode_16.2.app >> $GITHUB_ENV