From d4b18f4a165b11c49d244c4deafad86ca8d9dce1 Mon Sep 17 00:00:00 2001 From: Ramon Roche Date: Thu, 8 May 2025 14:12:03 -0700 Subject: [PATCH 1/2] ci: build px4-dev containers on demand Signed-off-by: Ramon Roche --- .github/workflows/dev_container.yml | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/.github/workflows/dev_container.yml b/.github/workflows/dev_container.yml index 17bf12161cc7..e90c05082719 100644 --- a/.github/workflows/dev_container.yml +++ b/.github/workflows/dev_container.yml @@ -1,4 +1,3 @@ - name: Container build on: @@ -10,19 +9,21 @@ on: - 'release/**' tags: - 'v*' - paths: - - 'Tools/setup/ubuntu.sh' - - 'Tools/setup/requirements.txt' - - 'Tools/setup/Dockerfile' - - 'Tools/setup/docker-entrypoint.sh' pull_request: branches: - '*' paths: + - '.github/workflows/dev_container.yml' - 'Tools/setup/ubuntu.sh' - 'Tools/setup/requirements.txt' - 'Tools/setup/Dockerfile' - 'Tools/setup/docker-entrypoint.sh' + workflow_dispatch: + inputs: + px4_version: + description: 'PX4 version (e.g. v1.16.0)' + required: true + type: string jobs: setup: @@ -42,10 +43,15 @@ jobs: submodules: false fetch-depth: 0 + # If manual dispatch, take the user‐provided input - name: Set PX4 Tag Version id: px4_version run: | - echo "px4_version=$(git describe --tags --match 'v[0-9]*')" >> $GITHUB_OUTPUT + if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then + echo "px4_version=${{ github.event.inputs.px4_version }}" >> $GITHUB_OUTPUT + else + echo "px4_version=$(git describe --tags --match 'v[0-9]*')" >> $GITHUB_OUTPUT + fi - name: Extract metadata (tags, labels) for Docker id: meta @@ -83,14 +89,14 @@ jobs: - name: Login to Docker Hub uses: docker/login-action@v3 - if: ${{ startsWith(github.ref, 'refs/tags/') }} + if: ${{ startsWith(github.ref, 'refs/tags/') || (github.event_name == 'workflow_dispatch' && github.event.inputs.deploy_to_registry) }} with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Login to GitHub Container Registry uses: docker/login-action@v3 - if: ${{ startsWith(github.ref, 'refs/tags/') }} + if: ${{ startsWith(github.ref, 'refs/tags/') || (github.event_name == 'workflow_dispatch' && github.event.inputs.deploy_to_registry) }} with: registry: ghcr.io username: ${{ github.actor }} @@ -113,7 +119,7 @@ jobs: labels: ${{ needs.setup.outputs.meta_labels }} platforms: ${{ matrix.platform }} load: false - push: ${{ startsWith(github.ref, 'refs/tags/') }} + push: ${{ startsWith(github.ref, 'refs/tags/') || (github.event_name == 'workflow_dispatch' && github.event.inputs.deploy_to_registry) }} provenance: false cache-from: type=gha,version=1 cache-to: type=gha,version=1,mode=max @@ -125,7 +131,7 @@ jobs: packages: write runs-on: [runs-on,"runner=8cpu-linux-x64","image=ubuntu24-full-x64","run-id=${{ github.run_id }}",spot=false,extras=s3-cache] needs: [build, setup] - if: ${{ startsWith(github.ref, 'refs/tags/') }} + if: ${{ startsWith(github.ref, 'refs/tags/') || (github.event_name == 'workflow_dispatch' && github.event.inputs.deploy_to_registry) }} steps: - uses: runs-on/action@v1 - uses: actions/checkout@v4 From 0c31bbfdc597c516c4c4fc14e70c99474a0c505e Mon Sep 17 00:00:00 2001 From: Ramon Roche Date: Sat, 10 May 2025 13:50:26 -0700 Subject: [PATCH 2/2] tools: initial draft of manifes generation Signed-off-by: Ramon Roche --- Tools/ci/generate_manifest_json.py | 280 +++++++++++++++++++++++++++++ 1 file changed, 280 insertions(+) create mode 100755 Tools/ci/generate_manifest_json.py diff --git a/Tools/ci/generate_manifest_json.py b/Tools/ci/generate_manifest_json.py new file mode 100755 index 000000000000..ee09e907f4a0 --- /dev/null +++ b/Tools/ci/generate_manifest_json.py @@ -0,0 +1,280 @@ +#!/usr/bin/env python3 + +import argparse +import os +import sys +import json +import re +from kconfiglib import Kconfig + +kconf = Kconfig() + +# Supress warning output +kconf.warn_assign_undef = False +kconf.warn_assign_override = False +kconf.warn_assign_redun = False + +dconf = Kconfig() + +# Supress warning output +dconf.warn_assign_undef = False +dconf.warn_assign_override = False +dconf.warn_assign_redun = False + +source_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..') + +parser = argparse.ArgumentParser(description='Generate build targets') + +parser.add_argument('-v', '--verbose', dest='verbose', action='store_true', + help='Verbose Output') +parser.add_argument('-p', '--pretty', dest='pretty', action='store_true', + help='Pretty output instead of a single line') +# parser.add_argument('-g', '--groups', dest='group', action='store_true', +# help='Groups targets') +# parser.add_argument('-m', '--manifest', dest='manifest', action='store_true', +# help='Firmware manifest') +parser.add_argument('-f', '--filter', dest='filter', help='comma separated list of board names to use instead of all') + +args = parser.parse_args() +verbose = args.verbose + +board_filter = [] +if args.filter: + for board in args.filter.split(','): + board_filter.append(board) + +manifest = [] +build_configs = [] +grouped_targets = {} +excluded_boards = [] +excluded_manufacturers = ['atlflight'] +excluded_platforms = ['qurt'] +excluded_labels = [ + 'stackcheck', + 'nolockstep', 'replay', 'test', + 'uavcanv1', # TODO: fix and enable +] + +github_action_config = { 'include': build_configs } +extra_args = {} +if args.pretty: + extra_args['indent'] = 2 + +def chunks(arr, size): + # splits array into parts + for i in range(0, len(arr), size): + yield arr[i:i + size] + +def comma_targets(targets): + # turns array of targets into a comma split string + return ",".join(targets) + +def process_variants(targets): + # returns the + return [{"name": v} for v in targets] + +def process_target(px4board_file, target_name, defconfig_data, defconfig_file): + # reads through the board file and grabs + # useful information for building + ret = None + platform = None + toolchain = None + group = None + hardware = { + 'architecture': '', + 'vendor_id': '', + 'product_id': '', + 'chip': '', + 'productstr': '', + } + manifest_item = { + 'name': target_name, + 'manufacturer': '', + 'hardware': {}, + 'toolchain': '', + 'artifact': '', + 'sha256sum': '' + } + + board_data = process_boardfile(px4board_file) + # manifest_item['boardfile'] = board_data['boardfile'] + # manifest_item['debug'] = os.path.abspath(defconfig_file) + manifest_item['toolchain'] = board_data['toolchain'] + manifest_item['manufacturer'] = board_data['manufacturer'] + + if defconfig_data is not None: + manifest_item['manufacturer'] = defconfig_data['manufacturer'] + hardware = defconfig_data['hardware'] + hardware['architecture'] = board_data['architecture'] + + manifest_item['hardware'] = hardware + + return manifest_item + +def process_boardfile(boardtarget_file): + return_boardfile = { + 'toolchain': '', + 'boardfile': '', + 'manufacturer': '', + 'architecture': '', + } + if boardtarget_file.endswith("default.px4board") or \ + boardtarget_file.endswith("performance-test.px4board") or \ + boardtarget_file.endswith("bootloader.px4board"): + kconf.load_config(boardtarget_file, replace=True) + else: # Merge config with default.px4board + default_kconfig = re.sub(r'[a-zA-Z\d_-]+\.px4board', 'default.px4board', boardtarget_file) + kconf.load_config(default_kconfig, replace=True) + kconf.load_config(boardtarget_file, replace=False) + + return_boardfile['boardfile'] = os.path.abspath(boardtarget_file) + + if "BOARD_TOOLCHAIN" in kconf.syms: + return_boardfile['toolchain'] = kconf.syms["BOARD_TOOLCHAIN"].str_value + + if "CONFIG_CDCACM_VENDORSTR" in kconf.syms: + return_boardfile['manufacturer'] = kconf.syms["CONFIG_CDCACM_VENDORSTR"].str_value + + if "BOARD_ARCHITECTURE" in kconf.syms: + return_boardfile['architecture'] = kconf.syms["BOARD_ARCHITECTURE"].str_value + + return return_boardfile + +def load_defconfig(defconfig_file): + hardware = { + 'architecture': '', + 'vendor_id': '', + 'product_id': '', + 'chip': '', + 'description': '', + } + manifest_item = { + 'name': '', + 'manufacturer': '', + 'hardware': {}, + 'toolchain': '', + 'artifact': '', + 'sha256sum': '', + 'boardfile': '', + 'debug': '' + } + defconfig_data = process_defconfig(defconfig_file) + manifest_item['manufacturer'] = defconfig_data['manufacturer'] + # hardware['architecture'] = defconfig_data['architecture'] + hardware['vendor_id'] = defconfig_data['vendor_id'] + hardware['product_id'] = defconfig_data['product_id'] + hardware['chip'] = defconfig_data['chip'] + hardware['description'] = defconfig_data['description'] + manifest_item['hardware'] = hardware + + return manifest_item + +def process_defconfig(defconfig_file): + return_defconfig = { + 'manufacturer': '', + 'vendor_id': '', + 'product_id': '', + 'chip': '', + 'description': '', + } + defconfig = {} + + defconfig_available = False + abs_defconfig_path = os.path.abspath(defconfig_file) + if(os.path.isfile(defconfig_file)): + defconfig_available = True + + if(defconfig_available): + with open(defconfig_file, "r") as f: + for line in f: + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + key, value = line.split("=", 1) + value = value.strip('"') + defconfig[key] = value + + if "CONFIG_CDCACM_VENDORSTR" in defconfig.keys(): + return_defconfig['manufacturer'] = defconfig.get("CONFIG_CDCACM_VENDORSTR") + + if "CONFIG_CDCACM_VENDORID" in defconfig.keys(): + return_defconfig['vendor_id'] = defconfig.get("CONFIG_CDCACM_VENDORID") + + if "CONFIG_CDCACM_PRODUCTID" in defconfig.keys(): + return_defconfig['product_id'] = defconfig.get("CONFIG_CDCACM_PRODUCTID") + + if "CONFIG_ARCH_CHIP" in defconfig.keys(): + return_defconfig['chip'] = defconfig.get("CONFIG_ARCH_CHIP") + + if "CONFIG_CDCACM_PRODUCTSTR" in defconfig.keys(): + return_defconfig['description'] = defconfig.get("CONFIG_CDCACM_PRODUCTSTR") + + return return_defconfig + +# Look for board targets in the ./boards directory +if(verbose): + print("=======================") + print("= scanning for boards =") + print("=======================") + +targets_list = {} +# list of build targets +# [ +# { +# "name": "px4_fmu-v6x", +# "manufacturer": "CONFIG_CDCACM_VENDORSTR", +# "hardware": { +# "architecture": "CONFIG_BOARD_ARCHITECTURE", +# "vendor_id": "CONFIG_CDCACM_VENDORID", +# "product_id": "CONFIG_CDCACM_PRODUCTID", +# "chip": "CONFIG_ARCH_CHIP", +# "description": "CONFIG_CDCACM_PRODUCTSTR", +# }, +# "toolchain": "", +# "artifact": "", +# "sha256sum": "" +# } +# ] + + + +for manufacturer in os.scandir(os.path.join(source_dir, '../boards')): + if not manufacturer.is_dir(): + continue + if manufacturer.name in excluded_manufacturers: + if verbose: print(f'excluding manufacturer {manufacturer.name}') + continue + + for board in os.scandir(manufacturer.path): + if not board.is_dir(): + continue + + defconfig_file = os.path.join(board.path, 'nuttx-config/nsh/defconfig') + defconfig_data = load_defconfig(defconfig_file) + + for files in os.scandir(board.path): + + if files.is_file() and files.name.endswith('.px4board'): + + board_name = manufacturer.name + '_' + board.name + label = files.name[:-9] + target_name = manufacturer.name + '_' + board.name + '_' + label + + if board_filter and not board_name in board_filter: + if verbose: print(f'excluding board {board_name} ({target_name})') + continue + + if board_name in excluded_boards: + if verbose: print(f'excluding board {board_name} ({target_name})') + continue + + if label in excluded_labels: + if verbose: print(f'excluding label {label} ({target_name})') + continue + + target = process_target(files.path, target_name, defconfig_data, defconfig_file) + + if target is not None: + build_configs.append(target) + +print(json.dumps(build_configs, **extra_args))