diff --git a/.github/workflows/build-artifact.yml b/.github/workflows/build-artifact.yml index c2d4b15a..59c2503c 100644 --- a/.github/workflows/build-artifact.yml +++ b/.github/workflows/build-artifact.yml @@ -27,12 +27,6 @@ jobs: tests/quake/** tests/scimark2/** tests/*.c - - name: Test file change of Linux image - id: test-linux-image-version-change - uses: tj-actions/changed-files@v45 - with: - files: | - mk/external.mk - name: Set alias id: has_changed_files run: | @@ -41,60 +35,8 @@ jobs: else echo "has_changed_files=false" >> $GITHUB_OUTPUT fi - if [[ ${{ steps.test-linux-image-version-change.outputs.any_modified }} == true ]]; then - # Determine if the changes are from Buildroot or the Linux version (The Linux might have several patches, so also need to check the SHA value) - echo -n $(git --no-pager diff HEAD^ HEAD | grep -e "+BUILDROOT_VERSION" -e "+LINUX_VERSION" -e "+LINUX_DATA_SHA1") >> linux-image-version-change - if [[ -s linux-image-version-change ]]; then - echo "has_changed_linux_image_version=true" >> $GITHUB_OUTPUT - else - echo "has_changed_linux_image_version=false" >> $GITHUB_OUTPUT - fi - else - echo "has_changed_linux_image_version=false" >> $GITHUB_OUTPUT - fi outputs: has_changed_files: ${{ steps.has_changed_files.outputs.has_changed_files }} - has_changed_linux_image_version: ${{ steps.has_changed_files.outputs.has_changed_linux_image_version }} - - build-linux-image-artifact: - needs: [detect-file-change] - if: ${{ needs.detect-file-change.outputs.has_changed_linux_image_version == 'true' || github.event_name == 'workflow_dispatch' }} - runs-on: ubuntu-22.04 - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - submodules: 'true' - - name: Install dependencies - run: | - sudo apt-get update -q=2 - sudo apt-get upgrade -q=2 - sudo apt-get install -q=2 build-essential git - - name: Build Linux image - run: | - make build-linux-image - make artifact ENABLE_PREBUILT=0 ENABLE_SYSTEM=1 - mkdir -p /tmp/rv32emu-linux-image-prebuilt/linux-image - mv build/linux-image/Image /tmp/rv32emu-linux-image-prebuilt/linux-image - mv build/linux-image/rootfs.cpio /tmp/rv32emu-linux-image-prebuilt/linux-image - mv build/sha1sum-linux-image /tmp - - name: Create tarball - run: | - cd /tmp - tar -zcvf rv32emu-linux-image-prebuilt.tar.gz rv32emu-linux-image-prebuilt - - name: Create GitHub Release - env: - GH_TOKEN: ${{ secrets.RV32EMU_PREBUILT_TOKEN }} - run: | - RELEASE_TAG=$(date +'%Y.%m.%d'-Linux-Image) - cd /tmp - gh release create $RELEASE_TAG \ - --repo sysprog21/rv32emu-prebuilt \ - --title "$RELEASE_TAG""-nightly" - gh release upload $RELEASE_TAG \ - rv32emu-linux-image-prebuilt.tar.gz \ - sha1sum-linux-image \ - --repo sysprog21/rv32emu-prebuilt build-artifact: needs: [detect-file-change] diff --git a/.github/workflows/build-linux-artifacts.yml b/.github/workflows/build-linux-artifacts.yml new file mode 100644 index 00000000..1ea70d2f --- /dev/null +++ b/.github/workflows/build-linux-artifacts.yml @@ -0,0 +1,78 @@ +name: Build Linux artifact + +on: + push: + branches: + - master + workflow_dispatch: + +jobs: + detect-file-change: + runs-on: ubuntu-24.04 + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + submodules: 'true' + - name: Test file change of Linux image + id: test-linux-image-version-change + uses: tj-actions/changed-files@v45 + with: + files: | + mk/external.mk + - name: Set alias + id: has_changed_files + run: | + if [[ ${{ steps.test-linux-image-version-change.outputs.any_modified }} == true ]]; then + # Determine if the changes are from Buildroot or the Linux version (The Linux might have several patches, so also need to check the SHA value) + echo -n $(git --no-pager diff HEAD^ HEAD | grep -e "+BUILDROOT_VERSION" -e "+LINUX_VERSION" -e "+LINUX_PATCHLEVEL") >> linux-image-version-change + if [[ -s linux-image-version-change ]]; then + echo "has_changed_linux_image_version=true" >> $GITHUB_OUTPUT + else + echo "has_changed_linux_image_version=false" >> $GITHUB_OUTPUT + fi + else + echo "has_changed_linux_image_version=false" >> $GITHUB_OUTPUT + fi + outputs: + has_changed_linux_image_version: ${{ steps.has_changed_files.outputs.has_changed_linux_image_version }} + + build-linux-image-artifact: + needs: [detect-file-change] + if: ${{ needs.detect-file-change.outputs.has_changed_linux_image_version == 'true' || github.event_name == 'workflow_dispatch' }} + runs-on: ubuntu-22.04 + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + submodules: 'true' + - name: Install dependencies + run: | + sudo apt-get update -q=2 + sudo apt-get upgrade -q=2 + sudo apt-get install -q=2 build-essential git + - name: Build Linux image + run: | + make build-linux-image + make artifact ENABLE_PREBUILT=0 ENABLE_SYSTEM=1 + mkdir -p /tmp/rv32emu-linux-image-prebuilt/linux-image + mv build/linux-image/Image /tmp/rv32emu-linux-image-prebuilt/linux-image + mv build/linux-image/rootfs.cpio /tmp/rv32emu-linux-image-prebuilt/linux-image + mv build/sha1sum-linux-image /tmp + - name: Create tarball + run: | + cd /tmp + tar -zcvf rv32emu-linux-image-prebuilt.tar.gz rv32emu-linux-image-prebuilt + - name: Create GitHub Release + env: + GH_TOKEN: ${{ secrets.RV32EMU_PREBUILT_TOKEN }} + run: | + RELEASE_TAG=$(date +'%Y.%m.%d'-Linux-Image) + cd /tmp + gh release create $RELEASE_TAG \ + --repo sysprog21/rv32emu-prebuilt \ + --title "$RELEASE_TAG""-nightly" + gh release upload $RELEASE_TAG \ + rv32emu-linux-image-prebuilt.tar.gz \ + sha1sum-linux-image \ + --repo sysprog21/rv32emu-prebuilt diff --git a/mk/artifact.mk b/mk/artifact.mk index 2963ff62..fed9cdb0 100644 --- a/mk/artifact.mk +++ b/mk/artifact.mk @@ -84,7 +84,7 @@ ifeq ($(call has, SYSTEM), 1) $(Q)$(eval PREBUILT_LINUX_IMAGE_FILENAME := $(shell cat $(BIN_DIR)/sha1sum-linux-image | awk '{ print $$2 };')) $(Q)$(eval $(foreach FILE,$(PREBUILT_LINUX_IMAGE_FILENAME), \ - $(call verify,$(shell grep -w $(FILE) $(BIN_DIR)/sha1sum-linux-image | awk '{ print $$1 };'),$(BIN_DIR)/linux-image/$(FILE),RES) \ + $(call verify,$(SHA1SUM),$(shell grep -w $(FILE) $(BIN_DIR)/sha1sum-linux-image | awk '{ print $$1 };'),$(BIN_DIR)/linux-image/$(FILE),RES) \ )) $(Q)$(eval RV32EMU_PREBUILT_TARBALL := rv32emu-linux-image-prebuilt.tar.gz) @@ -92,7 +92,7 @@ else ifeq ($(call has, ARCH_TEST), 1) $(Q)$(eval PREBUILT_SAIL_FILENAME := $(shell cat $(BIN_DIR)/rv32emu-prebuilt-sail-$(HOST_PLATFORM).sha | awk '{ print $$2 };')) $(Q)$(eval $(foreach FILE,$(PREBUILT_SAIL_FILENAME), \ - $(call verify,$(shell grep -w $(FILE) $(BIN_DIR)/rv32emu-prebuilt-sail-$(HOST_PLATFORM).sha | awk '{ print $$1 };'),$(BIN_DIR)/$(FILE),RES) \ + $(call verify,$(SHA1SUM),$(shell grep -w $(FILE) $(BIN_DIR)/rv32emu-prebuilt-sail-$(HOST_PLATFORM).sha | awk '{ print $$1 };'),$(BIN_DIR)/$(FILE),RES) \ )) $(Q)$(eval RV32EMU_PREBUILT_TARBALL := rv32emu-prebuilt-sail-$(HOST_PLATFORM)) @@ -101,10 +101,10 @@ else $(Q)$(eval PREBUILT_RV32_FILENAME := $(shell cat $(BIN_DIR)/sha1sum-riscv32 | awk '{ print $$2 };')) $(Q)$(eval $(foreach FILE,$(PREBUILT_X86_FILENAME), \ - $(call verify,$(shell grep -w $(FILE) $(BIN_DIR)/sha1sum-linux-x86-softfp | awk '{ print $$1 };'),$(BIN_DIR)/linux-x86-softfp/$(FILE),RES) \ + $(call verify,$(SHA1SUM),$(shell grep -w $(FILE) $(BIN_DIR)/sha1sum-linux-x86-softfp | awk '{ print $$1 };'),$(BIN_DIR)/linux-x86-softfp/$(FILE),RES) \ )) $(Q)$(eval $(foreach FILE,$(PREBUILT_RV32_FILENAME), \ - $(call verify,$(shell grep -w $(FILE) $(BIN_DIR)/sha1sum-riscv32 | awk '{ print $$1 };'),$(BIN_DIR)/riscv32/$(FILE),RES) \ + $(call verify,$(SHA1SUM),$(shell grep -w $(FILE) $(BIN_DIR)/sha1sum-riscv32 | awk '{ print $$1 };'),$(BIN_DIR)/riscv32/$(FILE),RES) \ )) $(Q)$(eval RV32EMU_PREBUILT_TARBALL := rv32emu-prebuilt.tar.gz) @@ -199,7 +199,7 @@ ifeq ($(call has, PREBUILT), 0) ifeq ($(call has, SYSTEM), 0) $(Q)$(call prologue,"scimark2") $(Q)$(call download,$(SCIMARK2_URL)) - $(Q)$(call verify,$(SCIMARK2_SHA1),$(notdir $(SCIMARK2_URL))) + $(Q)$(call verify,$(SHA1SUM),$(SCIMARK2_SHA1),$(notdir $(SCIMARK2_URL))) $(Q)$(call extract,"./tests/scimark2",$(notdir $(SCIMARK2_URL))) $(Q)$(call epilogue,$(notdir $(SCIMARK2_URL)),$(SHA1_FILE1),$(SHA1_FILE2)) $(Q)$(PRINTF) "Building scimark2 ...\n" diff --git a/mk/common.mk b/mk/common.mk index 6418974f..6935f1c9 100644 --- a/mk/common.mk +++ b/mk/common.mk @@ -64,3 +64,14 @@ ifndef SHA1SUM SHA1SUM := echo endif endif + +SHA256SUM = sha256sum +SHA256SUM := $(shell which $(SHA256SUM)) +ifndef SHA256SUM + SHA256SUM = shasum -a 256 + SHA256SUM := $(shell which shasum) + ifndef SHA256SUM + $(warning No sha256sum found. Disable checksums) + SHA256SUM := echo + endif +endif diff --git a/mk/external.mk b/mk/external.mk index c9f39b91..16f9fa44 100644 --- a/mk/external.mk +++ b/mk/external.mk @@ -4,10 +4,10 @@ COMPRESSED_IS_DIR := EXTRACTOR := VERIFIER := -# temporarily files to store correct SHA1 value and computed SHA1 value +# temporarily files to store correct SHA value and computed SHA value # respectively for verification of directory source -$(eval SHA1_FILE1 := $(shell mktemp)) -$(eval SHA1_FILE2 := $(shell mktemp)) +$(eval SHA_FILE1 := $(shell mktemp)) +$(eval SHA_FILE2 := $(shell mktemp)) # $(1): compressed source define prologue @@ -32,6 +32,7 @@ endef # $(1): destination directory # $(2): compressed source(.zip or.gz) +# $(3): strip component level (only for tar) # For Buildroot and Linux kernel, no need to extract define extract $(eval COMPRESSED_SUFFIX := $(suffix $(2))) @@ -42,85 +43,107 @@ define extract ($(eval _ := \ $(if $(COMPRESSED_IS_ZIP), \ ($(eval EXTRACTOR := unzip -d $(1) $(2))), \ - ($(eval EXTRACTOR := tar -xf $(2) -C $(1))) \ + ($(eval EXTRACTOR := tar -xf $(2) --strip-components=$(3) -C $(1))) \ )) \ $(eval _ := $(shell $(EXTRACTOR)))) ) \ ) endef -# $(1): correct SHA1 value -# $(2): filename or directory path -# $(3): (optional) returned result +# $(1): SHA algorithm command +# $(2): correct SHA value +# $(3): filename or directory path +# $(4): (optional) returned result # # Note: -# 1. for regular file, $(SHA1SUM) command's -c option generates keyword "FAILED" for indicating an unmatch +# 1. for regular file, $(1) command's -c option generates keyword "FAILED" for indicating an unmatch # 2. for directory, cmp command outputs keyword "differ" for indicating an unmatch define verify - $(eval COMPRESSED_IS_DIR := $(if $(wildcard $(2)/*),1,0)) + $(eval COMPRESSED_IS_DIR := $(if $(wildcard $(3)/*),1,0)) $(eval _ := \ $(if $(filter 1,$(COMPRESSED_IS_DIR)), \ ($(eval VERIFIER := \ - echo $(1) > $(SHA1_FILE1) \ - | find $(2) -type f -not -path '*/.git/*' -print0 \ + echo $(2) > $(SHA_FILE1) \ + | find $(3) -type f -not -path '*/.git/*' -print0 \ | sort -z \ - | xargs -0 $(SHA1SUM) \ + | xargs -0 $(1) \ | sort \ - | $(SHA1SUM) \ - | cut -f 1 -d ' ' > $(SHA1_FILE2) && cmp $(SHA1_FILE1) $(SHA1_FILE2))), \ - ($(eval VERIFIER := (ls $(2) >/dev/null 2>&1 || echo FAILED) && echo "$(strip $(1)) $(strip $(2))" | $(SHA1SUM) -c -)) \ + | $(1) \ + | cut -f 1 -d ' ' > $(SHA_FILE2) && cmp $(SHA_FILE1) $(SHA_FILE2))), \ + ($(eval VERIFIER := (ls $(3) >/dev/null 2>&1 || echo FAILED) && echo "$(strip $(2)) $(strip $(3))" | $(1) -c -)) \ )) $(eval _ := $(shell $(VERIFIER) 2>&1)) $(eval _ := \ $(if $(filter FAILED differ:,$(_)), \ - ($(if $(3), \ - $(eval $(3) := 1), \ + ($(if $(4), \ + $(eval $(4) := 1), \ $(error $(_)) \ )), \ - (# SHA1 value match, do nothing) \ + (# SHA value match, do nothing) \ )) endef # For each external target, the following must be defined in advance: # _DATA_URL : the hyperlink which points to archive. +# _DATA_DEST : the extract destination of the file (only for compressed file) # _DATA : the file to be read by specific executable. -# _DATA_SHA1 : the checksum of the content in _DATA +# _DATA_SKIP_DIR_LEVEL : the strip component level when using tar +# _DATA_SHA : the SHA value of the content in _DATA +# _DATA_SHA_CMD : the SHA command # Doom # https://tipsmake.com/how-to-run-doom-on-raspberry-pi-without-emulator DOOM_DATA_URL = http://www.doomworld.com/3ddownloads/ports/shareware_doom_iwad.zip -DOOM_DATA = $(OUT)/DOOM1.WAD -DOOM_DATA_SHA1 = 5b2e249b9c5133ec987b3ea77596381dc0d6bc1d +DOOM_DATA_DEST = $(OUT) +DOOM_DATA = $(DOOM_DATA_DEST)/DOOM1.WAD +DOOM_DATA_SHA = 5b2e249b9c5133ec987b3ea77596381dc0d6bc1d +DOOM_DATA_SHA_CMD = $(SHA1SUM) # Quake QUAKE_DATA_URL = https://www.libsdl.org/projects/quake/data/quakesw-1.0.6.zip -QUAKE_DATA = $(OUT)/id1/pak0.pak -QUAKE_DATA_SHA1 = 36b42dc7b6313fd9cabc0be8b9e9864840929735 +QUAKE_DATA_DEST = $(OUT) +QUAKE_DATA = $(QUAKE_DATA_DEST)/id1/pak0.pak +QUAKE_DATA_SHA = 36b42dc7b6313fd9cabc0be8b9e9864840929735 +QUAKE_DATA_SHA_CMD = $(SHA1SUM) # Timidity software synthesizer configuration for SDL2_mixer TIMIDITY_DATA_URL = http://www.libsdl.org/projects/mixer/timidity/timidity.tar.gz -TIMIDITY_DATA = $(OUT)/timidity -TIMIDITY_DATA_SHA1 = cf6217a5d824b717ec4a07e15e6c129a4657ca25 +TIMIDITY_DATA_DEST = $(OUT) +TIMIDITY_DATA = $(TIMIDITY_DATA_DEST)/timidity +TIMIDITY_DATA_SKIP_DIR_LEVEL = 0 +TIMIDITY_DATA_SHA = cf6217a5d824b717ec4a07e15e6c129a4657ca25 +TIMIDITY_DATA_SHA_CMD = $(SHA1SUM) # Buildroot BUILDROOT_VERSION = 2024.11 BUILDROOT_DATA = /tmp/buildroot BUILDROOT_DATA_URL = git clone https://github.com/buildroot/buildroot $(BUILDROOT_DATA) -b $(BUILDROOT_VERSION) --depth=1 -BUILDROOT_DATA_SHA1 = e678801287ab68369af1731dcf1acc39e4adccff +BUILDROOT_DATA_SHA = e678801287ab68369af1731dcf1acc39e4adccff +BUILDROOT_DATA_SHA_CMD = $(SHA1SUM) # Linux kernel -LINUX_VERSION = linux-6.1.y -LINUX_DATA = /tmp/linux -LINUX_DATA_URL = git clone https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git $(LINUX_DATA) -b $(LINUX_VERSION) --depth=1 -LINUX_DATA_SHA1 = 43b6b7fbf9231656d7b65f118445996172250fc0 +LINUX_VERSION = 6 +LINUX_PATCHLEVEL = 1 +LINUX_CDN_BASE_URL = https://cdn.kernel.org/pub/linux/kernel +LINUX_CDN_VERSION_URL = $(LINUX_CDN_BASE_URL)/v$(LINUX_VERSION).x +$(shell mkdir -p /tmp/linux) +LINUX_DATA_DEST = /tmp/linux +LINUX_DATA = $(shell wget -q -O- $(LINUX_CDN_VERSION_URL) | \ + grep -o 'linux-$(LINUX_VERSION).$(LINUX_PATCHLEVEL).[0-9]\+\.tar.gz' | \ + sort -V | tail -n 1) +LINUX_DATA_URL = $(LINUX_CDN_VERSION_URL)/$(LINUX_DATA) +LINUX_DATA_SKIP_DIR_LEVEL = 1 +LINUX_DATA_SHA = $(shell wget -q -O- $(LINUX_CDN_VERSION_URL)/sha256sums.asc | \ + grep $(LINUX_DATA) | awk '{print $$1}') +LINUX_DATA_SHA_CMD = $(SHA256SUM) define download-extract-verify $($(T)_DATA): $(Q)$$(call prologue,$$@) $(Q)$$(call download,$(strip $($(T)_DATA_URL))) - $(Q)$$(call extract,$(OUT),$(notdir $($(T)_DATA_URL))) - $(Q)$$(call verify,$($(T)_DATA_SHA1),$($(T)_DATA)) - $(Q)$$(call epilogue,$(notdir $($(T)_DATA_URL)),$(SHA1_FILE1),$(SHA1_FILE2)) + $(Q)$$(call extract,$($(T)_DATA_DEST),$(notdir $($(T)_DATA_URL)),$($(T)_DATA_SKIP_DIR_LEVEL)) + $(Q)$$(call verify,$($(T)_DATA_SHA_CMD),$($(T)_DATA_SHA),$($(T)_DATA)) + $(Q)$$(call epilogue,$(notdir $($(T)_DATA_URL)),$(SHA_FILE1),$(SHA_FILE2)) endef EXTERNAL_DATA = DOOM QUAKE TIMIDITY BUILDROOT LINUX