Skip to content

Avoid re-fetching binaries when they exist #492

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Sep 11, 2024
+45 −9
Merged

Avoid re-fetching binaries when they exist #492

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions .github/workflows/build-artifact.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,8 @@ jobs:
run: |
make artifact ENABLE_PREBUILT=0
mkdir -p /tmp/rv32emu-prebuilt
mv build/sha1sum-linux-x86-softfp /tmp
mv build/sha1sum-riscv32 /tmp
mv build/linux-x86-softfp build/riscv32 /tmp/rv32emu-prebuilt
- name: Build Sail model
run: |
Expand Down Expand Up @@ -88,4 +90,6 @@ jobs:
--title "$RELEASE_TAG""-nightly"
gh release upload $RELEASE_TAG \
rv32emu-prebuilt.tar.gz \
sha1sum-linux-x86-softfp \
sha1sum-riscv32 \
--repo sysprog21/rv32emu-prebuilt
1 change: 1 addition & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ build/path/
build/linux-x86-softfp/
build/riscv32/
build/sail_cSim/
build/sha1sum-*
*.a
*.o
*.o.d
Expand Down
35 changes: 31 additions & 4 deletions mk/artifact.mk
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,12 +48,29 @@ else
LDFLAGS_CROSS := -lm -lsemihost
endif

.PHONY: artifact scimark2 ieeelib
.PHONY: artifact fetch-checksum scimark2 ieeelib

artifact: ieeelib scimark2
artifact: fetch-checksum ieeelib scimark2
ifeq ($(call has, PREBUILT), 1)
$(Q)$(PRINTF) "Fetching prebuilt executables from \"rv32emu-prebuilt\" ...\n"
$(Q)wget -q --show-progress https://github.com/sysprog21/rv32emu-prebuilt/releases/download/$(LATEST_RELEASE)/rv32emu-prebuilt.tar.gz -O- | tar -C build --strip-components=1 -xz
$(Q)$(PRINTF) "Checking SHA1 of binaries ...\n"

$(Q)$(eval PREBUILT_X86_FILENAME := $(shell cat $(BIN_DIR)/sha1sum-linux-x86-softfp | awk '{ print $$2 };'))
$(Q)$(eval PREBUILT_RV32_FILENAME := $(shell cat $(BIN_DIR)/sha1sum-riscv32 | awk '{ print $$2 };'))

$(Q)$(eval RES := 0)
$(Q)$(eval $(foreach FILE,$(PREBUILT_X86_FILENAME), \
$(call verify,$(shell grep -w $(FILE) $(BIN_DIR)/sha1sum-linux-x86-softfp | awk '{ print $$1 };'),$(BIN_DIR)/linux-x86-softfp/$(FILE),RES) \
))
$(Q)$(eval $(foreach FILE,$(PREBUILT_RV32_FILENAME), \
$(call verify,$(shell grep -w $(FILE) $(BIN_DIR)/sha1sum-riscv32 | awk '{ print $$1 };'),$(BIN_DIR)/riscv32/$(FILE),RES) \
))

$(Q)if [ "$(RES)" = "1" ]; then \
$(PRINTF) "$(YELLOW)SHA1 verifications fail! Re-fetching prebuilt binaries from \"rv32emu-prebuilt\" ...\n$(NO_COLOR)"; \
wget -q --show-progress https://github.com/sysprog21/rv32emu-prebuilt/releases/download/$(LATEST_RELEASE)/rv32emu-prebuilt.tar.gz -O- | tar -C build --strip-components=1 -xz; \
else \
$(PRINTF) "$(PASS_COLOR)SHA1 verifications succeed!\n$(NO_COLOR)"; \
fi
else
git submodule update --init $(addprefix ./tests/,$(foreach tb,$(TEST_SUITES),$(tb)))
$(Q)for tb in $(TEST_SUITES); do \
Expand Down Expand Up @@ -82,6 +99,16 @@ else
-DCMAKE_BUILD_TYPE=RELEASE -DBOARD_NAME=rv32emu .. && \
make
$(Q)cp ./tests/quake/build/port/boards/rv32emu/quake $(BIN_DIR)/riscv32/quake

$(Q)(cd $(BIN_DIR)/linux-x86-softfp; for fd in *; do $(SHA1SUM) "$$fd"; done) >> $(BIN_DIR)/sha1sum-linux-x86-softfp
$(Q)(cd $(BIN_DIR)/riscv32; for fd in *; do $(SHA1SUM) "$$fd"; done) >> $(BIN_DIR)/sha1sum-riscv32
endif

fetch-checksum:
ifeq ($(call has, PREBUILT), 1)
$(Q)$(PRINTF) "Fetching SHA1 of binaries ...\n"
$(Q)wget -q --show-progress -O $(BIN_DIR)/sha1sum-linux-x86-softfp https://github.com/sysprog21/rv32emu-prebuilt/releases/download/$(LATEST_RELEASE)/sha1sum-linux-x86-softfp
$(Q)wget -q --show-progress -O $(BIN_DIR)/sha1sum-riscv32 https://github.com/sysprog21/rv32emu-prebuilt/releases/download/$(LATEST_RELEASE)/sha1sum-riscv32
endif

scimark2:
Expand Down
14 changes: 9 additions & 5 deletions mk/external.mk
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,15 +40,16 @@ endef

# $(1): correct SHA1 value
# $(2): filename or directory path
# $(3): (optional) returned result
#
# Note:
# 1. for regular file, $(SHA1SUM) command's -c option generates keyword "FAILED" for indicating an unmatch
# 2. for directory, cmp command outputs keyword "differ" for indicating an unmatch
define verify
$(eval COMPRESSED_IS_DIR := $(if $(wildcard $(2)/*),1,0))
$(eval _ := \
$(if $(filter 1,$(COMPRESSED_IS_DIR)), \
($(eval VERIFIER := \
$(if $(filter 1,$(COMPRESSED_IS_DIR)), \
($(eval VERIFIER := \
echo $(1) > $(SHA1_FILE1) \
| find $(2) -type f -print0 \
| sort -z \
Expand All @@ -58,10 +59,13 @@ define verify
| cut -f 1 -d ' ' > $(SHA1_FILE2) && cmp $(SHA1_FILE1) $(SHA1_FILE2))), \
($(eval VERIFIER := echo "$(strip $(1)) $(strip $(2))" | $(SHA1SUM) -c)) \
))
$(eval _ := $(shell $(VERIFIER)))
$(eval _ := $(shell $(VERIFIER) 2>&1))
$(eval _ := \
$(if $(filter FAILED differ:,$(_)), \
($(error $(_))), \
($(if $(3), \
$(eval $(3) := 1), \
$(error $(_)) \
)), \
(# SHA1 value match, do nothing) \
))
endef
Expand Down Expand Up @@ -92,7 +96,7 @@ $($(T)_DATA):
$(Q)$$(call prologue,$$@)
$(Q)$$(call download,$(strip $($(T)_DATA_URL)))
$(Q)$$(call extract,$(OUT),$(notdir $($(T)_DATA_URL)))
$(Q)$$(call verify,$($(T)_DATA_SHA1), $($(T)_DATA))
$(Q)$$(call verify,$($(T)_DATA_SHA1),$($(T)_DATA))
$(Q)$$(call epilogue,$(notdir $($(T)_DATA_URL)),$(SHA1_FILE1),$(SHA1_FILE2))
endef

Expand Down