Avoid re-fetching binaries when they exist

We fetch the SHA1 first and verify the binaries. If any verification
fails, the returned value of the "verify" macro becomes non-zero.
Only if the verification fails, the download begins.

Close #491

Author: vacantron

.github/workflows/build-artifact.yml

@@ -59,6 +59,8 @@ jobs:
         run: |
           make artifact ENABLE_PREBUILT=0
           mkdir -p /tmp/rv32emu-prebuilt
+          mv build/sha1sum-linux-x86-softfp /tmp
+          mv build/sha1sum-riscv32 /tmp
           mv build/linux-x86-softfp build/riscv32 /tmp/rv32emu-prebuilt
       - name: Build Sail model
         run: |
@@ -88,4 +90,6 @@ jobs:
             --title "$RELEASE_TAG""-nightly"
           gh release upload $RELEASE_TAG \
             rv32emu-prebuilt.tar.gz \
+            sha1sum-linux-x86-softfp \
+            sha1sum-riscv32 \
             --repo sysprog21/rv32emu-prebuilt

.gitignore

@@ -19,6 +19,7 @@ build/path/
 build/linux-x86-softfp/
 build/riscv32/
 build/sail_cSim/
+build/sha1sum-*
 *.a
 *.o
 *.o.d

mk/artifact.mk

@@ -48,12 +48,29 @@ else
   LDFLAGS_CROSS := -lm -lsemihost
 endif
 
-.PHONY: artifact scimark2 ieeelib
+.PHONY: artifact fetch-checksum scimark2 ieeelib
 
-artifact: ieeelib scimark2
+artifact: fetch-checksum ieeelib scimark2
 ifeq ($(call has, PREBUILT), 1)
-	$(Q)$(PRINTF) "Fetching prebuilt executables from \"rv32emu-prebuilt\" ...\n"
-	$(Q)wget -q --show-progress https://github.com/sysprog21/rv32emu-prebuilt/releases/download/$(LATEST_RELEASE)/rv32emu-prebuilt.tar.gz -O- | tar -C build --strip-components=1 -xz
+	$(Q)$(PRINTF) "Checking SHA1 of binaries ...\n"
+
+	$(Q)$(eval PREBUILT_X86_FILENAME := $(shell cat $(BIN_DIR)/sha1sum-linux-x86-softfp | awk '{  print $$2 };'))
+	$(Q)$(eval PREBUILT_RV32_FILENAME := $(shell cat $(BIN_DIR)/sha1sum-riscv32 | awk '{ print $$2 };'))
+
+	$(Q)$(eval RES := 0)
+	$(Q)$(eval $(foreach FILE,$(PREBUILT_X86_FILENAME), \
+	    $(call verify,$(shell grep -w $(FILE) $(BIN_DIR)/sha1sum-linux-x86-softfp | awk '{ print $$1 };'),$(BIN_DIR)/linux-x86-softfp/$(FILE),RES) \
+	))
+	$(Q)$(eval $(foreach FILE,$(PREBUILT_RV32_FILENAME), \
+	    $(call verify,$(shell grep -w $(FILE) $(BIN_DIR)/sha1sum-riscv32 | awk '{ print $$1 };'),$(BIN_DIR)/riscv32/$(FILE),RES) \
+	))
+
+	$(Q)if [ "$(RES)" = "1" ]; then \
+	    $(PRINTF) "$(YELLOW)SHA1 verifications fail! Re-fetching prebuilt binaries from \"rv32emu-prebuilt\" ...\n$(NO_COLOR)"; \
+	    wget -q --show-progress https://github.com/sysprog21/rv32emu-prebuilt/releases/download/$(LATEST_RELEASE)/rv32emu-prebuilt.tar.gz -O- | tar -C build --strip-components=1 -xz; \
+	else \
+		$(PRINTF) "$(PASS_COLOR)SHA1 verifications succeed!\n$(NO_COLOR)"; \
+	fi
 else
 	git submodule update --init $(addprefix ./tests/,$(foreach tb,$(TEST_SUITES),$(tb)))
 	$(Q)for tb in $(TEST_SUITES); do \
@@ -82,6 +99,16 @@ else
 	          -DCMAKE_BUILD_TYPE=RELEASE -DBOARD_NAME=rv32emu .. && \
 	    make
 	$(Q)cp ./tests/quake/build/port/boards/rv32emu/quake $(BIN_DIR)/riscv32/quake
+
+	$(Q)(cd $(BIN_DIR)/linux-x86-softfp; for fd in *; do $(SHA1SUM) "$$fd"; done) >> $(BIN_DIR)/sha1sum-linux-x86-softfp
+	$(Q)(cd $(BIN_DIR)/riscv32; for fd in *; do $(SHA1SUM) "$$fd"; done) >> $(BIN_DIR)/sha1sum-riscv32
+endif
+
+fetch-checksum:
+ifeq ($(call has, PREBUILT), 1)
+	$(Q)$(PRINTF) "Fetching SHA1 of binaries ...\n"
+	$(Q)wget -q --show-progress -O $(BIN_DIR)/sha1sum-linux-x86-softfp https://github.com/sysprog21/rv32emu-prebuilt/releases/download/$(LATEST_RELEASE)/sha1sum-linux-x86-softfp
+	$(Q)wget -q --show-progress -O $(BIN_DIR)/sha1sum-riscv32 https://github.com/sysprog21/rv32emu-prebuilt/releases/download/$(LATEST_RELEASE)/sha1sum-riscv32
 endif
 
 scimark2:

mk/external.mk

@@ -40,15 +40,16 @@ endef
 
 # $(1): correct SHA1 value
 # $(2): filename or directory path
+# $(3): (optional) returned result
 #
 # Note:
 # 1. for regular file, $(SHA1SUM) command's -c option generates keyword "FAILED" for indicating an unmatch
 # 2. for directory, cmp command outputs keyword "differ" for indicating an unmatch
 define verify
     $(eval COMPRESSED_IS_DIR := $(if $(wildcard $(2)/*),1,0))
     $(eval _ := \
-	    $(if $(filter 1,$(COMPRESSED_IS_DIR)), \
-		    ($(eval VERIFIER :=  \
+        $(if $(filter 1,$(COMPRESSED_IS_DIR)), \
+            ($(eval VERIFIER :=  \
                 echo $(1) > $(SHA1_FILE1) \
                 | find $(2) -type f -print0 \
                 | sort -z \
@@ -58,10 +59,13 @@ define verify
                 | cut -f 1 -d ' ' > $(SHA1_FILE2) && cmp $(SHA1_FILE1) $(SHA1_FILE2))), \
             ($(eval VERIFIER := echo "$(strip $(1))  $(strip $(2))" | $(SHA1SUM) -c)) \
     ))
-    $(eval _ := $(shell $(VERIFIER)))
+    $(eval _ := $(shell $(VERIFIER) 2>&1))
     $(eval _ := \
         $(if $(filter FAILED differ:,$(_)), \
-            ($(error $(_))), \
+            ($(if $(3), \
+                $(eval $(3) := 1), \
+                $(error $(_)) \
+            )), \
             (# SHA1 value match, do nothing) \
     ))
 endef
@@ -92,7 +96,7 @@ $($(T)_DATA):
 	$(Q)$$(call prologue,$$@)
 	$(Q)$$(call download,$(strip $($(T)_DATA_URL)))
 	$(Q)$$(call extract,$(OUT),$(notdir $($(T)_DATA_URL)))
-	$(Q)$$(call verify,$($(T)_DATA_SHA1), $($(T)_DATA))
+	$(Q)$$(call verify,$($(T)_DATA_SHA1),$($(T)_DATA))
 	$(Q)$$(call epilogue,$(notdir $($(T)_DATA_URL)),$(SHA1_FILE1),$(SHA1_FILE2))
 endef