Take a step toward complete BB domination (#33125)

This provides a number of benefits:

* Significantly reduces the amount of work in win-extras, which I've
always thought was an annoying wart in our build system.

* Provides a consistent 7z executable on all platforms, which can be
picked up by Pkg/BinaryProvider for consistent
unzipping/untarring/ungzipping on all platforms. Note that on Windows,
7z is a repackaged binary from sourceforge and is the "full" 7z, whereas
on all other platforms it is a p7zip build of the standalone 7za
executable with all relevant patches applied. There are some format
capability differences, but for the important things it should be fine.

Author: staticfloat

LICENSE.md

@@ -52,6 +52,8 @@ USE_SYSTEM_LIBSSH2:=0
 USE_SYSTEM_CURL:=0
 USE_SYSTEM_LIBGIT2:=0
 USE_SYSTEM_PATCHELF:=0
+USE_SYSTEM_ZLIB:=0
+USE_SYSTEM_P7ZIP:=0
 
 # Link to the LLVM shared library
 USE_LLVM_SHLIB := 1
@@ -1005,7 +1007,7 @@ USE_BINARYBUILDER ?= 0
 endif
 
 # This is the set of projects that BinaryBuilder dependencies are hooked up for.
-BB_PROJECTS := OPENBLAS LLVM SUITESPARSE OPENLIBM GMP MBEDTLS LIBSSH2 MPFR CURL LIBGIT2 PCRE LIBUV LIBUNWIND DSFMT OBJCONV
+BB_PROJECTS := OPENBLAS LLVM SUITESPARSE OPENLIBM GMP MBEDTLS LIBSSH2 MPFR CURL LIBGIT2 PCRE LIBUV LIBUNWIND DSFMT OBJCONV ZLIB P7ZIP
 define SET_BB_DEFAULT
 # First, check to see if BB is disabled on a global setting
 ifeq ($$(USE_BINARYBUILDER),0)

Make.inc

@@ -275,7 +275,7 @@ ifeq ($(BUNDLE_DEBUG_LIBS),1)
 else
 	@$(MAKE) $(QUIET_MAKE) release
 endif
-	@for subdir in $(bindir) $(datarootdir)/julia/stdlib/$(VERSDIR) $(docdir) $(man1dir) $(includedir)/julia $(libdir) $(private_libdir) $(sysconfdir); do \
+	@for subdir in $(bindir) $(datarootdir)/julia/stdlib/$(VERSDIR) $(docdir) $(man1dir) $(includedir)/julia $(libdir) $(private_libdir) $(sysconfdir) $(libexecdir); do \
 		mkdir -p $(DESTDIR)$$subdir; \
 	done
 
@@ -293,6 +293,9 @@ endif
 	-$(INSTALL_M) $(build_bindir)/libopenlibm.dll.a $(DESTDIR)$(libdir)/
 else
 
+	# Install `7z` into libexec/
+	$(INSTALL_M) $(build_bindir)/7z $(DESTDIR)$(libexecdir)/
+
 # Copy over .dSYM directories directly for Darwin
 ifneq ($(DARWIN_FRAMEWORK),1)
 ifeq ($(OS),Darwin)
@@ -454,8 +457,6 @@ ifeq ($(OS), Darwin)
 endif
 
 ifeq ($(OS), WINNT)
-	[ ! -d $(JULIAHOME)/dist-extras ] || ( cd $(JULIAHOME)/dist-extras && \
-		cp 7z.exe 7z.dll libexpat-1.dll zlib1.dll $(BUILDROOT)/julia-$(JULIA_COMMIT)/bin )
 	cd $(BUILDROOT)/julia-$(JULIA_COMMIT)/bin && rm -f llvm* llc.exe lli.exe opt.exe LTO.dll bugpoint.exe macho-dump.exe
 
 	# create file listing for uninstall. note: must have Windows path separators and line endings.
@@ -465,7 +466,7 @@ ifeq ($(OS), WINNT)
 	cd $(BUILDROOT) && $(call spawn,$(JULIAHOME)/dist-extras/nsis/makensis.exe) -NOCD -DVersion=$(JULIA_VERSION) -DArch=$(ARCH) -DCommit=$(JULIA_COMMIT) -DJULIAHOME="$(call cygpath_w,$(JULIAHOME))" $(call cygpath_w,$(JULIAHOME)/contrib/windows/build-installer.nsi) | iconv -f latin1
 
 	# compress nsis installer and combine with 7zip self-extracting header
-	cd $(BUILDROOT) && $(JULIAHOME)/dist-extras/7z a -mx=9 "julia-install-$(JULIA_COMMIT)-$(ARCH).7z" julia-installer.exe
+	cd $(BUILDROOT) && $(JULIAHOME)/usr/bin/7z a -mx=9 "julia-install-$(JULIA_COMMIT)-$(ARCH).7z" julia-installer.exe
 	cd $(BUILDROOT) && cat $(JULIAHOME)/contrib/windows/7zS.sfx $(JULIAHOME)/contrib/windows/7zSFX-config.txt "julia-install-$(JULIA_COMMIT)-$(ARCH).7z" > "$(JULIA_BINARYDIST_FILENAME).exe"
 	chmod a+x "$(BUILDROOT)/$(JULIA_BINARYDIST_FILENAME).exe"
 	-rm -f $(BUILDROOT)/julia-install-$(JULIA_COMMIT)-$(ARCH).7z
@@ -585,37 +586,12 @@ test-%: check-whitespace $(JULIA_BUILD_MODE)
 # download target for some hardcoded windows dependencies
 .PHONY: win-extras wine_path
 win-extras:
-	[ -d $(JULIAHOME)/dist-extras ] || mkdir $(JULIAHOME)/dist-extras
-ifneq ($(BUILD_OS),WINNT)
-ifeq (,$(findstring CYGWIN,$(BUILD_OS)))
-	cp /usr/lib/p7zip/7z /usr/lib/p7zip/7z.so $(JULIAHOME)/dist-extras
-endif
-endif
-ifneq (,$(filter $(ARCH), i386 i486 i586 i686))
-	cd $(JULIAHOME)/dist-extras && \
-	$(JLDOWNLOAD) https://sourceforge.net/projects/sevenzip/files/7-Zip/19.00/7z1900.exe && \
-	$(JLCHECKSUM) 7z1900.exe && \
-	7z x -y 7z1900.exe 7z.exe 7z.dll && \
-	../contrib/windows/winrpm.sh http://download.opensuse.org/repositories/windows:/mingw:/win32/openSUSE_Leap_42.2 \
-		"mingw32-libexpat1 mingw32-zlib1" && \
-	cp usr/i686-w64-mingw32/sys-root/mingw/bin/*.dll .
-else ifeq ($(ARCH),x86_64)
-	cd $(JULIAHOME)/dist-extras && \
-	$(JLDOWNLOAD) https://downloads.sourceforge.net/project/sevenzip/7-Zip/19.00/7z1900-x64.exe && \
-	$(JLCHECKSUM) 7z1900-x64.exe && \
-	7z x -y 7z1900-x64.exe 7z.exe 7z.dll && \
-	../contrib/windows/winrpm.sh http://download.opensuse.org/repositories/windows:/mingw:/win64/openSUSE_Leap_42.2 \
-		"mingw64-libexpat1 mingw64-zlib1" && \
-	cp usr/x86_64-w64-mingw32/sys-root/mingw/bin/*.dll .
-else
-	$(error no win-extras target for ARCH=$(ARCH))
-endif
+	@$(MAKE) -C $(BUILDROOT)/deps install-p7zip
+	mkdir -p $(JULIAHOME)/dist-extras
 	cd $(JULIAHOME)/dist-extras && \
 	$(JLDOWNLOAD) https://sourceforge.net/projects/nsis/files/NSIS%203/3.04/nsis-3.04-setup.exe && \
 	$(JLCHECKSUM) nsis-3.04-setup.exe && \
-	chmod a+x 7z.exe && \
-	chmod a+x 7z.dll && \
-	$(call spawn,./7z.exe) x -y -onsis nsis-3.04-setup.exe && \
+	$(call spawn,$(JULIAHOME)/usr/bin/7z.exe) x -y -onsis nsis-3.04-setup.exe && \
 	chmod a+x ./nsis/makensis.exe
 
 # various statistics about the build that may interest the user

Makefile

@@ -12,7 +12,7 @@
 TRIPLETS="i686-linux-gnu x86_64-linux-gnu aarch64-linux-gnu arm-linux-gnueabihf powerpc64le-linux-gnu i686-linux-musl x86_64-linux-musl aarch64-linux-musl arm-linux-musleabihf x86_64-apple-darwin14 x86_64-unknown-freebsd11.1 i686-w64-mingw32 x86_64-w64-mingw32"
 
 # These are the projects currently using BinaryBuilder; both GCC-expanded and non-GCC-expanded:
-BB_PROJECTS="gmp mbedtls libssh2 mpfr curl libgit2 pcre libuv unwind osxunwind dsfmt objconv"
+BB_PROJECTS="gmp mbedtls libssh2 mpfr curl libgit2 pcre libuv unwind osxunwind dsfmt objconv p7zip zlib"
 BB_GCC_EXPANDED_PROJECTS="llvm openblas suitesparse openlibm"
 
 # If we've been given a project name, filter down to that one:

contrib/refresh_bb_tarballs.sh

@@ -139,6 +139,15 @@ ifeq ($(USE_SYSTEM_UTF8PROC), 0)
 DEP_LIBS += utf8proc
 endif
 
+ifeq ($(USE_SYSTEM_LIBZ), 0)
+DEP_LIBS += libz
+endif
+
+ifeq ($(USE_SYSTEM_P7ZIP), 0)
+DEP_LIBS += p7zip
+endif
+
+
 # Only compile standalone LAPACK if we are not using OpenBLAS.
 # OpenBLAS otherwise compiles LAPACK as part of its build.
 # This is useful where one wants to use the vendor BLAS, but
@@ -191,3 +200,5 @@ include $(SRCDIR)/libssh2.mk
 include $(SRCDIR)/curl.mk
 include $(SRCDIR)/libgit2.mk
 include $(SRCDIR)/libwhich.mk
+include $(SRCDIR)/zlib.mk
+include $(SRCDIR)/p7zip.mk

deps/Makefile

@@ -32,6 +32,10 @@ LIBUV_VER = 1.29.1
 LIBUV_BB_REL = 0
 OBJCONV_VER = 2.49.0
 OBJCONV_BB_REL = 0
+ZLIB_VER = 1.2.11
+ZLIB_BB_REL = 3
+P7ZIP_VER = 16.2.0
+P7ZIP_BB_REL = 1
 
 # Specify the version of the Mozilla CA Certificate Store to obtain.
 # The versions of cacert.pem are identified by the date (YYYY-MM-DD) of their changes.

deps/Versions.make

@@ -0,0 +1 @@
+489c5ebbca174fdbfae86846e270e112

deps/checksums/Zlib.v1.2.11-3.aarch64-linux-gnu.tar.gz/md5

@@ -0,0 +1 @@
+da1bac181754711c50909492127f855c9acc2928951ce95310a6d32c03bb731d82fe2f0f97ae482c0da319b09d74b91aa143bd90b992489d386beee492d30e6f

deps/checksums/Zlib.v1.2.11-3.aarch64-linux-gnu.tar.gz/sha512

@@ -0,0 +1 @@
+e132449b6a56b91aa6db9e9b1c386517

deps/checksums/Zlib.v1.2.11-3.aarch64-linux-musl.tar.gz/md5

@@ -0,0 +1 @@
+7faf95166293b65f0c348417520b590b89cffcce6f2be19684398c9d313393a8405aeabbde0855e770a06a858c8684f298ec0123b60c75ff187dbf735be871b1