Upstream depends osx sdk 12.1 (#893)

* Updated depends to use newer OSX SDK (Upstream changes)

* Updated libsodium in depends to build with OSX SDK 12.1

* Updated OSX sdk file in gitian descriptor

* Re-added darwin custom ranlib and ar configs

* Testing the different params for libsodium osx build

* Temporarily removed env calls in darwin builds

contrib/gitian-descriptors/gitian-osx.yml

@@ -34,7 +34,7 @@ remotes:
 - "url": "https://github.com/navcoin/navcoin-core.git"
   "dir": "navcoin-core"
 files:
-- "MacOSX10.14.sdk.tar.gz"
+- "Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz"
 script: |
   set -e -o pipefail
 
@@ -94,7 +94,7 @@ script: |
   BASEPREFIX=`pwd`/depends
 
   mkdir -p ${BASEPREFIX}/SDKs
-  tar -C ${BASEPREFIX}/SDKs -xf ${BUILD_DIR}/MacOSX10.14.sdk.tar.gz
+  tar -C ${BASEPREFIX}/SDKs -xf ${BUILD_DIR}/Xcode-12.1-12A7403-extracted-SDK-with-libcxx-headers.tar.gz
 
   # Build dependencies for each host
   for i in $HOSTS; do

depends/builders/darwin.mk

@@ -1,5 +1,5 @@
-build_darwin_CC:=$(shell xcrun -f clang) --sysroot $(shell xcrun --show-sdk-path)
-build_darwin_CXX:=$(shell xcrun -f clang++) --sysroot $(shell xcrun --show-sdk-path)
+build_darwin_CC:=$(shell xcrun -f clang) -isysroot$(shell xcrun --show-sdk-path)
+build_darwin_CXX:=$(shell xcrun -f clang++) -isysroot$(shell xcrun --show-sdk-path)
 build_darwin_AR:=$(shell xcrun -f ar)
 build_darwin_RANLIB:=$(shell xcrun -f ranlib)
 build_darwin_STRIP:=$(shell xcrun -f strip)
@@ -10,13 +10,14 @@ build_darwin_SHA256SUM=shasum -a 256
 build_darwin_DOWNLOAD=curl --location --fail --connect-timeout $(DOWNLOAD_CONNECT_TIMEOUT) --retry $(DOWNLOAD_RETRIES) -o
 
 #darwin host on darwin builder. overrides darwin host preferences.
-darwin_CC=$(shell xcrun -f clang) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(shell xcrun --show-sdk-path)
-darwin_CXX:=$(shell xcrun -f clang++) -mmacosx-version-min=$(OSX_MIN_VERSION) -stdlib=libc++ --sysroot $(shell xcrun --show-sdk-path)
+darwin_CC=$(shell xcrun -f clang) -mmacosx-version-min=$(OSX_MIN_VERSION) -isysroot$(shell xcrun --show-sdk-path)
+darwin_CXX:=$(shell xcrun -f clang++) -mmacosx-version-min=$(OSX_MIN_VERSION) -stdlib=libc++ -isysroot$(shell xcrun --show-sdk-path)
 darwin_AR:=$(shell xcrun -f ar)
 darwin_RANLIB:=$(shell xcrun -f ranlib)
 darwin_STRIP:=$(shell xcrun -f strip)
 darwin_LIBTOOL:=$(shell xcrun -f libtool)
 darwin_OTOOL:=$(shell xcrun -f otool)
 darwin_NM:=$(shell xcrun -f nm)
 darwin_INSTALL_NAME_TOOL:=$(shell xcrun -f install_name_tool)
+darwin_native_binutils=
 darwin_native_toolchain=

depends/hosts/android.mk

@@ -1,12 +1,11 @@
 ifeq ($(HOST),armv7a-linux-android)
-android_AR=$(ANDROID_TOOLCHAIN_BIN)/arm-linux-androideabi-ar
 android_CXX=$(ANDROID_TOOLCHAIN_BIN)/$(HOST)eabi$(ANDROID_API_LEVEL)-clang++
 android_CC=$(ANDROID_TOOLCHAIN_BIN)/$(HOST)eabi$(ANDROID_API_LEVEL)-clang
-android_RANLIB=$(ANDROID_TOOLCHAIN_BIN)/arm-linux-androideabi-ranlib
 else
-android_AR=$(ANDROID_TOOLCHAIN_BIN)/$(HOST)-ar
 android_CXX=$(ANDROID_TOOLCHAIN_BIN)/$(HOST)$(ANDROID_API_LEVEL)-clang++
 android_CC=$(ANDROID_TOOLCHAIN_BIN)/$(HOST)$(ANDROID_API_LEVEL)-clang
-android_RANLIB=$(ANDROID_TOOLCHAIN_BIN)/$(HOST)-ranlib
 endif
+android_AR=$(ANDROID_TOOLCHAIN_BIN)/llvm-ar
+android_RANLIB=$(ANDROID_TOOLCHAIN_BIN)/llvm-ranlib
+
 android_cmake_system=Android

depends/hosts/darwin.mk

@@ -1,8 +1,107 @@
-OSX_MIN_VERSION=10.12
-OSX_SDK_VERSION=10.14
-OSX_SDK=$(SDK_PATH)/MacOSX$(OSX_SDK_VERSION).sdk
-darwin_CC=clang -target $(host) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(OSX_SDK)
-darwin_CXX=clang++ -target $(host) -mmacosx-version-min=$(OSX_MIN_VERSION) --sysroot $(OSX_SDK) -stdlib=libc++
+OSX_MIN_VERSION=10.15
+OSX_SDK_VERSION=10.15.6
+XCODE_VERSION=12.1
+XCODE_BUILD_ID=12A7403
+LD64_VERSION=609
+
+OSX_SDK=$(SDK_PATH)/Xcode-$(XCODE_VERSION)-$(XCODE_BUILD_ID)-extracted-SDK-with-libcxx-headers
+
+darwin_native_binutils=native_cctools
+
+ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),)
+# FORCE_USE_SYSTEM_CLANG is empty, so we use our depends-managed, pinned clang
+# from llvm.org
+
+# Clang is a dependency of native_cctools when FORCE_USE_SYSTEM_CLANG is empty
+darwin_native_toolchain=native_cctools
+
+clang_prog=$(build_prefix)/bin/clang
+clangxx_prog=$(clang_prog)++
+
+clang_resource_dir=$(build_prefix)/lib/clang/$(native_clang_version)
+else
+# FORCE_USE_SYSTEM_CLANG is non-empty, so we use the clang from the user's
+# system
+
+darwin_native_toolchain=
+
+# We can't just use $(shell command -v clang) because GNU Make handles builtins
+# in a special way and doesn't know that `command` is a POSIX-standard builtin
+# prior to 1af314465e5dfe3e8baa839a32a72e83c04f26ef, first released in v4.2.90.
+# At the time of writing, GNU Make v4.2.1 is still being used in supported
+# distro releases.
+#
+# Source: https://lists.gnu.org/archive/html/bug-make/2017-11/msg00017.html
+clang_prog=$(shell $(SHELL) $(.SHELLFLAGS) "command -v clang")
+clangxx_prog=$(shell $(SHELL) $(.SHELLFLAGS) "command -v clang++")
+
+clang_resource_dir=$(shell clang -print-resource-dir)
+endif
+
+cctools_TOOLS=AR RANLIB STRIP NM LIBTOOL OTOOL INSTALL_NAME_TOOL
+
+# Make-only lowercase function
+lc = $(subst A,a,$(subst B,b,$(subst C,c,$(subst D,d,$(subst E,e,$(subst F,f,$(subst G,g,$(subst H,h,$(subst I,i,$(subst J,j,$(subst K,k,$(subst L,l,$(subst M,m,$(subst N,n,$(subst O,o,$(subst P,p,$(subst Q,q,$(subst R,r,$(subst S,s,$(subst T,t,$(subst U,u,$(subst V,v,$(subst W,w,$(subst X,x,$(subst Y,y,$(subst Z,z,$1))))))))))))))))))))))))))
+
+# For well-known tools provided by cctools, make sure that their well-known
+# variable is set to the full path of the tool, just like how AC_PATH_{TOO,PROG}
+# would.
+$(foreach TOOL,$(cctools_TOOLS),$(eval darwin_$(TOOL) = $$(build_prefix)/bin/$$(host)-$(call lc,$(TOOL))))
+
+# Flag explanations:
+#
+#     -mlinker-version
+#
+#         Ensures that modern linker features are enabled. See here for more
+#         details: https://github.com/bitcoin/bitcoin/pull/19407.
+#
+#     -B$(build_prefix)/bin
+#
+#         Explicitly point to our binaries (e.g. cctools) so that they are
+#         ensured to be found and preferred over other possibilities.
+#
+#     -stdlib=libc++ -stdlib++-isystem$(OSX_SDK)/usr/include/c++/v1
+#
+#         Forces clang to use the libc++ headers from our SDK and completely
+#         forget about the libc++ headers from the standard directories
+#
+#     -Xclang -*system<path_a> \
+#     -Xclang -*system<path_b> \
+#     -Xclang -*system<path_c> ...
+#
+#         Adds path_a, path_b, and path_c to the bottom of clang's list of
+#         include search paths. This is used to explicitly specify the list of
+#         system include search paths and its ordering, rather than rely on
+#         clang's autodetection routine. This routine has been shown to:
+#             1. Fail to pickup libc++ headers in $SYSROOT/usr/include/c++/v1
+#                when clang was built manually (see: https://github.com/bitcoin/bitcoin/pull/17919#issuecomment-656785034)
+#             2. Fail to pickup C headers in $SYSROOT/usr/include when
+#                C_INCLUDE_DIRS was specified at configure time (see: https://gist.github.com/dongcarl/5cdc6990b7599e8a5bf6d2a9c70e82f9)
+#
+#         Talking directly to cc1 with -Xclang here grants us access to specify
+#         more granular categories for these system include search paths, and we
+#         can use the correct categories that these search paths would have been
+#         placed in if the autodetection routine had worked correctly. (see:
+#         https://gist.github.com/dongcarl/5cdc6990b7599e8a5bf6d2a9c70e82f9#the-treatment)
+#
+#         Furthermore, it places these search paths after any "non-Xclang"
+#         specified search paths. This prevents any additional clang options or
+#         environment variables from coming after or in between these system
+#         include search paths, as that would be wrong in general but would also
+#         break #include_next's.
+#
+darwin_CC=$(clang_prog) --target=$(host) -mmacosx-version-min=$(OSX_MIN_VERSION) \
+              -B$(build_prefix)/bin -mlinker-version=$(LD64_VERSION) \
+              -isysroot$(OSX_SDK) \
+              -Xclang -internal-externc-isystem$(clang_resource_dir)/include \
+              -Xclang -internal-externc-isystem$(OSX_SDK)/usr/include
+darwin_CXX=$(clangxx_prog) --target=$(host) -mmacosx-version-min=$(OSX_MIN_VERSION) \
+               -B$(build_prefix)/bin -mlinker-version=$(LD64_VERSION) \
+               -isysroot$(OSX_SDK) \
+               -stdlib=libc++ \
+               -stdlib++-isystem$(OSX_SDK)/usr/include/c++/v1 \
+               -Xclang -internal-externc-isystem$(clang_resource_dir)/include \
+               -Xclang -internal-externc-isystem$(OSX_SDK)/usr/include
 
 darwin_CFLAGS=-pipe
 darwin_CXXFLAGS=$(darwin_CFLAGS)
@@ -13,5 +112,4 @@ darwin_release_CXXFLAGS=$(darwin_release_CFLAGS)
 darwin_debug_CFLAGS=-O1
 darwin_debug_CXXFLAGS=$(darwin_debug_CFLAGS)
 
-darwin_native_toolchain=native_cctools
 darwin_cmake_system=Darwin

depends/hosts/linux.mk

@@ -7,7 +7,7 @@ linux_release_CXXFLAGS=$(linux_release_CFLAGS)
 linux_debug_CFLAGS=-O1
 linux_debug_CXXFLAGS=$(linux_debug_CFLAGS)
 
-linux_debug_CPPFLAGS=-D_GLIBCXX_DEBUG -D_GLIBCXX_DEBUG_PEDANTIC
+linux_debug_CPPFLAGS=-D_GLIBCXX_DEBUG -D_GLIBCXX_DEBUG_PEDANTIC -D_LIBCPP_DEBUG=1
 
 ifeq (86,$(findstring 86,$(build_arch)))
 i686_linux_CC=gcc -m32

depends/packages/native_cctools.mk

@@ -1,55 +1,23 @@
 package=native_cctools
-$(package)_version=3764b223c011574971ee3ae09ce968ba5dc2f00f
+$(package)_version=2ef2e931cf641547eb8a68cfebde61003587c9fd
 $(package)_download_path=https://github.com/tpoechtrager/cctools-port/archive
 $(package)_file_name=$($(package)_version).tar.gz
-$(package)_sha256_hash=3e35907bf376269a844df08e03cbb43e345c88125374f2228e03724b5f9a2a04
+$(package)_sha256_hash=6b73269efdf5c58a070e7357b66ee760501388549d6a12b423723f45888b074b
 $(package)_build_subdir=cctools
-$(package)_clang_version=6.0.1
-$(package)_clang_download_path=https://releases.llvm.org/$($(package)_clang_version)
-$(package)_clang_download_file=clang+llvm-$($(package)_clang_version)-x86_64-linux-gnu-ubuntu-14.04.tar.xz
-$(package)_clang_file_name=clang-llvm-$($(package)_clang_version)-x86_64-linux-gnu-ubuntu-14.04.tar.xz
-$(package)_clang_sha256_hash=fa5416553ca94a8c071a27134c094a5fb736fe1bd0ecc5ef2d9bc02754e1bef0
-
-$(package)_libtapi_version=3efb201881e7a76a21e0554906cf306432539cef
-$(package)_libtapi_download_path=https://github.com/tpoechtrager/apple-libtapi/archive
-$(package)_libtapi_download_file=$($(package)_libtapi_version).tar.gz
-$(package)_libtapi_file_name=$($(package)_libtapi_version).tar.gz
-$(package)_libtapi_sha256_hash=380c1ca37cfa04a8699d0887a8d3ee1ad27f3d08baba78887c73b09485c0fbd3
-
-$(package)_extra_sources=$($(package)_clang_file_name)
-$(package)_extra_sources += $($(package)_libtapi_file_name)
-
-define $(package)_fetch_cmds
-$(call fetch_file,$(package),$($(package)_download_path),$($(package)_download_file),$($(package)_file_name),$($(package)_sha256_hash)) && \
-$(call fetch_file,$(package),$($(package)_clang_download_path),$($(package)_clang_download_file),$($(package)_clang_file_name),$($(package)_clang_sha256_hash)) && \
-$(call fetch_file,$(package),$($(package)_libtapi_download_path),$($(package)_libtapi_download_file),$($(package)_libtapi_file_name),$($(package)_libtapi_sha256_hash))
-endef
-
-define $(package)_extract_cmds
-  mkdir -p $($(package)_extract_dir) && \
-  echo "$($(package)_sha256_hash)  $($(package)_source)" > $($(package)_extract_dir)/.$($(package)_file_name).hash && \
-  echo "$($(package)_clang_sha256_hash)  $($(package)_source_dir)/$($(package)_clang_file_name)" >> $($(package)_extract_dir)/.$($(package)_file_name).hash && \
-  echo "$($(package)_libtapi_sha256_hash)  $($(package)_source_dir)/$($(package)_libtapi_file_name)" >> $($(package)_extract_dir)/.$($(package)_file_name).hash && \
-  $(build_SHA256SUM) -c $($(package)_extract_dir)/.$($(package)_file_name).hash && \
-  mkdir -p toolchain/bin toolchain/lib/clang/$($(package)_clang_version)/include && \
-  mkdir -p libtapi && \
-  tar --no-same-owner --strip-components=1 -C libtapi -xf $($(package)_source_dir)/$($(package)_libtapi_file_name) && \
-  tar --no-same-owner --strip-components=1 -C toolchain -xf $($(package)_source_dir)/$($(package)_clang_file_name) && \
-  rm -f toolchain/lib/libc++abi.so* && \
-  tar --no-same-owner --strip-components=1 -xf $($(package)_source)
-endef
+$(package)_dependencies=native_libtapi
 
 define $(package)_set_vars
-  $(package)_config_opts=--target=$(host) --disable-lto-support --with-libtapi=$($(package)_extract_dir)
+  $(package)_config_opts=--target=$(host)
   $(package)_ldflags+=-Wl,-rpath=\\$$$$$$$$\$$$$$$$$ORIGIN/../lib
-  $(package)_cc=$($(package)_extract_dir)/toolchain/bin/clang
-  $(package)_cxx=$($(package)_extract_dir)/toolchain/bin/clang++
+  ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),)
+  $(package)_config_opts+=--enable-lto-support --with-llvm-config=$(build_prefix)/bin/llvm-config
+  endif
+  $(package)_cc=$(clang_prog)
+  $(package)_cxx=$(clangxx_prog)
 endef
 
 define $(package)_preprocess_cmds
-  CC=$($(package)_cc) CXX=$($(package)_cxx) INSTALLPREFIX=$($(package)_extract_dir) ./libtapi/build.sh && \
-  CC=$($(package)_cc) CXX=$($(package)_cxx) INSTALLPREFIX=$($(package)_extract_dir) ./libtapi/install.sh && \
-  sed -i.old "/define HAVE_PTHREADS/d" $($(package)_build_subdir)/ld64/src/ld/InputFiles.h
+  cp -f $(BASEDIR)/config.guess $(BASEDIR)/config.sub cctools
 endef
 
 define $(package)_config_cmds
@@ -61,18 +29,9 @@ define $(package)_build_cmds
 endef
 
 define $(package)_stage_cmds
-  $(MAKE) DESTDIR=$($(package)_staging_dir) install && \
-  mkdir -p $($(package)_staging_prefix_dir)/lib/ && \
-  cd $($(package)_extract_dir) && \
-  cp lib/libtapi.so.6 $($(package)_staging_prefix_dir)/lib/ && \
-  cd $($(package)_extract_dir)/toolchain && \
-  mkdir -p $($(package)_staging_prefix_dir)/lib/clang/$($(package)_clang_version)/include && \
-  mkdir -p $($(package)_staging_prefix_dir)/bin $($(package)_staging_prefix_dir)/include && \
-  cp bin/clang $($(package)_staging_prefix_dir)/bin/ &&\
-  cp -P bin/clang++ $($(package)_staging_prefix_dir)/bin/ &&\
-  cp lib/libLTO.so $($(package)_staging_prefix_dir)/lib/ && \
-  cp -rf lib/clang/$($(package)_clang_version)/include/* $($(package)_staging_prefix_dir)/lib/clang/$($(package)_clang_version)/include/ && \
-  cp bin/llvm-dsymutil $($(package)_staging_prefix_dir)/bin/$(host)-dsymutil && \
-  if `test -d include/c++/`; then cp -rf include/c++/ $($(package)_staging_prefix_dir)/include/; fi && \
-  if `test -d lib/c++/`; then cp -rf lib/c++/ $($(package)_staging_prefix_dir)/lib/; fi
+  $(MAKE) DESTDIR=$($(package)_staging_dir) install
+endef
+
+define $(package)_postprocess_cmds
+  rm -rf share
 endef

depends/packages/native_clang.mk

@@ -0,0 +1,32 @@
+package=native_clang
+$(package)_version=10.0.1
+$(package)_download_path=https://github.com/llvm/llvm-project/releases/download/llvmorg-$($(package)_version)
+ifneq (,$(findstring aarch64,$(BUILD)))
+$(package)_download_file=clang+llvm-$($(package)_version)-aarch64-linux-gnu.tar.xz
+$(package)_file_name=clang+llvm-$($(package)_version)-aarch64-linux-gnu.tar.xz
+$(package)_sha256_hash=90dc69a4758ca15cd0ffa45d07fbf5bf4309d47d2c7745a9f0735ecffde9c31f
+else
+$(package)_download_file=clang+llvm-$($(package)_version)-x86_64-linux-gnu-ubuntu-16.04.tar.xz
+$(package)_file_name=clang+llvm-$($(package)_version)-x86_64-linux-gnu-ubuntu-16.04.tar.xz
+$(package)_sha256_hash=48b83ef827ac2c213d5b64f5ad7ed082c8bcb712b46644e0dc5045c6f462c231
+endif
+
+define $(package)_preprocess_cmds
+  rm -f $($(package)_extract_dir)/lib/libc++abi.so*
+endef
+
+define $(package)_stage_cmds
+  mkdir -p $($(package)_staging_prefix_dir)/lib/clang/$($(package)_version)/include && \
+  mkdir -p $($(package)_staging_prefix_dir)/bin && \
+  mkdir -p $($(package)_staging_prefix_dir)/include && \
+  cp bin/clang $($(package)_staging_prefix_dir)/bin/ && \
+  cp -P bin/clang++ $($(package)_staging_prefix_dir)/bin/ && \
+  cp bin/dsymutil $($(package)_staging_prefix_dir)/bin/$(host)-dsymutil && \
+  cp bin/llvm-config $($(package)_staging_prefix_dir)/bin/ && \
+  cp lib/libLTO.so $($(package)_staging_prefix_dir)/lib/ && \
+  cp -rf lib/clang/$($(package)_version)/include/* $($(package)_staging_prefix_dir)/lib/clang/$($(package)_version)/include/
+endef
+
+define $(package)_postprocess_cmds
+  rmdir include
+endef

depends/packages/native_libtapi.mk

@@ -0,0 +1,20 @@
+package=native_libtapi
+$(package)_version=664b8414f89612f2dfd35a9b679c345aa5389026
+$(package)_download_path=https://github.com/tpoechtrager/apple-libtapi/archive
+$(package)_download_file=$($(package)_version).tar.gz
+$(package)_file_name=$($(package)_version).tar.gz
+$(package)_sha256_hash=62e419c12d1c9fad67cc1cd523132bc00db050998337c734c15bc8d73cc02b61
+
+ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),)
+$(package)_dependencies=native_clang
+endif
+
+define $(package)_build_cmds
+  CC=$(clang_prog) CXX=$(clangxx_prog) INSTALLPREFIX=$($(package)_staging_prefix_dir) ./build.sh
+endef
+
+define $(package)_stage_cmds
+  ./install.sh && \
+  mkdir -p $($(package)_staging_prefix_dir)/include/llvm-c && \
+  cp src/llvm/include/llvm-c/lto.h $($(package)_staging_prefix_dir)/include/llvm-c
+endef

depends/packages/packages.mk

@@ -17,5 +17,10 @@ upnp_packages=miniupnpc
 darwin_native_packages = native_biplist native_ds_store native_mac_alias
 
 ifneq ($(build_os),darwin)
-darwin_native_packages += native_cctools native_cdrkit native_libdmg-hfsplus
+darwin_native_packages += native_cctools native_libtapi native_cdrkit native_libdmg-hfsplus
+
+ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),)
+darwin_native_packages+= native_clang
+endif
+
 endif

depends/packages/sodium.mk

@@ -8,7 +8,7 @@ $(package)_patches=disable-glibc-getrandom-getentropy.patch fix-whitespace.patch
 define $(package)_set_vars
 $(package)_config_opts=--enable-static --disable-shared --with-pic="yes"
 $(package)_config_opts+=--prefix=$(host_prefix)
-$(package)_config_opts_darwin=RANLIB="$(host_prefix)/native/bin/$(host)-ranlib" AR="$(host_prefix)/native/bin/$(host)-ar" CC="$(host_prefix)/native/bin/$($(package)_cc)"
+$(package)_config_opts_darwin=RANLIB="$($(package)_ranlib)" AR="$($(package)_ar)" CC="$($(package)_cc)"
 endef
 
 define $(package)_config_cmds