Merge pull request #253 from hoopoepg/topic/static-build-backport-int3

BUILD/STATIC: backport of static build

.gitignore

@@ -52,6 +52,8 @@ test/apps/test_link_map
 test/apps/test_ucp_dlopen
 test/apps/test_ucs_dlopen
 test/apps/sockaddr/sa
+test/apps/test_init_mt
+test/apps/test_memtrack_limit
 test/examples/ucp_client_server
 test/examples/ucp_hello_world
 test/examples/uct_hello_world
@@ -63,7 +65,7 @@ debian/control
 debian/rules
 debian/ucx.postinst
 ucx.spec
-ucx.pc
+ucx*.pc
 doc/doxygen-doc
 doc/uml/uct.pdf
 doc/doxygen/header.tex

buildlib/check_tls.sh

@@ -0,0 +1,29 @@
+#!/bin/sh
+#
+# Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# See file LICENSE for terms.
+#
+
+# check_tls.sh [extra-tls-to-check]
+
+res=true;
+for run in ./uct_info_static ./uct_info; do
+    echo Check for $run;
+    output=`$run`;
+    for i in tcp self sysv posix $@; do
+        if `echo $output | grep -q "Transport: $i"`; then
+            echo $i... ok;
+        else
+            res=false;
+            echo $i... fail;
+        fi;
+    done;
+    if test $res = false; then
+        echo failed to check tl in $run;
+        break;
+    fi;
+done;
+$res
+
+# done

buildlib/pr/io_demo/io-demo.yml

@@ -74,6 +74,14 @@ jobs:
           ./contrib/configure-devel --prefix=$(Build.Repository.LocalPath)/install --without-java
           make -j`nproc`
           make install
+          # build static modules
+          source ./buildlib/az-helpers.sh
+          az_init_modules
+          module load dev/libnl
+          module load dev/numactl
+          PKG_CONFIG_PATH=$(Build.Repository.LocalPath)/install/lib/pkgconfig:$PKG_CONFIG_PATH make -C test/apps/uct_info EXTRA_MODULES="ucx-ib ucx-cma ucx-rdmacm"
+          module unload dev/numactl
+          module unload dev/libnl
         displayName: Build
         name: build
       - task: CopyFiles@2
@@ -82,8 +90,10 @@ jobs:
           contents: |
             buildlib/az-helpers.sh
             buildlib/az-network-corrupter.sh
+            buildlib/check_tls.sh
             install/**
             test/apps/iodemo/run_io_demo.sh
+            test/apps/uct_info/*
           targetFolder: '$(Build.ArtifactStagingDirectory)'
       - task: PublishBuildArtifacts@1
         inputs:
@@ -140,3 +150,48 @@ jobs:
           extra_run_args: $(test_extra_run_args)
           ${{ if eq(variables['Build.Reason'], 'PullRequest') }}:
             analyzer_allow_list_args: '--allow_list $(System.PullRequest.TargetBranch)'
+
+  - job: test_static
+    dependsOn: io_build
+    workspace:
+      clean: all
+
+    pool:
+      name: MLNX
+      demands: ${{ parameters.demands }}
+
+    variables:
+      workspace: drop_$(Build.BuildId)
+      EXECUTOR_NUMBER: $(Build.BuildId)
+
+    displayName: "Test static"
+    steps:
+      - checkout: none
+        clean: true
+      - task: DownloadBuildArtifacts@0
+        displayName: 'Download Build Artifacts'
+        inputs:
+          artifactName: drop_$(Build.BuildId)
+          downloadPath: $(System.DefaultWorkingDirectory)
+      - bash: chmod u+rwx $(workspace) -R
+      - bash: |
+          set -eEx
+          cd $(workspace)/test/apps/uct_info
+          LD_LIBRARY_PATH=$(System.DefaultWorkingDirectory)/$(workspace)/install/lib:$LD_LIBRARY_PATH \
+                  $(System.DefaultWorkingDirectory)/$(workspace)/buildlib/check_tls.sh \
+                  dc_mlx5 rc_mlx5 ud_mlx5 rc_verbs ud_verbs cma
+          # Set port number for hello_world applications
+          server_port=$((10000 + (EXECUTOR_NUMBER % 1000)))
+          server_port_arg="-p $server_port"
+
+          for tls in ib rc rc_x; do
+            echo UCX_TLS=$tls
+            UCX_TLS=$tls ./ucp_hello_world_static ${server_port_arg} &
+            # allow server to start
+            sleep 3
+            UCX_TLS=$tls ./ucp_hello_world_static ${server_port_arg} -n localhost
+            # allow server to complete
+            sleep 3
+          done
+
+          cd -

buildlib/pr/main.yml

@@ -1,7 +1,7 @@
 resources:
   containers:
     - container: centos7
-      image: rdmz-harbor.rdmz.labs.mlnx/ucx/centos7:3
+      image: rdmz-harbor.rdmz.labs.mlnx/ucx/centos7:5
       options: -v /hpc/local:/hpc/local -v /auto/sw_tools:/auto/sw_tools
 
 stages:

config/m4/ucs.m4

@@ -68,6 +68,7 @@ AS_IF([test "x$enable_backtrace_detail" = xyes],
 
 	if test "x$BT" = "x1"; then
 		AC_CHECK_FUNCS([cplus_demangle])
+		AC_SUBST([BFD_LDFLAGS], ["-lbfd -liberty -lz"])
 		AC_DEFINE([HAVE_DETAILED_BACKTRACE], 1, [Enable detailed backtrace])
         case ${host} in
             aarch64*) CFLAGS="$CFLAGS -funwind-tables" ;;

config/module-pkg-config.am

@@ -0,0 +1,9 @@
+#
+# Copyright (C) 2022 Nvidia Corporation. All Rights Reserved.
+# See file LICENSE for terms.
+#
+
+EXTRA_DIST = ucx-$(PKG_CONFIG_NAME).pc.in
+pkgconfigdir = $(libdir)/pkgconfig
+pkgconfig_DATA = ucx-$(PKG_CONFIG_NAME).pc
+

configure.ac

@@ -85,6 +85,12 @@ AC_FUNC_STRERROR_R
 AC_PATH_TOOL([PKG_CONFIG], [pkg-config], [pkg-config])
 
 
+#
+# Define SHARED_LIB preprocessor macro when building a shared library
+#
+lt_prog_compiler_pic="$lt_prog_compiler_pic -DUCX_SHARED_LIB"
+
+
 #
 # Force link_all_deplibs=yes for libtool, otherwise it will not
 # link against dependency libs
@@ -222,9 +228,11 @@ AS_IF([test "x$with_docs_only" = xyes],
      m4_include([config/m4/rocm.m4])
      m4_include([config/m4/gdrcopy.m4])
      m4_include([src/ucm/configure.m4])
+     m4_include([src/ucs/configure.m4])
      m4_include([src/uct/configure.m4])
      m4_include([src/tools/perf/configure.m4])
      m4_include([test/gtest/configure.m4])
+     m4_include([test/apps/uct_info/configure.m4])
 
 
      #
@@ -356,7 +364,6 @@ AC_CONFIG_FILES([
                  debian/rules
                  debian/control
                  debian/changelog
-                 src/ucs/Makefile
                  src/ucp/Makefile
                  src/ucp/api/ucp_version.h
                  src/ucp/core/ucp_version.c

src/ucm/Makefile.am

@@ -57,3 +57,7 @@ libucm_la_SOURCES += \
 noinst_HEADERS += \
     ptmalloc286/malloc-2.8.6.h
 endif
+
+PKG_CONFIG_NAME=ucm
+
+include $(top_srcdir)/config/module-pkg-config.am

src/ucm/api/ucm.h

@@ -210,7 +210,7 @@ typedef struct ucm_global_config {
 
 /*
  * Global UCM configuration to be set externally.
- * @deprecated replaced by @ref ucm_library_init.
+ * @deprecated replaced by @ref ucm_set_global_opts.
  */
 extern ucm_global_config_t ucm_global_opts;
 
@@ -254,14 +254,14 @@ typedef void (*ucm_event_callback_t)(ucm_event_type_t event_type,
 
 
 /**
- * Initialize UCM library and set its configuration.
+ * Set UCM library configuration.
  *
  * @param [in]  ucm_opts   UCM library global configuration. If NULL, default
  *                         configuration is applied.
  *
  * @note Calling this function more than once in the same process has no effect.
  */
-void ucm_library_init(const ucm_global_config_t *ucm_opts);
+void ucm_set_global_opts(const ucm_global_config_t *ucm_opts);
 
 
 /**

src/ucm/bistro/bistro_aarch64.c

@@ -59,10 +59,9 @@
  */
 #define BR(_reg) ((0xd61f << 16) + ((_reg) << 5))
 
-ucs_status_t ucm_bistro_patch(const char *symbol, void *hook,
+ucs_status_t ucm_bistro_patch(void *func_ptr, void *hook,
                               ucm_bistro_restore_point_t **rp)
 {
-    void *func;
     ucs_status_t status;
 
     ucm_bistro_patch_t patch = {
@@ -73,14 +72,12 @@ ucs_status_t ucm_bistro_patch(const char *symbol, void *hook,
         .br   = BR(R15)
     };
 
-    UCM_LOOKUP_SYMBOL(func, symbol);
-
-    status = ucm_bistro_create_restore_point(func, sizeof(patch), rp);
+    status = ucm_bistro_create_restore_point(func_ptr, sizeof(patch), rp);
     if (UCS_STATUS_IS_ERR(status)) {
         return status;
     }
 
-    return ucm_bistro_apply_patch(func, &patch, sizeof(patch));
+    return ucm_bistro_apply_patch(func_ptr, &patch, sizeof(patch));
 }
 
 #endif

src/ucm/bistro/bistro_aarch64.h

@@ -28,14 +28,14 @@ typedef struct ucm_bistro_patch {
  * Set library function call hook using Binary Instrumentation
  * method (BISTRO): replace function body by user defined call
  *
- * @param symbol function name to replace
- * @param hook   user-defined function-replacer
- * @param rp     restore point used to restore original function,
- *               optional, may be NULL
+ * @param symbol_ptr function pointer to replace
+ * @param hook       user-defined function-replacer
+ * @param rp         restore point used to restore original function,
+ *                   optional, may be NULL
  *
  * @return Error code as defined by @ref ucs_status_t
  */
-ucs_status_t ucm_bistro_patch(const char *symbol, void *hook,
+ucs_status_t ucm_bistro_patch(void *func_ptr, void *hook,
                               ucm_bistro_restore_point_t **rp);
 
 #endif

src/ucm/bistro/bistro_ppc64.c

@@ -134,20 +134,17 @@ static void *ucm_bistro_get_text_addr(void *addr)
 #endif
 }
 
-ucs_status_t ucm_bistro_patch_toc(const char *symbol, void *hook,
+ucs_status_t ucm_bistro_patch_toc(void *func_ptr, void *hook,
                                   ucm_bistro_restore_point_t **rp,
                                   uint64_t toc)
 {
     ucs_status_t status;
-    void *func;
     ucm_bistro_restore_point_t restore;
     ucm_bistro_patch_t patch;
 
-    UCM_LOOKUP_SYMBOL(func, symbol);
+    restore.entry = func_ptr;
 
-    restore.entry = func;
-
-    func = ucm_bistro_get_text_addr(func);
+    func_ptr = ucm_bistro_get_text_addr(func_ptr);
     hook = ucm_bistro_get_text_addr(hook);
 
     status = ucm_bistro_patch_hook(hook, &restore, toc);
@@ -156,16 +153,16 @@ ucs_status_t ucm_bistro_patch_toc(const char *symbol, void *hook,
     }
 
 #if defined(_CALL_ELF) && (_CALL_ELF == 2)
-    func += 8;
-    hook += 8;
+    func_ptr += 8;
+    hook     += 8;
 #endif
 
     ucm_bistro_fill_patch(&patch, R11, (uintptr_t)hook);
 
-    restore.func       = func;
-    restore.func_patch = *(ucm_bistro_patch_t*)func;
+    restore.func       = func_ptr;
+    restore.func_patch = *(ucm_bistro_patch_t*)func_ptr;
 
-    status = ucm_bistro_apply_patch(func, &patch, sizeof(patch));
+    status = ucm_bistro_apply_patch(func_ptr, &patch, sizeof(patch));
     if (UCS_STATUS_IS_ERR(status)) {
         return status;
     }

src/ucm/bistro/bistro_ppc64.h

@@ -26,26 +26,26 @@
  * Set library function call hook using Binary Instrumentation
  * method (BISTRO): replace function body by user defined call
  *
- * @param symbol function name to replace
- * @param hook   user-defined function-replacer
- * @param rp     restore point used to restore original function,
- *               optional, may be NULL
+ * @param func_ptr function pointer to replace
+ * @param hook     user-defined function-replacer
+ * @param rp       restore point used to restore original function,
+ *                 optional, may be NULL
  *
  * @return Error code as defined by @ref ucs_status_t
  */
 /* we have to use inline proxy call to save TOC register
  * value - PPC is very sensible to this register value */
-ucs_status_t ucm_bistro_patch_toc(const char *symbol, void *hook,
+ucs_status_t ucm_bistro_patch_toc(void *func_ptr, void *hook,
                                   ucm_bistro_restore_point_t **rp,
                                   uint64_t toc);
 
 static inline
-ucs_status_t ucm_bistro_patch(const char *symbol, void *hook,
+ucs_status_t ucm_bistro_patch(void *func_ptr, void *hook,
                               ucm_bistro_restore_point_t **rp)
 {
     uint64_t toc;
     asm volatile ("std 2, %0" : "=m" (toc));
-    return ucm_bistro_patch_toc(symbol, hook, rp, toc);
+    return ucm_bistro_patch_toc(func_ptr, hook, rp, toc);
 }
 
 #endif

src/ucm/bistro/bistro_x86_64.c

@@ -26,7 +26,7 @@
 #include <ucs/debug/assert.h>
 
 
-ucs_status_t ucm_bistro_patch(const char *symbol, void *hook,
+ucs_status_t ucm_bistro_patch(void *func_ptr, void *hook,
                               ucm_bistro_restore_point_t **rp)
 {
     ucm_bistro_jmp_r11_patch_t patch_jmp_r11   = {
@@ -36,14 +36,12 @@ ucs_status_t ucm_bistro_patch(const char *symbol, void *hook,
     ucm_bistro_jmp_near_patch_t patch_jmp_near = {
         .jmp_rel = 0xe9
     };
-    void *func, *patch, *jmp_base;
+    void *patch, *jmp_base;
     ucs_status_t status;
     ptrdiff_t jmp_disp;
     size_t patch_len;
 
-    UCM_LOOKUP_SYMBOL(func, symbol);
-
-    jmp_base = UCS_PTR_BYTE_OFFSET(func, sizeof(patch_jmp_near));
+    jmp_base = UCS_PTR_BYTE_OFFSET(func_ptr, sizeof(patch_jmp_near));
     jmp_disp = UCS_PTR_BYTE_DIFF(jmp_base, hook);
     if (labs(jmp_disp) < INT32_MAX) {
         /* if 32-bit near jump is possible, use it, since it's a short 5-byte
@@ -58,11 +56,11 @@ ucs_status_t ucm_bistro_patch(const char *symbol, void *hook,
         patch_len           = sizeof(patch_jmp_r11);
     }
 
-    status = ucm_bistro_create_restore_point(func, patch_len, rp);
+    status = ucm_bistro_create_restore_point(func_ptr, patch_len, rp);
     if (UCS_STATUS_IS_ERR(status)) {
         return status;
     }
 
-    return ucm_bistro_apply_patch(func, patch, patch_len);
+    return ucm_bistro_apply_patch(func_ptr, patch, patch_len);
 }
 #endif