Specify each lib only once; allow llvm build in shared libs mode. (#77)

* Specify each lib only once; allow llvm build in shared libs mode.

* Remove debug code.

* For library targets, retain dependency information using add_dependencies, but do not link using taget_link_libraries.

* Do not set LD_PRELOAD by default.

Co-authored-by: Gong Su <gongsu@us.ibm.com>
Co-authored-by: Gheorghe-Teodor Bercea <gt.bercea@gmail.com>

MLIR.cmake

@@ -44,14 +44,26 @@ set(
 )
 include_directories(${MLIR_INCLUDE_PATHS})
 
+# Force BUILD_SHARED_LIBS to be the same as LLVM build
+file(STRINGS ${LLVM_PROJ_BUILD}/CMakeCache.txt shared REGEX BUILD_SHARED_LIBS)
+string(REGEX REPLACE "BUILD_SHARED_LIBS:BOOL=" "" shared ${shared})
+set(BUILD_SHARED_LIBS ${shared} CACHE BOOL "" FORCE)
+message(STATUS "BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS}")
+
 # Threading libraries required due to parallel pass execution.
 find_package(Threads REQUIRED)
+# libcurses and libz required by libLLVMSupport
+find_package(Curses REQUIRED)
+find_package(ZLIB REQUIRED)
 
 function(find_mlir_lib lib)
   find_library(${lib}
           NAMES ${lib}
           PATHS ${LLVM_PROJECT_LIB}
           NO_DEFAULT_PATH)
+  if(${${lib}} STREQUAL ${lib}-NOTFOUND)
+    message(FATAL_ERROR "${lib} not found")
+  endif()
 endfunction(find_mlir_lib)
 
 find_mlir_lib(MLIRAffine)
@@ -106,26 +118,15 @@ find_mlir_lib(LLVMProfileData)
 find_mlir_lib(LLVMDemangle)
 find_mlir_lib(LLVMFrontendOpenMP)
 
-
+set(MLIRLibs
-set(MLIRLibsOnce
+        ${MLIRLLVMIR}
-        ${LLVMAnalysis}
+        ${MLIROptLib}
-        ${LLVMAsmParser}
+        ${MLIRParser}
-        ${LLVMBinaryFormat}
+        ${MLIRPass}
-        ${LLVMBitReader}
+        ${MLIRTargetLLVMIR}
-        ${LLVMBitstreamReader}
+        ${MLIRTargetLLVMIRModuleTranslation}
-        ${LLVMBitWriter}
+        ${MLIRTransforms}
-        ${LLVMCore}
+        ${MLIRTransformUtils}
-        ${LLVMFrontendOpenMP}
-        ${LLVMIRReader}
-        ${LLVMMC}
-        ${LLVMMCParser}
-        ${LLVMMLIRTableGen}        
-        ${LLVMObject}
-        ${LLVMRemarks}
-        ${LLVMSupport}
-        ${LLVMTransformUtils}
-        ${LLVMProfileData}
-        ${LLVMDemangle}
         ${MLIRAffine}
         ${MLIRAffineToStandard}
         ${MLIRAnalysis}
@@ -135,7 +136,6 @@ set(MLIRLibsOnce
         ${MLIREDSC}
         ${MLIRExecutionEngine}
         ${MLIRIR}
-        ${MLIRLLVMIR}
         ${MLIRLLVMIRTransforms}        
         ${MLIRLoopToStandard}
         ${MLIRLoopOps}
@@ -143,24 +143,39 @@ set(MLIRLibsOnce
         ${MLIRLoopLikeInterface}
         ${MLIROpenMP}
         ${MLIRMlirOptMain}
-        ${MLIROptLib}
-        ${MLIRParser}
-        ${MLIRPass}
         ${MLIRSideEffects}        
         ${MLIRStandardOps}
         ${MLIRStandardToLLVM}
         ${MLIRSupport}
-        ${MLIRTargetLLVMIR}
+        ${MLIRTranslation}
-        ${MLIRTargetLLVMIRModuleTranslation}
+        # strict order verified
-        ${MLIRTransforms}
+        ${LLVMBitWriter}
-        ${MLIRTransformUtils}
+        ${LLVMObject}
-        ${MLIRTranslation})
+        ${LLVMBitReader}
-
+        # strict order verified
-set(MLIRLibs
+        ${LLVMFrontendOpenMP}
-        ${MLIRLibsOnce}
+        ${LLVMTransformUtils}
-        ${MLIRLibsOnce}
+        ${LLVMAnalysis}
-        Threads::Threads)
+        # strict order verified
+        ${LLVMAsmParser}
+        ${LLVMCore}
+        # strict order not yet verified
+        ${LLVMRemarks}
+        ${LLVMMCParser}
+        ${LLVMMC}
+        ${LLVMProfileData}
+        ${LLVMBinaryFormat}
+        ${LLVMBitstreamReader}
+        ${LLVMIRReader}
+        ${LLVMMLIRTableGen}
+        ${LLVMSupport}
+        ${LLVMDemangle}
+        ${CMAKE_THREAD_LIBS_INIT}
+	${CURSES_LIBRARIES}
+	${ZLIB_LIBRARIES})
 
+# MLIR libraries that must be linked with --whole-archive for static build or
+# must be specified on LD_PRELOAD for shared build.
 set(MLIRWholeArchiveLibs
         MLIRAffineToStandard
         MLIRAffine
@@ -172,6 +187,20 @@ set(MLIRWholeArchiveLibs
         MLIRVector
         MLIRLoopOps)
 
+# ONNX MLIR libraries that must be linked with --whole-archive for static build or
+# must be specified on LD_PRELOAD for shared build.
+set(ONNXMLIRWholeArchiveLibs
+        OMKrnlToAffine
+        OMKrnlToLLVM
+        OMONNXToKrnl
+        OMONNXRewrite
+        OMShapeInference
+        OMShapeInferenceOpInterface
+        OMAttributePromotion
+        OMPromotableConstOperandsOpInterface)
+
+# Function to construct linkage option for the static libraries that must be
+# linked with --whole-archive (or equivalent).
 function(whole_archive_link target lib_dir)
   get_property(link_flags TARGET ${target} PROPERTY LINK_FLAGS)
   if("${CMAKE_SYSTEM_NAME}" STREQUAL "Darwin")
@@ -194,15 +223,39 @@ function(whole_archive_link target lib_dir)
   set_target_properties(${target} PROPERTIES LINK_FLAGS ${link_flags})
 endfunction(whole_archive_link)
 
+# Function to construct LD_PRELOAD value for the shared libraries whose
+# static counterpart need --whole-archive linkage option.
+function(ld_preload_libs target lib_dir)
+  foreach(lib ${ARGN})
+    if("${${lib}}" STREQUAL "")
+      set(ONNX_MLIR_LD_PRELOAD_${target}
+	"${ONNX_MLIR_LD_PRELOAD_${target}}:${lib_dir}/lib${lib}.so"
+	CACHE STRING "" FORCE)
+    else()
+      set(ONNX_MLIR_LD_PRELOAD_${target}
+	"${ONNX_MLIR_LD_PRELOAD_${target}}:${${lib}}"
+	CACHE STRING "" FORCE)
+    endif()
+  endforeach(lib)
+endfunction(ld_preload_libs)
+
 function(whole_archive_link_mlir target)
-  whole_archive_link(${target} ${LLVM_PROJ_BUILD}/lib ${ARGN})
+  if(BUILD_SHARED_LIBS)
+    ld_preload_libs(${target} ${LLVM_PROJ_BUILD}/lib ${ARGN})
+  else()
+    whole_archive_link(${target} ${LLVM_PROJ_BUILD}/lib ${ARGN})
+  endif()
 endfunction(whole_archive_link_mlir)
 
 function(whole_archive_link_onnx_mlir target)
   foreach(lib_target ${ARGN})
     add_dependencies(${target} ${lib_target})
   endforeach(lib_target)
-  whole_archive_link(${target} ${CMAKE_BINARY_DIR}/lib ${ARGN})
+  if(BUILD_SHARED_LIBS)
+    ld_preload_libs(${target} ${CMAKE_BINARY_DIR}/lib ${ARGN})
+  else()
+    whole_archive_link(${target} ${CMAKE_BINARY_DIR}/lib ${ARGN})
+  endif()
 endfunction(whole_archive_link_onnx_mlir)
 
 set(LLVM_CMAKE_DIR

src/Builder/CMakeLists.txt

@@ -8,14 +8,19 @@ target_include_directories(OMBuilder PRIVATE ${ONNX_MLIR_SRC_ROOT})
 target_include_directories(OMBuilder PRIVATE ${CMAKE_BINARY_DIR})
 target_include_directories(OMBuilder PRIVATE ${ONNX_MLIR_BIN_ROOT})
 
+# This will cause onnx to be built. More importantly, some variable definitions
+# when building onnx such as -DONNX_ML=1 -DONNX_NAMESPACE=onnx will be carried over
+# when compiling FrontendDialectHelper.cpp, etc.
 target_link_libraries(OMBuilder
-        OMONNXOps
         onnx
-        ${MLIRLibs}
-        curses
         mpark_variant)
 target_include_directories(OMBuilder
                            PUBLIC
                            ${ONNX_MLIR_SRC_ROOT}/third_party/onnx
                            ${ONNX_MLIR_SRC_ROOT}/third_party/variant
                            ${ONNX_MLIR_SRC_ROOT})
+# If you add onnx here, it will also cause onnx to be built. However, some
+# variable definitions when building onnx such as -DONNX_ML=1 -DONNX_NAMESPACE=onnx
+# will NOT be carried over when compiling FrontendDialectHelper.cpp, etc. so
+# the compilation will fail.
+add_dependencies(OMBuilder OMONNXOps)

src/CMakeLists.txt

@@ -10,24 +10,36 @@ add_executable(onnx-mlir
         MainUtils.hpp
         MainUtils.cpp
         main.cpp)
+
+set(ONNX_MLIR_LD_PRELOAD_onnx-mlir "" CACHE STRING "" FORCE)
+whole_archive_link_mlir(onnx-mlir ${MLIRWholeArchiveLibs})
+if(BUILD_SHARED_LIBS)
+  message(STATUS "To run dynamically linked onnx-mlir, you must specify:")
+  message(STATUS "LD_PRELOAD=${ONNX_MLIR_LD_PRELOAD_onnx-mlir}")
+endif()
+
+# Libraries specified on the target_link_libraries for the add_subdirectory
+# targets get added to the end of the list here. This creates two problems:
+# 1. It produces duplicated libraries being specified for the link command.
+# 2. The libraries added at the end may depend on other libraries thus
+#    cause linkage errors due to undefined symbols.
+# So it's better not to use target_link_libraries for the add_subdirectory
+# targets, or only use it for libraries that have no further dependencies
+# (except system libraries such as libc).
 target_link_libraries(onnx-mlir
-        ${MLIRLibs}
         OMBuilder
         OMKrnlOps
         OMONNXOps
+        OMShapeInference
+        OMShapeInferenceOpInterface
+        OMAttributePromotion
+        OMPromotableConstOperandsOpInterface
         OMKrnlToAffine
         OMKrnlToLLVM
-        OMShapeInference
         OMONNXToKrnl
         OMONNXRewrite
-        OMAttributePromotion
+        ${MLIRLibs}
-        ${CMAKE_THREAD_LIBS_INIT}
         ${CMAKE_DL_LIBS})
-whole_archive_link_mlir(onnx-mlir
-        ${MLIRWholeArchiveLibs})
-
-find_package(ZLIB REQUIRED)
-target_link_libraries(onnx-mlir ${ZLIB_LIBRARIES})
 
 target_include_directories(onnx-mlir PRIVATE ${ONNX_MLIR_SRC_ROOT})
 target_include_directories(onnx-mlir PRIVATE ${CMAKE_BINARY_DIR})

src/Conversion/ONNXToKrnl/CMakeLists.txt

@@ -22,6 +22,7 @@ target_include_directories(OMONNXToKrnl
         ${ONNX_MLIR_SRC_ROOT}
         ${ONNX_MLIR_BIN_ROOT}
         ${ONNX_MLIR_SRC_ROOT})
-target_link_libraries(OMONNXToKrnl
+
-        ${MLIRLibs}
+# Linking dependencies:
+add_dependencies(OMONNXToKrnl
         OMKrnlOps)

src/Dialect/ONNX/CMakeLists.txt

@@ -13,7 +13,8 @@ target_include_directories(OMONNXOps
         ${ONNX_MLIR_BIN_ROOT}
         ${ONNX_MLIR_SRC_ROOT})
 add_dependencies(OMONNXOps OMONNXOpsIncGen)
-target_link_libraries(OMONNXOps
+# Linking dependencies:
+add_dependencies(OMONNXOps
         OMPromotableConstOperandsOpInterface
         OMShapeInferenceOpInterface)
 

src/Tool/ONNXMLIROpt/CMakeLists.txt

@@ -4,22 +4,17 @@ add_dependencies(onnx-mlir-opt OMKrnlOpsIncGen)
 target_include_directories(onnx-mlir-opt PRIVATE ${ONNX_MLIR_SRC_ROOT})
 target_include_directories(onnx-mlir-opt PRIVATE ${ONNX_MLIR_BIN_ROOT})
 
+set(ONNX_MLIR_LD_PRELOAD_onnx-mlir-opt "" CACHE STRING "" FORCE)
+whole_archive_link_onnx_mlir(onnx-mlir-opt ${ONNXMLIRWholeArchiveLibs})
+whole_archive_link_mlir(onnx-mlir-opt ${MLIRWholeArchiveLibs})
+if(BUILD_SHARED_LIBS)
+  message(STATUS "To run dynamically linked onnx-mlir-opt, you must specify:")
+  message(STATUS "LD_PRELOAD=${ONNX_MLIR_LD_PRELOAD_onnx-mlir-opt}")
+endif()
+
 target_link_libraries(onnx-mlir-opt
-        ${MLIRLibs}
         OMBuilder
         OMKrnlOps
-        OMKrnlToAffine
+        OMONNXOps
-        OMKrnlToLLVM
-        OMShapeInference
-        OMONNXToKrnl
-        OMPromotableConstOperandsOpInterface
-        curses)
-whole_archive_link_mlir(onnx-mlir-opt
-        ${MLIRWholeArchiveLibs})
-whole_archive_link_onnx_mlir(onnx-mlir-opt
-        OMKrnlToAffine
-        OMKrnlToLLVM
-        OMONNXToKrnl
         OMONNXRewrite
-        OMShapeInference
+        ${MLIRLibs})
-        OMAttributePromotion)

src/Transform/CMakeLists.txt

@@ -5,8 +5,9 @@ target_include_directories(OMKrnlToAffine
         ${ONNX_MLIR_SRC_ROOT}
         ${ONNX_MLIR_BIN_ROOT}
         ${ONNX_MLIR_SRC_ROOT})
-target_link_libraries(OMKrnlToAffine
+
-        ${MLIRLibs}
+# Linking dependencies:
+add_dependencies(OMKrnlToAffine
         OMKrnlOps
         OMONNXOps)
 
@@ -17,8 +18,9 @@ target_include_directories(OMKrnlToLLVM
         ${ONNX_MLIR_SRC_ROOT}
         ${ONNX_MLIR_BIN_ROOT}
         ${ONNX_MLIR_SRC_ROOT})
-target_link_libraries(OMKrnlToLLVM
+
-        ${MLIRLibs}
+#Linking dependencies:
+add_dependencies(OMKrnlToLLVM
         OMKrnlOps
         OMONNXOps)
 

src/Transform/ONNX/CMakeLists.txt

@@ -3,7 +3,9 @@ add_library(OMAttributePromotion
 target_include_directories(OMAttributePromotion
         PRIVATE ${ONNX_MLIR_SRC_ROOT} ${ONNX_MLIR_BIN_ROOT}
         ${ONNF_MLIR_SRC_ROOT})
-target_link_libraries(OMAttributePromotion
+
+# Linking dependencies:
+add_dependencies(OMAttributePromotion
         OMPromotableConstOperandsOpInterface)
 
 set(LLVM_TARGET_DEFINITIONS ONNXRewrite.td)
@@ -25,18 +27,19 @@ add_library(OMONNXRewrite
 target_include_directories(OMONNXRewrite
         PRIVATE ${ONNX_MLIR_SRC_ROOT} ${ONNX_MLIR_BIN_ROOT}
         ${ONNF_MLIR_SRC_ROOT})
-target_link_libraries(OMONNXRewrite
-        OMONNXOps)
 add_dependencies(OMONNXRewrite
         OMONNXRewriteIncGen
         OMONNXDecomposeIncGen
         OMONNXCombineIncGen)
+# Linking dependencies:
+add_dependencies(OMONNXRewrite
+        OMONNXOps)
 
 add_library(OMShapeInference ShapeInferencePass.cpp)
 target_include_directories(OMShapeInference
         PRIVATE ${ONNX_MLIR_SRC_ROOT} ${ONNX_MLIR_BIN_ROOT}
         ${ONNX_MLIR_SRC_ROOT})
-target_link_libraries(OMShapeInference
-        ${MLIRLibs}
-        OMShapeInferenceOpInterface)
 add_dependencies(OMShapeInference ShapeInferenceOpInterfaceIncGen)
+# Linking dependencies:
+add_dependencies(OMShapeInference
+        OMShapeInferenceOpInterface)

test/mlir/lit.site.cfg.py.in

@@ -1,6 +1,9 @@
 
 import lit.llvm
 
+if '@BUILD_SHARED_LIBS@' == 'ON':
+    config.environment['LD_PRELOAD'] = "@ONNX_MLIR_LD_PRELOAD_onnx-mlir-opt@"
+
 config.llvm_tools_dir = "@MLIR_TOOLS_DIR@"
 config.mlir_obj_root = "@LLVM_PROJ_BUILD@"
 config.mlir_tools_dir = "@MLIR_TOOLS_DIR@"