Remove whole archive linkage (#173)

* Explicit pass registration.

* Remove whole-archive linking, replace with regular linking.

* Remove whole-archive linkage related scripts.

* No need to preload library, simply expose them through LD_LIBRARY_PATH.

* Use OMLibs to record all onnx-mlir libs.

* Add OMResultTypeInferenceOpInterface lib to OMLibs.

* nit.

* No need to expose libs through LD_LIBRARY_PATH.

* Fix missing onnx header file issue.

* Define OMLibs before Tool subdirectory is imported.

* Define OMLibs at parent scope.

* Specify dependency of MainUtils on OMLibs early.

* Set OMLibs both at current & parent scope.

* Add comment about what future pass implementation should do.
This commit is contained in:
Tian Jin 2020-06-19 00:21:27 +08:00 committed by GitHub
parent 1fc43fa181
commit f81f44662b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 121 additions and 168 deletions

View File

@ -186,6 +186,16 @@ find_mlir_lib(LLVMDemangle)
find_mlir_lib(LLVMFrontendOpenMP)
set(MLIRLibs
${MLIRAffineToStandard}
${MLIRAffineOps}
${MLIRLLVMIR}
${MLIRStandardOps}
${MLIRStandardToLLVM}
${MLIRTransforms}
${MLIRSCFToStandard}
${MLIRVector}
${MLIRSCF}
${MLIRIR}
${MLIRLLVMIR}
${MLIROptLib}
${MLIRParser}
@ -242,95 +252,6 @@ set(MLIRLibs
${CURSES_LIBRARIES}
${ZLIB_LIBRARIES})
# MLIR libraries that must be linked with --whole-archive for static build or
# must be specified on LD_PRELOAD for shared build.
set(MLIRWholeArchiveLibs
MLIRAffineToStandard
MLIRAffineOps
MLIRLLVMIR
MLIRStandardOps
MLIRStandardToLLVM
MLIRTransforms
MLIRSCFToStandard
MLIRVector
MLIRSCF
MLIRIR)
# ONNX MLIR libraries that must be linked with --whole-archive for static build or
# must be specified on LD_PRELOAD for shared build.
set(ONNXMLIRWholeArchiveLibs
OMKrnlToAffine
OMKrnlToLLVM
OMONNXToKrnl
OMONNXRewrite
OMShapeInference
OMShapeInferenceOpInterface
OMAttributePromotion
OMPromotableConstOperandsOpInterface
OMElideConstants
OMElideKrnlGlobalConstants
OMPackKrnlGlobalConstants
OMEnableMemoryPool)
# Function to construct linkage option for the static libraries that must be
# linked with --whole-archive (or equivalent).
function(whole_archive_link target lib_dir)
get_property(link_flags TARGET ${target} PROPERTY LINK_FLAGS)
if("${CMAKE_SYSTEM_NAME}" STREQUAL "Darwin")
set(link_flags "${link_flags} -L${lib_dir} ")
foreach(LIB ${ARGN})
string(CONCAT link_flags ${link_flags}
"-Wl,-force_load, ${lib_dir}/lib${LIB}.a ")
endforeach(LIB)
elseif(MSVC)
foreach(LIB ${ARGN})
string(CONCAT link_flags ${link_flags} "/WHOLEARCHIVE:${lib_dir}/${LIB} ")
endforeach(LIB)
else()
set(link_flags "${link_flags} -L${lib_dir} -Wl,--whole-archive,")
foreach(LIB ${ARGN})
string(CONCAT link_flags ${link_flags} "-l${LIB},")
endforeach(LIB)
string(CONCAT link_flags ${link_flags} "--no-whole-archive")
endif()
set_target_properties(${target} PROPERTIES LINK_FLAGS ${link_flags})
endfunction(whole_archive_link)
# Function to construct LD_PRELOAD value for the shared libraries whose
# static counterpart need --whole-archive linkage option.
function(ld_preload_libs target lib_dir)
foreach(lib ${ARGN})
if("${${lib}}" STREQUAL "")
set(ONNX_MLIR_LD_PRELOAD_${target}
"${ONNX_MLIR_LD_PRELOAD_${target}}:${lib_dir}/lib${lib}.so"
CACHE STRING "" FORCE)
else()
set(ONNX_MLIR_LD_PRELOAD_${target}
"${ONNX_MLIR_LD_PRELOAD_${target}}:${${lib}}"
CACHE STRING "" FORCE)
endif()
endforeach(lib)
endfunction(ld_preload_libs)
function(whole_archive_link_mlir target)
if(BUILD_SHARED_LIBS)
ld_preload_libs(${target} ${LLVM_PROJECT_LIB} ${ARGN})
else()
whole_archive_link(${target} ${LLVM_PROJECT_LIB} ${ARGN})
endif()
endfunction(whole_archive_link_mlir)
function(whole_archive_link_onnx_mlir target)
foreach(lib_target ${ARGN})
add_dependencies(${target} ${lib_target})
endforeach(lib_target)
if(BUILD_SHARED_LIBS)
ld_preload_libs(${target} ${ONNX_MLIR_LIB_DIR} ${ARGN})
else()
whole_archive_link(${target} ${ONNX_MLIR_LIB_DIR} ${ARGN})
endif()
endfunction(whole_archive_link_onnx_mlir)
set(LLVM_CMAKE_DIR
"${LLVM_PROJ_BUILD}/lib/cmake/llvm"
CACHE PATH "Path to LLVM cmake modules")

View File

@ -2,14 +2,40 @@ add_subdirectory(Interface)
add_subdirectory(Dialect)
add_subdirectory(Conversion)
add_subdirectory(Transform)
add_subdirectory(Tool)
add_subdirectory(Builder)
add_subdirectory(Runtime)
# All ONNX-MLIR libraries.
set(OMLibs
OMBuilder
OMKrnlOps
OMONNXOps
OMKrnlToAffine
OMKrnlToLLVM
OMONNXToKrnl
OMONNXRewrite
OMShapeInference
OMShapeInferenceOpInterface
OMAttributePromotion
OMPromotableConstOperandsOpInterface
OMResultTypeInferenceOpInterface
OMElideConstants
OMElideKrnlGlobalConstants
OMPackKrnlGlobalConstants
OMEnableMemoryPool)
set(OMLibs ${OMLibs} PARENT_SCOPE)
message(SATUS "OMLibs" ${OMLibs})
add_subdirectory(Tool)
add_library(MainUtils
MainUtils.hpp
MainUtils.cpp)
target_link_libraries(MainUtils onnx)
target_link_libraries(MainUtils
${OMLibs}
${MLIRLibs}
${CMAKE_DL_LIBS}
onnx)
target_include_directories(MainUtils PRIVATE ${ONNX_MLIR_SRC_ROOT})
target_include_directories(MainUtils PRIVATE ${CMAKE_BINARY_DIR})
@ -29,7 +55,6 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/ExternalUtil.hpp.in
${CMAKE_CURRENT_BINARY_DIR}/ExternalUtil.hpp)
set(ONNX_MLIR_LD_PRELOAD_onnx-mlir "" CACHE STRING "" FORCE)
whole_archive_link_mlir(onnx-mlir ${MLIRWholeArchiveLibs})
if(BUILD_SHARED_LIBS)
message(STATUS "To run dynamically linked onnx-mlir, you must specify:")
message(STATUS "LD_PRELOAD=${ONNX_MLIR_LD_PRELOAD_onnx-mlir}")
@ -43,25 +68,6 @@ endif()
# So it's better not to use target_link_libraries for the add_subdirectory
# targets, or only use it for libraries that have no further dependencies
# (except system libraries such as libc).
target_link_libraries(MainUtils
OMBuilder
OMKrnlOps
OMONNXOps
OMShapeInference
OMShapeInferenceOpInterface
OMAttributePromotion
OMPromotableConstOperandsOpInterface
OMResultTypeInferenceOpInterface
OMElideConstants
OMElideKrnlGlobalConstants
OMPackKrnlGlobalConstants
OMEnableMemoryPool
OMKrnlToAffine
OMKrnlToLLVM
OMONNXToKrnl
OMONNXRewrite
${MLIRLibs}
${CMAKE_DL_LIBS})
add_dependencies(onnx-mlir OMKrnlOpsInc OMONNXOpsInc)
if (INCLUDE_ONNX_ML)

View File

@ -119,6 +119,3 @@ void FrontendToKrnlLoweringPass::runOnOperation() {
std::unique_ptr<Pass> mlir::createLowerToKrnlPass() {
return std::make_unique<FrontendToKrnlLoweringPass>();
}
static PassRegistration<FrontendToKrnlLoweringPass> pass(
"lower-frontend", "Lower frontend ops to Krnl dialect.");

73
src/InitOMPasses.hpp Normal file
View File

@ -0,0 +1,73 @@
#include "mlir/Pass/Pass.h"
#include "src/Pass/Passes.hpp"
namespace onnx_mlir {
void initOMPasses() {
// All passes implemented within onnx-mlir should register within this
// function to make themselves available as a command-line option.
mlir::registerPass("decompose-onnx",
"Decompose ONNX operations into composition of other ONNX operations.",
[]() -> std::unique_ptr<mlir::Pass> {
return mlir::createDecomposeONNXToONNXPass();
});
mlir::registerPass("shape-inference",
"Shape inference for frontend dialects.",
[]() -> std::unique_ptr<mlir::Pass> {
return mlir::createShapeInferencePass();
});
mlir::registerPass("constprop-onnx",
"ConstProp ONNX operations into composition of other ONNX operations.",
[]() -> std::unique_ptr<mlir::Pass> {
return mlir::createConstPropONNXToONNXPass();
});
mlir::registerPass("attribute-promotion",
"Promote constant operands to attributes.",
[]() -> std::unique_ptr<mlir::Pass> {
return mlir::createAttributePromotionPass();
});
mlir::registerPass("elide-constants", "Elide values of constant operations.",
[]() -> std::unique_ptr<mlir::Pass> {
return mlir::createElideConstantValuePass();
});
mlir::registerPass("enable-memory-pool",
"Enable a memory pool for allocating internal MemRefs.",
[]() -> std::unique_ptr<mlir::Pass> {
return mlir::createKrnlEnableMemoryPoolPass();
});
mlir::registerPass(
"lower-krnl", "Lower Krnl dialect.", []() -> std::unique_ptr<mlir::Pass> {
return mlir::createLowerKrnlPass();
});
mlir::registerPass("lower-frontend", "Lower frontend ops to Krnl dialect.",
[]() -> std::unique_ptr<mlir::Pass> {
return mlir::createLowerToKrnlPass();
});
mlir::registerPass("elide-krnl-constants",
"Elide the constant values of the Global Krnl operations.",
[]() -> std::unique_ptr<mlir::Pass> {
return mlir::createElideConstGlobalValuePass();
});
mlir::registerPass("lower-all-llvm",
"Lower the Krnl Affine and Std dialects to LLVM.",
[]() -> std::unique_ptr<mlir::Pass> {
return mlir::createKrnlLowerToLLVMPass();
});
mlir::registerPass("pack-krnl-constants",
"Elide the constant values of the Global Krnl operations.",
[]() -> std::unique_ptr<mlir::Pass> {
return mlir::createPackKrnlGlobalConstantsPass();
});
}
} // namespace onnx_mlir

View File

@ -5,19 +5,15 @@ target_include_directories(onnx-mlir-opt PRIVATE ${ONNX_MLIR_SRC_ROOT})
target_include_directories(onnx-mlir-opt PRIVATE ${ONNX_MLIR_BIN_ROOT})
set(ONNX_MLIR_LD_PRELOAD_onnx-mlir-opt "" CACHE STRING "" FORCE)
whole_archive_link_onnx_mlir(onnx-mlir-opt ${ONNXMLIRWholeArchiveLibs})
whole_archive_link_mlir(onnx-mlir-opt ${MLIRWholeArchiveLibs})
if(BUILD_SHARED_LIBS)
message(STATUS "To run dynamically linked onnx-mlir-opt, you must specify:")
message(STATUS "LD_PRELOAD=${ONNX_MLIR_LD_PRELOAD_onnx-mlir-opt}")
endif()
target_link_libraries(onnx-mlir-opt
OMBuilder
OMKrnlOps
OMONNXOps
OMONNXRewrite
${MLIRLibs})
${OMLibs}
${MLIRLibs}
onnx)
if (INCLUDE_ONNX_ML)
target_link_libraries(onnx-mlir-opt OMMLONNXOps)

View File

@ -22,6 +22,7 @@
#include "src/Dialect/Krnl/KrnlOps.hpp"
#include "src/Dialect/MLONNX/MLONNXOps.hpp"
#include "src/Dialect/ONNX/ONNXOps.hpp"
#include "src/InitOMPasses.hpp"
#include "src/Pass/Passes.hpp"
using namespace onnx_mlir;
@ -70,6 +71,7 @@ int main(int argc, char **argv) {
mlir::registerDialect<mlir::ONNXOpsDialect>();
mlir::registerDialect<mlir::MLONNXOpsDialect>();
mlir::registerDialect<mlir::KrnlOpsDialect>();
initOMPasses();
mlir::registerAsmPrinterCLOptions();
mlir::registerMLIRContextCLOptions();

View File

@ -69,6 +69,3 @@ public:
std::unique_ptr<Pass> mlir::createElideConstGlobalValuePass() {
return std::make_unique<ElideConstGlobalValuePass>();
}
static PassRegistration<ElideConstGlobalValuePass> pass("elide-krnl-constants",
"Elide the constant values of the Global Krnl operations.");

View File

@ -154,6 +154,3 @@ public:
std::unique_ptr<Pass> mlir::createKrnlEnableMemoryPoolPass() {
return std::make_unique<KrnlEnableMemoryPoolPass>();
}
static PassRegistration<KrnlEnableMemoryPoolPass> pass("enable-memory-pool",
"Enable a memory pool for allocating internal MemRefs.");

View File

@ -181,6 +181,3 @@ void KrnlToAffineLoweringPass::runOnFunction() {
std::unique_ptr<Pass> mlir::createLowerKrnlPass() {
return std::make_unique<KrnlToAffineLoweringPass>();
}
static PassRegistration<KrnlToAffineLoweringPass> pass(
"lower-krnl", "Lower Krnl dialect.");

View File

@ -865,6 +865,3 @@ void KrnlToLLVMLoweringPass::runOnOperation() {
std::unique_ptr<mlir::Pass> mlir::createKrnlLowerToLLVMPass() {
return std::make_unique<KrnlToLLVMLoweringPass>();
}
static PassRegistration<KrnlToLLVMLoweringPass> pass(
"lower-all-llvm", "Lower the Krnl Affine and Std dialects to LLVM.");

View File

@ -101,6 +101,3 @@ public:
std::unique_ptr<mlir::Pass> mlir::createAttributePromotionPass() {
return std::make_unique<AttributePromotionPass>();
}
static PassRegistration<AttributePromotionPass> pass(
"attribute-promotion", "Promote constant operands to attributes.");

View File

@ -375,6 +375,3 @@ void ConstPropONNXToONNXPass::runOnFunction() {
std::unique_ptr<mlir::Pass> mlir::createConstPropONNXToONNXPass() {
return std::make_unique<ConstPropONNXToONNXPass>();
}
static PassRegistration<ConstPropONNXToONNXPass> pass("constprop-onnx",
"ConstProp ONNX operations into composition of other ONNX operations.");

View File

@ -61,6 +61,3 @@ void DecomposeONNXToONNXPass::runOnFunction() {
std::unique_ptr<mlir::Pass> mlir::createDecomposeONNXToONNXPass() {
return std::make_unique<DecomposeONNXToONNXPass>();
}
static PassRegistration<DecomposeONNXToONNXPass> pass("decompose-onnx",
"Decompose ONNX operations into composition of other ONNX operations.");

View File

@ -78,6 +78,3 @@ public:
std::unique_ptr<mlir::Pass> mlir::createElideConstantValuePass() {
return std::make_unique<ElideConstantValuePass>();
}
static PassRegistration<ElideConstantValuePass> pass(
"elide-constants", "Elide values of constant operations.");

View File

@ -87,6 +87,3 @@ public:
std::unique_ptr<mlir::Pass> mlir::createShapeInferencePass() {
return std::make_unique<ShapeInferencePass>();
}
static PassRegistration<ShapeInferencePass> pass(
"shape-inference", "Shape inference for frontend dialects.");

View File

@ -1,8 +1,5 @@
import lit.llvm
if '@BUILD_SHARED_LIBS@' == 'ON':
config.environment['LD_PRELOAD'] = r"@ONNX_MLIR_LD_PRELOAD_onnx-mlir-opt@"
config.llvm_tools_dir = r"@MLIR_TOOLS_DIR@"
config.mlir_obj_root = r"@LLVM_PROJ_BUILD@"
config.mlir_tools_dir = r"@MLIR_TOOLS_DIR@"

View File

@ -1,25 +1,13 @@
add_executable(TestConv TestConv.cpp)
target_link_libraries(TestConv
OMBuilder
OMKrnlOps
OMONNXOps
OMShapeInference
OMShapeInferenceOpInterface
OMAttributePromotion
OMPromotableConstOperandsOpInterface
OMElideConstants
OMElideKrnlGlobalConstants
OMKrnlToAffine
OMKrnlToLLVM
OMONNXToKrnl
OMONNXRewrite
${OMLibs}
${MLIRLibs}
${CMAKE_DL_LIBS}
rapidcheck
MainUtils
ExecutionSession
DynMemRefUtils)
whole_archive_link_mlir(TestConv ${MLIRWholeArchiveLibs})
target_include_directories(TestConv
PRIVATE
${ONNX_MLIR_SRC_ROOT}