Support build openssl from local repo (#582)

Type: Code Improvement Signed-off-by: Chen Xin <jack.chen@verisilicon.com> Co-authored-by: Chen Xin <jack.chen@verisilicon.com>
2023-05-05 23:38:39 +08:00 · 2023-05-05 23:38:39 +08:00 · 308a967bcf
parent 1543efe098
commit 308a967bcf
7 changed files with 337 additions and 5 deletions
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@ -15,7 +15,7 @@ option(TIM_VX_ENABLE_PLATFORM           "Enable multi devices support"
 option(TIM_VX_ENABLE_PLATFORM_LITE      "Enable lite multi-device support"      OFF)
 option(TIM_VX_ENABLE_GRPC               "Enable gPRC support"                   OFF)
 option(TIM_VX_DBG_ENABLE_TENSOR_HNDL    "Enable built-in tensor from handle: use malloced memory instead of VideoMemory by kernel driver" ON)
-option(TIM_VX_ENABLE_TENSOR_CACHE       "Enable tensor cache for const tensor"  ON)
+option(TIM_VX_ENABLE_TENSOR_CACHE       "Enable tensor cache for const tensor"  OFF)

 set(CMAKE_CXX_STANDARD 14)
 set(CMAKE_POSITION_INDEPENDENT_CODE ON)
@ -100,7 +100,7 @@ if(TIM_VX_ENABLE_GRPC)
 endif()

 if(TIM_VX_ENABLE_TENSOR_CACHE)
-    find_package(OpenSSL REQUIRED)
+    include(cmake/openssl.cmake)
 endif()
 add_subdirectory("src/tim")

--- a/README.md
+++ b/README.md
@ -84,7 +84,7 @@ cmake options:
 |`VIP_LITE_SDK` | full path to VIPLite sdk, required when `TIM_VX_ENABLE_PLATFORM_LITE`=ON | Not set |
 |`TIM_VX_ENABLE_GRPC` | Enable gPRC support, only work when `TIM_VX_ENABLE_PLATFORM`=ON | OFF |
 |`TIM_VX_DBG_ENABLE_TENSOR_HNDL` | Enable built-in tensor from handle | ON |
-|`TIM_VX_ENABLE_TENSOR_CACHE` | Enable tensor cache for const tensor | ON |
+|`TIM_VX_ENABLE_TENSOR_CACHE` | Enable tensor cache for const tensor, check [OpenSSL build notes](docs/openssl_build.md) | OFF |

 ----
 Run unit test:
--- a/cmake/openssl.cmake
+++ b/cmake/openssl.cmake
@ -0,0 +1,24 @@
+if(${TIM_VX_ENABLE_TENSOR_CACHE})
+
+# set(OPENSSL_CMAKE_URL ON CACHE STRING "https://github.com/viaduck/openssl-cmake")
+# git@gitlab-cn.verisilicon.com:npu_sw/openssl/openssl-cmake.git
+message("If use -DFEATCHCONTENT_SOURCE_DIR_OPENSSL-CMAKE, apply patch ${CMAKE_SOURCE_DIR}/cmake/openssl.patch required")
+
+if(NOT OPENSSL_CMAKE_URL)
+    set(OPENSSL_CMAKE_URL "https://github.com/viaduck/openssl-cmake")
+endif()
+
+message("Using openssl cmake project from ${OPENSSL_CMAKE_URL}")
+include(FetchContent)
+FetchContent_Declare(
+    openssl-cmake
+    GIT_REPOSITORY ${OPENSSL_CMAKE_URL}
+    GIT_TAG 79c122d1606556610477cfae07ff27d8c6e5f260
+    PATCH_COMMAND echo && git reset --hard 79c122d1606556610477cfae07ff27d8c6e5f260 && git apply ${CMAKE_SOURCE_DIR}/cmake/openssl.patch
+    )
+
+set(openssl_force_shared_crt ON CACHE BOOL "" FORCE)
+set(INSTALL_openssl OFF CACHE BOOL "" FORCE)
+
+FetchContent_MakeAvailable(openssl-cmake)
+endif()
--- a/cmake/openssl.patch
+++ b/cmake/openssl.patch
@ -0,0 +1,275 @@
+diff --git a/CMakeLists.txt b/CMakeLists.txt
+index 22ab3cc..00bf90d 100644
+--- a/CMakeLists.txt
+++ b/CMakeLists.txt
+@@ -38,9 +38,15 @@ set(CROSS OFF CACHE BOOL "Cross-compiling?")
+ set(CROSS_TARGET OFF CACHE STRING "Cross-compilation target")
+ set(SYSTEM_OPENSSL OFF CACHE STRING "Use system-provided openssl libraries (instead of prebuilts or building)")
+ 
+option(LOCAL_BUILD "Build in the intranet" OFF)
+set(LOCAL_OPENSSL OFF CACHE STRING "The local openssl resource dir")
+ # allow including our modules
+ list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/cmake")
+ 
+if(NOT OPENSSL_BUILD_VERSION)
+    set(OPENSSL_BUILD_VERSION "1.1.1t")
+endif()
+
+ # mimic system ssl and crypto targets
+ add_library(ssl INTERFACE)
+ add_library(crypto INTERFACE)
+@@ -55,7 +61,7 @@ if (SYSTEM_OPENSSL)
+     add_custom_target(openssl)
+ else()
+     # build our own or use prebuilts
+-    
+
+     # set up fake targets
+     add_library(ssl_lib STATIC IMPORTED GLOBAL)
+     add_library(crypto_lib STATIC IMPORTED GLOBAL)
+@@ -70,16 +76,16 @@ else()
+         set(OPENSSL_PREFIX ${CMAKE_CURRENT_BINARY_DIR}/openssl-prefix/src/openssl)
+     endif()
+ 
+-    set(OPENSSL_LIBSSL_PATH ${OPENSSL_PREFIX}/usr/local/lib/libssl.a)
+-    set(OPENSSL_LIBCRYPTO_PATH ${OPENSSL_PREFIX}/usr/local/lib/libcrypto.a)
+-    
+    set(OPENSSL_LIBSSL_PATH ${OPENSSL_PREFIX}/usr/local/lib/libssl.so)
+    set(OPENSSL_LIBCRYPTO_PATH ${OPENSSL_PREFIX}/usr/local/lib/libcrypto.so)
+
+     # set up openssl target
+     if (BUILD_OPENSSL)
+         include(BuildOpenSSL)
+     else()
+         include(PrebuiltOpenSSL)
+     endif()
+-    
+
+     # set import locations
+     set_target_properties(ssl_lib PROPERTIES IMPORTED_LOCATION ${OPENSSL_LIBSSL_PATH})
+     set_target_properties(crypto_lib PROPERTIES IMPORTED_LOCATION ${OPENSSL_LIBCRYPTO_PATH})
+@@ -91,4 +97,11 @@ else()
+     # add fake targets to common target
+     add_dependencies(ssl_lib openssl)
+     add_dependencies(crypto_lib openssl)
+
+    install(FILES
+        ${OPENSSL_LIBCRYPTO_PATH}.1.1
+        ${OPENSSL_LIBCRYPTO_PATH}
+        ${OPENSSL_LIBSSL_PATH}.1.1
+        ${OPENSSL_LIBSSL_PATH}
+        DESTINATION ${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR})
+ endif()
+diff --git a/cmake/BuildOpenSSL.cmake b/cmake/BuildOpenSSL.cmake
+index e31f4bb..a5c5f72 100644
+--- a/cmake/BuildOpenSSL.cmake
+++ b/cmake/BuildOpenSSL.cmake
+@@ -50,29 +50,25 @@ endif()
+ if ((EXISTS ${OPENSSL_LIBSSL_PATH}) AND (EXISTS ${OPENSSL_LIBCRYPTO_PATH}))
+     message(WARNING "Not building OpenSSL again. Remove ${OPENSSL_LIBSSL_PATH} and ${OPENSSL_LIBCRYPTO_PATH} for rebuild")
+ else()
+-    if (NOT OPENSSL_BUILD_VERSION)
+-        message(FATAL_ERROR "You must specify OPENSSL_BUILD_VERSION!")
+-    endif()
+-
+     if (WIN32 AND NOT CROSS)
+         # yep, windows needs special treatment, but neither cygwin nor msys, since they provide an UNIX-like environment
+-        
+
+         if (MINGW)
+             set(OS "WIN32")
+             message(WARNING "Building on windows is experimental")
+-            
+
+             find_program(MSYS_BASH "bash.exe" PATHS "C:/Msys/" "C:/MinGW/msys/" PATH_SUFFIXES "/1.0/bin/" "/bin/"
+                     DOC "Path to MSYS installation")
+             if (NOT MSYS_BASH)
+                 message(FATAL_ERROR "Specify MSYS installation path")
+             endif(NOT MSYS_BASH)
+-            
+
+             set(MINGW_MAKE ${CMAKE_MAKE_PROGRAM})
+             message(WARNING "Assuming your make program is a sibling of your compiler (resides in same directory)")
+         elseif(NOT (CYGWIN OR MSYS))
+             message(FATAL_ERROR "Unsupported compiler infrastructure")
+         endif(MINGW)
+-        
+
+         set(MAKE_PROGRAM ${CMAKE_MAKE_PROGRAM})
+     elseif(NOT UNIX)
+         message(FATAL_ERROR "Unsupported platform")
+@@ -113,12 +109,12 @@ else()
+     if (OPENSSL_DEBUG_BUILD)
+         set(CONFIGURE_OPENSSL_PARAMS "${CONFIGURE_OPENSSL_PARAMS} no-asm -g3 -O0 -fno-omit-frame-pointer -fno-inline-functions")
+     endif()
+-    
+
+     # set install command depending of choice on man page generation
+     if (OPENSSL_INSTALL_MAN)
+         set(INSTALL_OPENSSL_MAN "install_docs")
+     endif()
+-    
+
+     # disable building tests
+     if (NOT OPENSSL_ENABLE_TESTS)
+         set(CONFIGURE_OPENSSL_MODULES ${CONFIGURE_OPENSSL_MODULES} no-tests)
+@@ -130,19 +126,19 @@ else()
+         set(COMMAND_CONFIGURE ./Configure ${CONFIGURE_OPENSSL_PARAMS} --cross-compile-prefix=${CROSS_PREFIX} ${CROSS_TARGET} ${CONFIGURE_OPENSSL_MODULES} --prefix=/usr/local/)
+         set(COMMAND_TEST "true")
+     elseif(CROSS_ANDROID)
+-        
+
+         # Android specific configuration options
+         set(CONFIGURE_OPENSSL_MODULES ${CONFIGURE_OPENSSL_MODULES} no-hw)
+-                
+
+         # silence warnings about unused arguments (Clang specific)
+         set(CFLAGS "${CMAKE_C_FLAGS} -Qunused-arguments")
+         set(CXXFLAGS "${CMAKE_CXX_FLAGS} -Qunused-arguments")
+-    
+
+         # required environment configuration is already set (by e.g. ndk) so no need to fiddle around with all the OpenSSL options ...
+         if (NOT ANDROID)
+             message(FATAL_ERROR "Use NDK cmake toolchain or cmake android autoconfig")
+         endif()
+-        
+
+         if (ARMEABI_V7A)
+             set(OPENSSL_PLATFORM "arm")
+             set(CONFIGURE_OPENSSL_PARAMS ${CONFIGURE_OPENSSL_PARAMS} "-march=armv7-a")
+@@ -153,42 +149,64 @@ else()
+                 set(OPENSSL_PLATFORM ${CMAKE_ANDROID_ARCH_ABI})
+             endif()
+         endif()
+-                
+
+         # ... but we have to convert all the CMake options to environment variables!
+         set(PATH "${ANDROID_TOOLCHAIN_ROOT}/bin/:${ANDROID_TOOLCHAIN_ROOT}/${ANDROID_TOOLCHAIN_NAME}/bin/")
+         set(LDFLAGS ${CMAKE_MODULE_LINKER_FLAGS})
+-        
+
+         set(COMMAND_CONFIGURE ./Configure android-${OPENSSL_PLATFORM} ${CONFIGURE_OPENSSL_PARAMS} ${CONFIGURE_OPENSSL_MODULES})
+         set(COMMAND_TEST "true")
+     else()                   # detect host system automatically
+         set(COMMAND_CONFIGURE ./config ${CONFIGURE_OPENSSL_PARAMS} ${CONFIGURE_OPENSSL_MODULES})
+-        
+
+         if (NOT COMMAND_TEST)
+             set(COMMAND_TEST ${BUILD_ENV_TOOL} <SOURCE_DIR> ${MAKE_PROGRAM} test)
+         endif()
+     endif()
+-    
+
+     # add openssl target
+-    ExternalProject_Add(openssl
+-        URL https://mirror.viaduck.org/openssl/openssl-${OPENSSL_BUILD_VERSION}.tar.gz
+-        ${OPENSSL_CHECK_HASH}
+-        UPDATE_COMMAND ""
+    if(LOCAL_BUILD)
+        ExternalProject_Add(openssl
+            SOURCE_DIR ${LOCAL_OPENSSL}
+            UPDATE_COMMAND ${GIT_EXECUTABLE} clean -dfx -e buildenv.txt && ${GIT_EXECUTABLE} stash && ${GIT_EXECUTABLE} checkout OpenSSL_1_1_1t
+ 
+-        CONFIGURE_COMMAND ${BUILD_ENV_TOOL} <SOURCE_DIR> ${COMMAND_CONFIGURE}
+-        PATCH_COMMAND ${PATCH_PROGRAM} -p1 --forward -r - < ${CMAKE_CURRENT_SOURCE_DIR}/patches/0001-Fix-failing-cms-test-when-no-des-is-used.patch || echo
+            CONFIGURE_COMMAND ${BUILD_ENV_TOOL} <SOURCE_DIR> ${COMMAND_CONFIGURE}
+            PATCH_COMMAND ${PATCH_PROGRAM} -p1 --forward -r - < ${CMAKE_CURRENT_SOURCE_DIR}/patches/0001-Fix-failing-cms-test-when-no-des-is-used.patch || echo
+ 
+-        BUILD_COMMAND ${BUILD_ENV_TOOL} <SOURCE_DIR> ${MAKE_PROGRAM} -j ${NUM_JOBS}
+-        BUILD_BYPRODUCTS ${OPENSSL_LIBSSL_PATH} ${OPENSSL_LIBCRYPTO_PATH}
+            BUILD_COMMAND ${BUILD_ENV_TOOL} <SOURCE_DIR> ${MAKE_PROGRAM} -j ${NUM_JOBS}
+            BUILD_BYPRODUCTS ${OPENSSL_LIBSSL_PATH} ${OPENSSL_LIBCRYPTO_PATH}
+ 
+-        TEST_BEFORE_INSTALL 1
+-        TEST_COMMAND ${COMMAND_TEST}
+            TEST_BEFORE_INSTALL 0
+            TEST_COMMAND ${COMMAND_TEST}
+ 
+-        INSTALL_COMMAND ${BUILD_ENV_TOOL} <SOURCE_DIR> ${PERL_PATH_FIX_INSTALL}
+-        COMMAND ${BUILD_ENV_TOOL} <SOURCE_DIR> ${MAKE_PROGRAM} DESTDIR=${CMAKE_CURRENT_BINARY_DIR} install_sw ${INSTALL_OPENSSL_MAN}
+-        COMMAND ${CMAKE_COMMAND} -G ${CMAKE_GENERATOR} ${CMAKE_BINARY_DIR}                    # force CMake-reload
+            INSTALL_COMMAND ${BUILD_ENV_TOOL} <SOURCE_DIR> ${PERL_PATH_FIX_INSTALL}
+            COMMAND ${BUILD_ENV_TOOL} <SOURCE_DIR> ${MAKE_PROGRAM} DESTDIR=${CMAKE_CURRENT_BINARY_DIR} install_sw ${INSTALL_OPENSSL_MAN}
+            COMMAND ${CMAKE_COMMAND} -G ${CMAKE_GENERATOR} ${CMAKE_BINARY_DIR}                    # force CMake-reload
+ 
+-        LOG_INSTALL 1
+-    )
+            LOG_INSTALL 1
+        )
+    else()
+        ExternalProject_Add(openssl
+            URL https://mirror.viaduck.org/openssl/openssl-${OPENSSL_BUILD_VERSION}.tar.gz
+            ${OPENSSL_CHECK_HASH}
+            UPDATE_COMMAND ""
+
+            CONFIGURE_COMMAND ${BUILD_ENV_TOOL} <SOURCE_DIR> ${COMMAND_CONFIGURE}
+            PATCH_COMMAND ${PATCH_PROGRAM} -p1 --forward -r - < ${CMAKE_CURRENT_SOURCE_DIR}/patches/0001-Fix-failing-cms-test-when-no-des-is-used.patch || echo
+
+            BUILD_COMMAND ${BUILD_ENV_TOOL} <SOURCE_DIR> ${MAKE_PROGRAM} -j ${NUM_JOBS}
+            BUILD_BYPRODUCTS ${OPENSSL_LIBSSL_PATH} ${OPENSSL_LIBCRYPTO_PATH}
+
+            TEST_BEFORE_INSTALL 1
+            TEST_COMMAND ${COMMAND_TEST}
+
+            INSTALL_COMMAND ${BUILD_ENV_TOOL} <SOURCE_DIR> ${PERL_PATH_FIX_INSTALL}
+            COMMAND ${BUILD_ENV_TOOL} <SOURCE_DIR> ${MAKE_PROGRAM} DESTDIR=${CMAKE_CURRENT_BINARY_DIR} install_sw ${INSTALL_OPENSSL_MAN}
+            COMMAND ${CMAKE_COMMAND} -G ${CMAKE_GENERATOR} ${CMAKE_BINARY_DIR}                    # force CMake-reload
+
+            LOG_INSTALL 1
+        )
+    endif()
+ 
+     # set git config values to openssl requirements (no impact on linux though)
+     ExternalProject_Add_Step(openssl setGitConfig
+@@ -230,11 +248,11 @@ else()
+     # write environment to file, is picked up by python script
+     get_cmake_property(_variableNames VARIABLES)
+     foreach (_variableName ${_variableNames})
+-        if (NOT _variableName MATCHES "lines")
+        if ((NOT _variableName MATCHES "lines") OR (NOT _variableName MATCHES "CMAKE_CXX_COMPILER_ID_PLATFORM_CONTENT"))
+             set(OUT_FILE "${OUT_FILE}${_variableName}=\"${${_variableName}}\"\n")
+         endif()
+     endforeach()
+-    file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/buildenv.txt ${OUT_FILE})
+    file(WRITE ${LOCAL_OPENSSL}/buildenv.txt ${OUT_FILE})
+ 
+     set_target_properties(ssl_lib PROPERTIES IMPORTED_LOCATION ${OPENSSL_LIBSSL_PATH})
+     set_target_properties(crypto_lib PROPERTIES IMPORTED_LOCATION ${OPENSSL_LIBCRYPTO_PATH})
+diff --git a/scripts/building_env.py b/scripts/building_env.py
+index aab2f42..d4a0fe0 100644
+--- a/scripts/building_env.py
+++ b/scripts/building_env.py
+@@ -58,22 +58,25 @@ l.extend(argv[offset+1:])                             # routed commands
+ l[0] = '"'+l[0]+'"'
+ 
+ # ensure target dir exists for mingw cross
+-target_dir = binary_openssl_dir_source+"/../../../usr/local/bin"
+-if not os.path.exists(target_dir):
+-    os.makedirs(target_dir)
+# target_dir = binary_openssl_dir_source+"/../../../usr/local/bin"
+# if not os.path.exists(target_dir):
+#     os.makedirs(target_dir)
+ 
+ # read environment from file if cross-compiling
+ if os_s == "LINUX_CROSS_ANDROID":
+     expr = re.compile('^(.*?)="(.*?)"', re.MULTILINE | re.DOTALL)
+-    f = open(binary_openssl_dir_source+"/../../../buildenv.txt", "r")
+    f = open(binary_openssl_dir_source+"/buildenv.txt", "r")
+     content = f.read()
+     f.close()
+ 
+     for k, v in expr.findall(content):
+-        if k != "PATH":
+-            env[k] = v.replace('"', '')
+-        else:
+-            env[k] = v.replace('"', '')+":"+env[k]
+        try:
+            if k != "PATH":
+                env[k] = v.replace('"', '')
+            else:
+                env[k] = v.replace('"', '')+":"+env[k]
+        except:
+            pass
+ 
+ proc = None
+ if os_s == "WIN32":
--- a/docs/openssl_build.md
+++ b/docs/openssl_build.md
@ -0,0 +1,33 @@
+OpenSSL doesn't have cmake project by default, so we use [ssl.cmake](https://github.com/viaduck/openssl-cmake) to integrate OpenSSL build with cmake.
+
+# Build openssl from downloaded source archive
+```bash
+mkdir build && cd build
+cmake .. -DTIM_VX_ENABLE_TENSOR_CACHE=ON -DBUILD_OPENSSL=ON
+```
+
+# Build openssl with local source repo
+```bash
+git clone git@github.com:openssl/openssl.git ${local_ssl_repo}
+# checkout to 1.1.1t or other release tag
+# build tim-vx with local ssl source repo
+mkdir build && cd build
+cmake .. -DTIM_VX_ENABLE_TENSOR_CACHE=ON -DBUILD_OPENSSL=ON -DLOCAL_BUILD=ON -DLOCAL_OPENSSL=${local_ssl_repo}
+```
+
+# Build openssl with android-ndk
+```bash
+cmake .. -DTIM_VX_ENABLE_TENSOR_CACHE=ON -DBUILD_OPENSSL=ON -DLOCAL_BUILD=ON -DLOCAL_OPENSSL=${local_ssl_repo} -DCMAKE_TOOLCHAIN_FILE=${ndk_root}/build/cmake/android.toolchain.cmake -DANDROID_ABI=arm64-v8a -DCROSS_ANDROID=ON -DEXTERNAL_VIV_SDK=${ANDROID_OVX_DRV_PREBUILD}
+# change ANDROID_ABI accroding to target platform
+```
+
+# Build openssl with mirrored ssl.cmake
+If [ssl.cmake](https://github.com/viaduck/openssl-cmake) not always accessable from your originization, you can mirror it and replace the download url by append **-DOPENSSL_CMAKE_URL=<ssl.cmake mirror>** to cmake command line.
+
+# Deployment with OpenSSL libraries
+All libraries can be installed in single place by
+```bash
+make install # will install all required library(libtim-vx, libssl, libcrypto) to <build>/install/lib/
+```
+
+# TODO: General cross build with openssl not support yet
--- a/src/tim/CMakeLists.txt
+++ b/src/tim/CMakeLists.txt
@ -142,7 +142,8 @@ target_link_libraries(${TARGET_NAME} PUBLIC
    -Wl,--no-whole-archive  ${OVXDRV_LIBRARIES} ${EXTERNAL_LIBS})

 if(${TIM_VX_ENABLE_TENSOR_CACHE})
-    target_link_libraries(${TARGET_NAME} PUBLIC ${OPENSSL_CRYPTO_LIBRARY})
+    add_dependencies(${TARGET_NAME} crypto)
+    target_link_libraries(${TARGET_NAME} PUBLIC crypto)
 endif()

 if(${TIM_VX_USE_EXTERNAL_OVXLIB})
--- a/src/tim/vx/graph.cc
+++ b/src/tim/vx/graph.cc
@ -74,7 +74,6 @@ const std::string GraphImpl::caclulateMd5Secret32(const std::string& src) {
  unsigned char md_value[MD5_SECRET_LEN_16] = {0};
  char tmp[MD5_BYTE_STRING_LEN] = {0};

-  OpenSSL_add_all_digests();
  md = EVP_md5();
  if (md == NULL) {
    VSILOGE("Unknown EVP_md5 message.");