koba-jon / pytorch_cpp

Deep Learning sample programs using PyTorch in C++
MIT License
245 stars 52 forks source link

Statically building pix2pix #18

Open fork-while-1 opened 2 years ago

fork-while-1 commented 2 years ago

Hi! I want to be able to run pix2pix in SGX, but SGX only supports statically linked programs. So I was wondering if you think it would be possible to build pix2pix statically? I would need the *.a files instead of the *.so files for each of the dependencies, but where do I make the changes in the cmake files for this? Thank you so much for your help!

koba-jon commented 2 years ago

It may be possible.

In my environment, I was able to find dynamic libraries corresponding to the execution file by running the following command:

pytorch_cpp/static_library/Image-to-Image_Translation/pix2pix$ ldd pix2pix

    linux-vdso.so.1 (0x00007fff626a2000)
    libc10.so => /home/koba/libtorch/lib/libc10.so (0x00007fd4905aa000)
    libboost_program_options.so.1.71.0 => /lib/x86_64-linux-gnu/libboost_program_options.so.1.71.0 (0x00007fd4904dc000)
    libpng16.so.16 => /lib/x86_64-linux-gnu/libpng16.so.16 (0x00007fd4904a4000)
    libtorch_cpu.so => /home/koba/libtorch/lib/libtorch_cpu.so (0x00007fd479853000)
    libpthread.so.0 => /lib/x86_64-linux-gnu/libpthread.so.0 (0x00007fd479830000)
    libtorch_cuda.so => /home/koba/libtorch/lib/libtorch_cuda.so (0x00007fd431028000)
    libcudart.so.10.2 => /usr/local/cuda/lib64/libcudart.so.10.2 (0x00007fd430daa000)
    libnvToolsExt.so.1 => /usr/local/cuda/lib64/libnvToolsExt.so.1 (0x00007fd430ba1000)
    libtorch.so => /home/koba/libtorch/lib/libtorch.so (0x00007fd43090e000)
    libopencv_imgcodecs.so.3.3 => /usr/local/lib/libopencv_imgcodecs.so.3.3 (0x00007fd43056a000)
    libopencv_imgproc.so.3.3 => /usr/local/lib/libopencv_imgproc.so.3.3 (0x00007fd42ca4a000)
    libopencv_core.so.3.3 => /usr/local/lib/libopencv_core.so.3.3 (0x00007fd42b8cb000)
    libstdc++.so.6 => /lib/x86_64-linux-gnu/libstdc++.so.6 (0x00007fd42b6e9000)
    libm.so.6 => /lib/x86_64-linux-gnu/libm.so.6 (0x00007fd42b59a000)
    libgomp.so.1 => /lib/x86_64-linux-gnu/libgomp.so.1 (0x00007fd42b558000)
    libgcc_s.so.1 => /lib/x86_64-linux-gnu/libgcc_s.so.1 (0x00007fd42b53d000)
    libc.so.6 => /lib/x86_64-linux-gnu/libc.so.6 (0x00007fd42b34b000)
    libgomp-52f2fd74.so.1 => /home/koba/libtorch/lib/libgomp-52f2fd74.so.1 (0x00007fd42b116000)
    /lib64/ld-linux-x86-64.so.2 (0x00007fd4908f9000)
    libz.so.1 => /lib/x86_64-linux-gnu/libz.so.1 (0x00007fd42b0fa000)
    librt.so.1 => /lib/x86_64-linux-gnu/librt.so.1 (0x00007fd42b0ef000)
    libdl.so.2 => /lib/x86_64-linux-gnu/libdl.so.2 (0x00007fd42b0e9000)
    libcudart-80664282.so.10.2 => /home/koba/libtorch/lib/libcudart-80664282.so.10.2 (0x00007fd42ae68000)
    libc10_cuda.so => /home/koba/libtorch/lib/libc10_cuda.so (0x00007fd42ac01000)
    libnvToolsExt-3965bdd0.so.1 => /home/koba/libtorch/lib/libnvToolsExt-3965bdd0.so.1 (0x00007fd42a9f7000)
    libtbb.so.2 => /lib/x86_64-linux-gnu/libtbb.so.2 (0x00007fd42a9b1000)
    libjpeg.so.8 => /lib/x86_64-linux-gnu/libjpeg.so.8 (0x00007fd42a92c000)

The following is an example of replacing /lib/x86_64-linux-gnu/libboost_program_options.so.1.71.0 of libboost_program_options.so.1.71.0 => /lib/x86_64-linux-gnu/libboost_program_options.so.1.71.0 (0x00007fd4904dc000) with a static library.

$ ls /lib/x86_64-linux-gnu/libboost_program_options.*

/lib/x86_64-linux-gnu/libboost_program_options.a
/lib/x86_64-linux-gnu/libboost_program_options.so
/lib/x86_64-linux-gnu/libboost_program_options.so.1.71.0

In this case, .a is a static library. Therefore, I added the following command to the utils/CMakeLists.txt.

Please notice ############### Deleted ############### and ############### Addition ###############

cmake_minimum_required(VERSION 3.0 FATAL_ERROR)

# LibTorch
set(LIBTORCH_DIR $ENV{HOME}/libtorch)
list(APPEND CMAKE_PREFIX_PATH ${LIBTORCH_DIR})

# Set Compiler Options
set(CXX_LIBRARIES "stdc++fs")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O2 -Wall")
if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 8)
    set(CMAKE_CXX_COMPILER "g++-8")
endif ()

# Directory Name
set(UTILS_DIR ${CMAKE_CURRENT_SOURCE_DIR})

# Find Package
find_package(Torch REQUIRED)
find_package(OpenCV REQUIRED)
find_package(OpenMP REQUIRED)
find_package(Boost REQUIRED COMPONENTS program_options)
find_package(PNG REQUIRED)

# For OpenMP
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")

# Set Include Directories
set(INCLUDE_DIRS
    ${TORCH_INCLUDE_DIRS}
    ${OpenCV_INCLUDE_DIRS}
    ${Boost_INCLUDE_DIRS}
    ${PNG_INCLUDE_DIRS}
    ${UTILS_DIR}
    ${SRC_DIR}
)

# Set Libraries
set(LIBRARIES
    ${CXX_LIBRARIES}
    ${TORCH_LIBRARIES}
    ${OpenCV_LIBRARIES}
    # ${Boost_LIBRARIES} ############### Deleted ###############
    ${PNG_LIBRARIES}
)

############### Addition (Start) ###############

add_library(boost_program_options_static 
    STATIC
    IMPORTED
)

set_target_properties(boost_program_options_static
    PROPERTIES
    IMPORTED_LOCATION /usr/lib/x86_64-linux-gnu/libboost_program_options.a
)

############### Addition (End) ###############

# Set Utility Code
set(UTILITY
    ${UTILS_DIR}/transforms.cpp
    ${UTILS_DIR}/datasets.cpp
    ${UTILS_DIR}/dataloader.cpp
    ${UTILS_DIR}/losses.cpp
    ${UTILS_DIR}/visualizer.cpp
    ${UTILS_DIR}/progress.cpp
)

# Link
add_executable(${PROJECT_NAME} ${SRCS} ${UTILITY})
include_directories(${INCLUDE_DIRS})
# target_link_libraries(${PROJECT_NAME} ${LIBRARIES}) ############### Deleted ###############
target_link_libraries(${PROJECT_NAME} ${LIBRARIES} boost_program_options_static) ############### Addition ###############
set_property(TARGET ${PROJECT_NAME} PROPERTY CXX_STANDARD 17)

# Display Message

message(STATUS "")
message(STATUS "~~~ PyTorch Information ~~~")
message(STATUS "${TORCH_INCLUDE_DIRS};")
message(STATUS "${TORCH_LIBRARIES};")
message(STATUS "~~~~~~~~~~~~~~~~~~~~~~~~~~~")
message(STATUS "")

message(STATUS "~~~ OpenCV Information ~~~")
message(STATUS "${OpenCV_INCLUDE_DIRS};")
message(STATUS "${OpenCV_LIBRARIES};")
message(STATUS "~~~~~~~~~~~~~~~~~~~~~~~~~~")
message(STATUS "")

message(STATUS "~~~ Boost Information ~~~")
message(STATUS "${Boost_INCLUDE_DIRS};")
message(STATUS "${Boost_LIBRARIES};")
message(STATUS "~~~~~~~~~~~~~~~~~~~~~~~~~~")
message(STATUS "")

message(STATUS "~~~ libpng Information ~~~")
message(STATUS "${PNG_INCLUDE_DIRS};")
message(STATUS "${PNG_LIBRARIES};")
message(STATUS "~~~~~~~~~~~~~~~~~~~~~~~~~~")
message(STATUS "")

This requires adding text for each dynamic library, and it's a little troublesome, but I'm sure it can be done.