Loading python imports from dlopen-loaded C++ library that uses pybind11.

[jcelerier]

I have the following case :

• an app which does not have any dependency on pybind11, python or whatever.
• a plug-in for this app loaded at run-time with dlopen. This plug-in uses pybind11.

So, basically :

[ app ]  --loads--> [ plugin ] --executes--> [ stuff.py ]

When I want to execute some python code that uses scikit_learn from this plug-in, I get the following error:

terminate called after throwing an instance of 'pybind11::error_already_set'
  what():  ImportError: /usr/lib/python3.6/site-packages/sklearn/__check_build/_check_build.cpython-36m-x86_64-linux-gnu.so: undefined symbol: _Py_NoneStruct
___________________________________________________________________________
Contents of /usr/lib/python3.6/site-packages/sklearn/__check_build:
setup.py                  __init__.py               __pycache__
_check_build.cpython-36m-x86_64-linux-gnu.so
___________________________________________________________________________
It seems that scikit-learn has not been built correctly.

If you have installed scikit-learn from source, please do not forget
to build the package before using it: run `python setup.py install` or
`make` in the source directory.

If you have used an installer, please check that it is suited for your
Python version, your operating system and your platform.

At:
  /usr/lib/python3.6/site-packages/sklearn/__check_build/__init__.py(41): raise_build_error
  /usr/lib/python3.6/site-packages/sklearn/__check_build/__init__.py(46): <module>
  <frozen importlib._bootstrap>(219): _call_with_frames_removed
  <frozen importlib._bootstrap_external>(678): exec_module
  <frozen importlib._bootstrap>(665): _load_unlocked
  <frozen importlib._bootstrap>(955): _find_and_load_unlocked
  <frozen importlib._bootstrap>(971): _find_and_load
  <frozen importlib._bootstrap>(219): _call_with_frames_removed
  <frozen importlib._bootstrap>(1024): _handle_fromlist
  /usr/lib/python3.6/site-packages/sklearn/__init__.py(56): <module>
  <frozen importlib._bootstrap>(219): _call_with_frames_removed
  <frozen importlib._bootstrap_external>(678): exec_module
  <frozen importlib._bootstrap>(665): _load_unlocked
  <frozen importlib._bootstrap>(955): _find_and_load_unlocked
  <frozen importlib._bootstrap>(971): _find_and_load
  <string>(2): <module>

However, this works if I load the python code from my "main" executable instead.

Here's a minimal example :

CMakeLists.txt:

set(CMAKE_BUILD_TYPE Debug)
cmake_minimum_required(VERSION 3.1)
project(blah)
add_subdirectory(pybind11)

add_executable(main main.cpp)
add_library(blah SHARED lib.cpp)

target_link_libraries(main dl)
target_link_libraries(blah pybind11::embed)

main.cpp:

#include <dlfcn.h>
int main()
{
  auto dll = dlopen("libblah.so", RTLD_NOW);
  typedef void (*fun_type)();
  auto fun = reinterpret_cast<fun_type>(dlsym(dll, "run"));
  fun();
}

lib.cpp:

#include <pybind11/embed.h>

extern "C" void run()
{
  pybind11::scoped_interpreter guard{};
  pybind11::exec("import sklearn");
}

running:

$ cmake  --build .
$ LD_LIBRARY_PATH=. ./main

[AraHaan]

It seems like my issue where I have to freeze my import hook while my main program does this:

/*
 * komextract_new.c
 *
 * Entrypoint of the embeded python interpreter for
 * running code to extract from kom files.
 *
 * Note: This code is for WIN32/WIN64 only.
 */
#include <Python.h>
/* for loading python script from the resource
 * section.
 */
#include <Windows.h>
#define WITH_ENCRYPTION
#ifdef WITH_ENCRYPTION
#include "encryption.h"
#include "frozenlist.h"
#endif
#include "resource.h"

#ifdef WITH_ENCRYPTION
PyMODINIT_FUNC PyInit_aes(void);
PyMODINIT_FUNC PyInit__zipimport(void);
#endif

int
wmain(int argc, wchar_t **argv)
{
  int err, err2;
  wchar_t *program = Py_DecodeLocale((char *)argv[0], NULL);
  wchar_t **argv_copy = argv;
  if (program == NULL) {
    LPSTR buffer1[36];
    LoadStringA(GetModuleHandle(NULL), IDS_STRING1, (LPSTR)buffer1, 36);
    fprintf(stderr, (const char *const)buffer1);
    exit(1);
  }
#ifdef WITH_ENCRYPTION
  PyImport_AppendInittab("aes", PyInit_aes);
  PyImport_AppendInittab("_zipimport", PyInit__zipimport);
#endif
  Py_SetProgramName(program);  /* optional but recommended */
  Py_Initialize();
  if (Py_IsInitialized() != 0) {
    /* allows use of sys.argv to be possible
     * with no tracebacks.
     */
    argv_copy[0] = L"";
    PySys_SetArgvEx(argc, argv_copy, 0);
#ifdef WITH_ENCRYPTION
    HRSRC script2_resource = FindResource(
      NULL, MAKEINTRESOURCE(IDR_RCDATA2), RT_RCDATA);
    unsigned int script2_size = SizeofResource(NULL, script2_resource);
    HGLOBAL main2_script = LoadResource(NULL, script2_resource);
    void* pmain2_script = LockResource(main2_script);
    if (script2_size >= 0) {
      err = PyEncryptionExec((const char *)pmain2_script);
    } else {
      /* python script is empty. */
      LPSTR buffer4[62];
      LoadStringA(GetModuleHandle(NULL), IDS_STRING4, (LPSTR)buffer4, 62);
      fprintf(stderr, (const char *const)buffer4);
    }
    if (err > 0)
      PyErr_Print();
#endif
    HRSRC script_resource = FindResource(
      NULL, MAKEINTRESOURCE(IDR_RCDATA1), RT_RCDATA);
    unsigned int script_size = SizeofResource(NULL, script_resource);
    HGLOBAL main_script = LoadResource(NULL, script_resource);
    void* pmain_script = LockResource(main_script);
    if (script_size >= 0) {
      err2 = PyRun_SimpleString((const char *)pmain_script);
    } else {
      /* python script is empty. */
      LPSTR buffer2[38];
      LoadStringA(GetModuleHandle(NULL), IDS_STRING2, (LPSTR)buffer2, 38);
      fprintf(stderr, (const char *const)buffer2);
    }
    if (err2 > 0)
      PyErr_Print();
    Py_Finalize();
  } else {
    /* python is not initialized. */
    LPSTR buffer3[41];
    LoadStringA(GetModuleHandle(NULL), IDS_STRING3, (LPSTR)buffer3, 41);
    fprintf(stderr, (const char *const)buffer3);
  }
  PyMem_RawFree(program);
  return 0;
}

and frozenlist.h with this code:

#pragma once
#include <importlib.h>
#include <importlib_external.h>
#include "pyeimport.h"

static const struct _frozen _PyImport_FrozenModules[] = {
    /* importlib */
    {"_frozen_importlib", _Py_M__importlib, (int)sizeof(_Py_M__importlib)},
    {"_frozen_importlib_external", _Py_M__importlib_external,
        (int)sizeof(_Py_M__importlib_external)},
    /* pye import hook. */
    {"pyeimport", M_pyeimport, (int)sizeof(M_pyeimport)},
    {0, 0, 0} /* sentinel */
};

const struct _frozen * PyImport_FrozenModules = _PyImport_FrozenModules;

And basically when I compile with frozenlist.h the following happens and I am not sure why, I tried everything too.

<path trimmed>\bin\x86\Release>kompact_new
Traceback (most recent call last):
  File "<string>", line 1, in <module>
ModuleNotFoundError: No module named 'pyeimport'
Traceback (most recent call last):
  File "<string>", line 7, in <module>
ModuleNotFoundError: No module named 'komformat'

What is going on? I told python that pyeimport is an frozen module though...

[Tabrizian]

I’m interested in the same feature. Is there any way to do this in pybind11?