Python wrapping a VTK module file from outside VTK. Issue with supermodule

Support
1

I am python wrapping a vtk class using the new way of doing it (not vtk_wrap_python3) as follows -

vtk_module_wrap_python(
MODULES VTK::ModuleName
SOABI “{Python{VTK_PYTHON_VERSION}_SOABI}”
…
)

The class derives from vtkObject. It builds fine, but we end up with an undefined symbol for: PyvtkObject_ClassNew

This is because we are unable to find the vtkObject superclass in the hierarchy and attempt to new the object as follows in vtkWrapPythonClass.c

vtkWrapPython_GenerateObjectNew(
…
if (!supermodule) /* superclass is in the same module */
{
fprintf(fp, " pytype->tp_base = (PyTypeObject )Py%s_ClassNew();\n\n", superclassname);
}
else / superclass is in a different module */
{
fprintf(
fp, " pytype->tp_base = vtkPythonUtil::FindClassTypeObject("%s");\n\n", superclassname);
}
…

How do I tell the hierarchy that this should be treated as a supermodule ?

If I manually replace the PyvtkObject_ClassNew in the wrapped cxx with vtkPythonUtil::FindClassTypeObject, all is good.

What’s the way to fix this ? May we inject to some comment devilry in the superclass (like long ago) ?

Thanks

Karthik

2

Hi Karthik, good to hear from you.

In VTK 9, the hierarchy.txt files are even more important to the wrapping than they were in VTK 6, 7 and 8. So if an external module isn’t finding VTK’s hierarchy.txt files, then build errors occur, as you have seen here.

Tweaking or hinting the code so that it can handle certain situations (like deriving directly from vtkObject) even when the hierarchy files are not found is not the best solution (obviously!). To properly solve the build issue, we have to find out why cmake isn’t handling the hierarchy.txt files while it is wrapping your module.

Can you try the VTK “Wrapping” example to make sure it works on your system? It can be found in Examples/Modules/Wrapping. When this example module builds, it should create it’s own heirarchy.txt file that includes information about the classes in all the VTK modules it depends on (including vtkObject, of course). The hierarchy file will be in the lib/vtk/hierarchy/Wrapping directory:

lib/vtk/hierarchy/Wrapping/vtkWrappable-hierarchy.txt

So please build this example module, and confirm that it creates a hierarchy file, and that the hierarchy file contains an entry for vtkObject:

vtkObject : vtkObjectBase ; vtkObject.h ; vtkCommonCore

Once you have confirmed that the example works, you can compare its cmake files to yours, the most important files in the example to look at are:

CMakeLists.txt
module/vtk.module
module/CMakeLists.txt

For your own project, the module directory will not be called “module” (I think we should have chosen a different directory name for the example, too).

Also, feel free to post your CMakeLists.txt files and your vtk.module files, we can check them over to make sure they call the module macros correctly.

3

Indeed this fixed it. Thanks for the pointer on the wrap hierarchy.

4

Thanks David for your earlier response.

I have this issue with VTK-8.2.0 now. (With VTK 9.0.3 all is fine with the vtk_module_scan and vtk_module_wrap_python.)

But VTK 8.2.0 still errors me out

ImportError: …/module/libvtkModulePythonD.so: undefined symbol: PyvtkObject_ClassNew

The symbol clearly exists in libvtkCommonCorePython38D-8.2.so

include(${VTK_USE_FILE})
include(vtkWrapPython)
include_directories(${PYTHON_INCLUDE_PATH})
include_directories( ${PYTHON_INCLUDE_DIR}
${CMAKE_CURRENT_SOURCE_DIR}
${CMAKE_CURRENT_SOURCE_DIR}/cpp
)
set(modules "Ns::Module")
set(python_destination "lib/python3.8/site-packages")
set(vtkModule_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/cpp/vtkModuleApplicaiton.cxx )

add_library(vtkModule ${vtkModule_SRCS})
target_link_libraries(vtkModule ${VTK_LIBRARIES} ${vtkPython_LIBRARIES})
VTK_WRAP_PYTHON3(vtkModulePython vtkModulePYTHON_SRCS "${vtkModule_SRCS}")
ADD_LIBRARY(vtkModulePythonD ${vtkModulePYTHON_SRCS} )
ADD_LIBRARY(vtkModulePython MODULE ${vtkModule_SRCS} vtkModulePythonInit.cxx vtkModulePythonInitImpl.cxx)
target_link_libraries(vtkModulePythonD ${VTK_LIBRARIES} ${vtkPython_LIBRARIES} vtkModule)
target_link_libraries(vtkModulePython ${VTK_LIBRARIES} ${vtkPython_LIBRARIES} vtkModule vtkModulePythonD)

set(MODULE_HIERARCHY_NAME vtkModuleHierarchy)
set(vtkModule_WRAP_DEPENDS ${vtkModule_LIBS})
include(vtkWrapHierarchy)
vtk_wrap_hierarchy(vtkModule ${CMAKE_CURRENT_BINARY_DIR} "${vtkModule_SRCS}")
set(KIT_HIERARCHY_FILE ${CMAKE_CURRENT_BINARY_DIR}/${MODULE_HIERARCHY_NAME}.txt)
set(LIB_HIERARCHY_STAMP ${CMAKE_CURRENT_BINARY_DIR}/${MODULE_HIERARCHY_NAME}.stamp.txt)


file(WRITE "${CMAKE_BINARY_DIR}/${python_destination}/module/__init__.py" 
 "from __future__ import absolute_import

try:
    # use relative import for installed modules
    from .libvtkModulePython import *
except ImportError:
    # during build and testing, the modules will be elsewhere,
    # e.g. in lib directory or Release/Debug config directories
    from libvtkModulePython import *")
5

Hi Karthik, can you edit your post and add backtick-quoting so that the cmake text is readable? For example,

Markup text:

```cmake
set(MODULE_HIERARCHY_NAME vtkModuleHierarchy)
```

Displayed result:

set(MODULE_HIERARCHY_NAME vtkModuleHierarchy)
6

Done. Apologies

7

You can look at vtkObjectPython.cxx to double-check the symbol visibility. In VTK 8.2 it should be declared VTK_ABI_EXPORT (not in VTK 9, though, since VTK 9 deals with module dependencies in a different way).

extern "C" { VTK_ABI_EXPORT PyObject *PyvtkObject_ClassNew(); }

Please also check that cmake is linking libvtkCommonCorePythonD. You can add a diagnostic line to your cmake code:

message(STATUS "vtkPython_LIBRARIES = ${vtkPython_LIBRARIES}")

Also double-check by running ldd on the Python module

ldd libvtkModulePythonD.so

or do a verbose build to see what linking is being done

VERBOSE=1 make

or, for ninja,

ninja -v
8

Dear David

I have created a minimal test here : TestWrapPythonVTK820.tar.gz (2.2 KB)

Running this as

cmake -DVTK_DIR=/path/to/VTK8.2.0Build/
declare -x LD_LIBRARY_PATH="<CWD>/lib/python3.8/site-packages/:...."
declare -x PYTHONPATH="<CWD>/lib/python3.8/site-packages/:...."
vtkpython -c "import evw"

gives the following error:

Traceback (most recent call last):
  File "/data2/work/sandbox/testwrapvtk820/TestWrapPythonVTK820Build/lib/python3.8/site-packages/evw/__init__.py", line 5, in <module>
    from .libvtkEvWPython import vtkEvW
ImportError: /data2/work/sandbox/testwrapvtk820/TestWrapPythonVTK820Build/lib/python3.8/site-packages//evw/libvtkEvWPythonD.so: undefined symbol: PyvtkObject_ClassNew

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<string>", line 1, in <module>
  File "/data2/work/sandbox/testwrapvtk820/TestWrapPythonVTK820Build/lib/python3.8/site-packages/evw/__init__.py", line 9, in <module>
    from libvtkEvWPython import vtkEvW
ImportError: /data2/work/sandbox/testwrapvtk820/TestWrapPythonVTK820Build/lib/python3.8/site-packages//evw/libvtkEvWPythonD.so: undefined symbol: PyvtkObject_ClassNew
9

Thanks for putting together a test package. I don’t have a VTK 8 build to test against at the moment, so please do some of the checks that I outlined in my previous response, and if they don’t help you resolve the issue, then I can build my own copy of VTK 8 and then run your test.

10

Will do and report back to you
Thanks

11

My suspicion is that you need something like this, where REQUIRED_VTK_LIBRARIES is a list of the VTK libraries you use.

    set(REQUIRED_PYTHON_LIBS vtkWrappingPythonCore)
    foreach(_lib ${REQUIRED_VTK_LIBRARIES})
      list(APPEND REQUIRED_PYTHON_LIBS ${_lib}PythonD)
    endforeach()

Then add REQUIRED_PYTHON_LIBS to your target_link_libraries().

12

Thanks David. Your last suggestion was spot on. I needed to explicitly indicate the required vtk*PythonD libs and link them in.

Another minor thing was to drop the prefix lib in usage.

For completeness, I am attaching the (now) working with VTK8.2.0 tar ball.

TestWrapPythonVTK820.tar.gz (2.3 KB)