Discuss multi planar reconstruction (MPR) rendering on the web

Support
1

Hello. I’m trying to build an MPR (serve side rendering) application that uses the wslink library for client-server communication. I followed the template: vue-vtkjs-pvw-template. I have rendered the axial, coronal and sagittal of a dicom image collection on the web but there is a problem that when I interact it will not display anymore. Below is the code.
vtkw-server.py

r"""
    This module is a ParaViewWeb server application.
    The following command line illustrates how to use it::

        $ vtkpython .../server.py

    Any ParaViewWeb executable script comes with a set of standard arguments that can be overrides if need be::

        --port 8080
            Port number on which the HTTP server will listen.

        --content /path-to-web-content/
            Directory that you want to serve as static web content.
            By default, this variable is empty which means that we rely on another
            server to deliver the static content and the current process only
            focuses on the WebSocket connectivity of clients.

        --authKey vtkweb-secret
            Secret key that should be provided by the client to allow it to make
            any WebSocket communication. The client will assume if none is given
            that the server expects "vtkweb-secret" as secret key.

"""
import os
import sys
import argparse

# Try handle virtual env if provided
if '--virtual-env' in sys.argv:
  virtualEnvPath = sys.argv[sys.argv.index('--virtual-env') + 1]
  virtualEnv = virtualEnvPath + '/bin/activate_this.py'
  with open(virtualEnv) as venv:
    exec(venv.read(), dict(__file__=virtualEnv))

# from __future__ import absolute_import, division, print_function

from wslink import server
from wslink import register as exportRpc

from vtk.web import wslink as vtk_wslink
from vtk.web import protocols as vtk_protocols

import vtk
from vtk_protocol import VtkCone

# =============================================================================
# Server class
# =============================================================================

class _Server(vtk_wslink.ServerProtocol):
    # Defaults
    authKey = "wslink-secret"
    view = None

    @staticmethod
    def add_arguments(parser):
        parser.add_argument("--virtual-env", default=None,
                            help="Path to virtual environment to use")

    @staticmethod
    def configure(args):
        # Standard args
        _Server.authKey = args.authKey

    def initialize(self):
        # Bring used components
        self.registerVtkWebProtocol(vtk_protocols.vtkWebMouseHandler())
        self.registerVtkWebProtocol(vtk_protocols.vtkWebViewPort())
        self.registerVtkWebProtocol(vtk_protocols.vtkWebPublishImageDelivery(decode=False))

        # Custom API
        self.registerVtkWebProtocol(VtkCone())

        # tell the C++ web app to use no encoding.
        # ParaViewWebPublishImageDelivery must be set to decode=False to match.
        self.getApplication().SetImageEncoding(0)

        # Update authentication key to use
        self.updateSecret(_Server.authKey)

        if not _Server.view:
            renderWindow = vtk.vtkRenderWindow()
            renderWindow.OffScreenRenderingOn()

            renderWindowInteractor = vtk.vtkRenderWindowInteractor()
            renderWindowInteractor.SetInteractorStyle(vtk.vtkInteractorStyleTrackballCamera())
            
            self.getApplication().GetObjectIdMap().SetActiveObject("VIEW", renderWindow)

# =============================================================================
# Main: Parse args and start serverviewId
# =============================================================================

if __name__ == "__main__":
    # Create argument parser
    parser = argparse.ArgumentParser(description="Cone example")

    # Add arguments
    server.add_arguments(parser)
    _Server.add_arguments(parser)
    args = parser.parse_args()
    _Server.configure(args)

    # Start server
    server.start_webserver(options=args, protocol=_Server, disableLogging=True)

vtk_protocol.py

from vtk.web import protocols as vtk_protocols

from wslink import register as exportRpc

import vtk, logging

from vtkmodules.vtkCommonCore import vtkCommand

logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(name)s - %(levelname)s - %(message)s")

# -------------------------------------------------------------------------
# ViewManager
# -------------------------------------------------------------------------

class MPR(vtk_protocols.vtkWebProtocol):
    def __init__(self) -> None:
        # Dicom directory path
        self.dicomDirPath = None

    @exportRpc("vtk.initialize")
    def createVisualization(self) -> None:
        def resliceCursorCallback(obj, event) -> None:
            for i in range(0, 3):
                ps = planeWidgetArray[i].GetPolyDataAlgorithm()
                origin = resliceCursorWidgetArray[i].GetResliceCursorRepresentation().GetPlaneSource().GetOrigin()
                ps.SetOrigin(origin)
                point1 = resliceCursorWidgetArray[i].GetResliceCursorRepresentation().GetPlaneSource().GetPoint1()
                ps.SetPoint1(point1)
                point2 = resliceCursorWidgetArray[i].GetResliceCursorRepresentation().GetPlaneSource().GetPoint2()
                ps.SetPoint2(point2)
                planeWidgetArray[i].UpdatePlacement()
        
        reader = vtk.vtkDICOMImageReader()
        volumeMapper = vtk.vtkPolyDataMapper()
        volumeActor = vtk.vtkActor()
        renderWindow = self.getApplication().GetObjectIdMap().GetActiveObject("VIEW")
        renderWindowInteractor = self.getApplication().GetObjectIdMap().GetActiveObject("INTERACTOR")
        picker = vtk.vtkCellPicker()

        picker.SetTolerance(0.005)
        renderWindow.SetInteractor(renderWindowInteractor)

        reader.SetDirectoryName(self.dicomDirPath)
        reader.Update()
        imageData = reader.GetOutput()
        imageDims = imageData.GetDimensions()
        scalarRange = imageData.GetScalarRange()
        center = imageData.GetCenter()

        # Bounding box
        outline = vtk.vtkOutlineFilter()
        outlineMapper = vtk.vtkPolyDataMapper()
        outlineActor = vtk.vtkActor()

        outline.SetInputConnection(reader.GetOutputPort())
        outlineMapper.SetInputConnection(outline.GetOutputPort())

        outlineActor.SetMapper(outlineMapper)
        outlineActor.GetProperty().SetColor(1,1,0)

        # Mapper
        volumeMapper.SetInputConnection(reader.GetOutputPort())

        # Actor
        volumeActor.SetMapper(volumeMapper)

        # Renderers and render window
        rendererArray = [None]*4
        for i in range(4):
            rendererArray[i] = vtk.vtkRenderer()
            renderWindow.AddRenderer(rendererArray[i])
        renderWindow.SetMultiSamples(0)

        # Property
        ipwProp = vtk.vtkProperty()

        # 3D plane widgets
        planeWidgetArray = [None]*3
        for i in range(3):
            planeWidgetArray[i] = vtk.vtkImagePlaneWidget()
            planeWidgetArray[i].SetInteractor(renderWindowInteractor)
            planeWidgetArray[i].SetPicker(picker)
            planeWidgetArray[i].RestrictPlaneToVolumeOn()
            color = [0, 0, 0 ]
            color[i] = 1
            planeWidgetArray[i].GetPlaneProperty().SetColor(color)
            planeWidgetArray[i].SetTexturePlaneProperty(ipwProp)
            planeWidgetArray[i].TextureInterpolateOff()
            planeWidgetArray[i].SetResliceInterpolateToLinear()
            planeWidgetArray[i].SetInputConnection(reader.GetOutputPort())
            planeWidgetArray[i].SetPlaneOrientation(i)
            planeWidgetArray[i].SetSliceIndex(int(imageDims[i] / 2))
            planeWidgetArray[i].DisplayTextOn()
            planeWidgetArray[i].SetDefaultRenderer(rendererArray[3])
            planeWidgetArray[i].On()
            planeWidgetArray[i].InteractionOff()

        planeWidgetArray[1].SetLookupTable(planeWidgetArray[0].GetLookupTable()) 
        planeWidgetArray[2].SetLookupTable(planeWidgetArray[0].GetLookupTable())

        # Reslice Cursor
        resliceCursor = vtk.vtkResliceCursor()
        resliceCursor.SetCenter(center[0], center[1], center[2])
        resliceCursor.SetThickMode(0)
        resliceCursor.SetThickness(10, 10, 10)
        resliceCursor.SetHole(1)
        resliceCursor.SetImage(imageData)

        # 2D Reslice cursor widgets
        resliceCursorWidgetArray = [None]*3
        resliceCursorRepArray = [None]*3
        viewUp = [[0, 0, -1], [0, 0, -1], [0, 1, 0]]
        for i in range(3):
            resliceCursorWidgetArray[i] = vtk.vtkResliceCursorWidget()
            resliceCursorRepArray[i] = vtk.vtkResliceCursorLineRepresentation()
            resliceCursorWidgetArray[i].SetInteractor(renderWindowInteractor)
            resliceCursorWidgetArray[i].SetRepresentation(resliceCursorRepArray[i])

            resliceCursorRepArray[i].GetResliceCursorActor().GetCursorAlgorithm().SetResliceCursor(resliceCursor)
            resliceCursorRepArray[i].GetResliceCursorActor().GetCursorAlgorithm().SetReslicePlaneNormal(i)

            resliceCursorActor = resliceCursorRepArray[i].GetResliceCursorActor()
            centerlineProperty = resliceCursorActor.GetCenterlineProperty(0) # // x
            centerlineProperty.SetLineWidth(1)
            centerlineProperty.SetColor([1, 0, 0]) # red
            centerlineProperty.SetRepresentationToWireframe()
            thickSlabProperty = resliceCursorActor.GetThickSlabProperty(0)
            thickSlabProperty.SetColor([1, 0, 0]) # red
            thickSlabProperty.SetRepresentationToWireframe()
            centerlineProperty = resliceCursorActor.GetCenterlineProperty(1) # // y
            centerlineProperty.SetLineWidth(1)
            centerlineProperty.SetColor([0, 1, 0]) # green
            centerlineProperty.SetRepresentationToWireframe()
            thickSlabProperty = resliceCursorActor.GetThickSlabProperty(1)
            thickSlabProperty.SetColor([0, 1, 0]) # green
            thickSlabProperty.SetRepresentationToWireframe()
            centerlineProperty = resliceCursorActor.GetCenterlineProperty(2) # // z
            centerlineProperty.SetLineWidth(1)
            centerlineProperty.SetColor([0, 0, 1]) # blue
            centerlineProperty.SetRepresentationToWireframe()
            thickSlabProperty = resliceCursorActor.GetThickSlabProperty(2)
            thickSlabProperty.SetColor([0, 0, 1]) # blue
            thickSlabProperty.SetRepresentationToWireframe()

            resliceCursorWidgetArray[i].SetDefaultRenderer(rendererArray[i])
            resliceCursorWidgetArray[i].SetEnabled(True)

            rendererArray[i].GetActiveCamera().SetFocalPoint(0, 0, 0)
            camPos = [0, 0, 0]
            camPos[i] = 1
            rendererArray[i].GetActiveCamera().SetPosition(camPos[0], camPos[1], camPos[2])
            rendererArray[i].GetActiveCamera().ParallelProjectionOn()
            rendererArray[i].GetActiveCamera().SetViewUp(viewUp[i][0], viewUp[i][1], viewUp[i][2])
            rendererArray[i].ResetCamera()

            resliceCursorRepArray[i].SetWindowLevel(scalarRange[1] - scalarRange[0], (scalarRange[0] + scalarRange[1]) / 2.0, 0)
            planeWidgetArray[i].SetWindowLevel(scalarRange[1] - scalarRange[0], (scalarRange[0] + scalarRange[1]) / 2.0, 0)
            
            resliceCursorRepArray[i].SetLookupTable(resliceCursorRepArray[0].GetLookupTable())
            planeWidgetArray[i].GetColorMap().SetLookupTable(resliceCursorRepArray[0].GetLookupTable())

            resliceCursorWidgetArray[i].AddObserver('InteractionEvent', resliceCursorCallback)
            resliceCursorWidgetArray[i].On()

        rendererArray[0].SetBackground(0.3, 0.1, 0.1)
        rendererArray[0].SetViewport(0, 0, 0.5, 0.5)

        rendererArray[1].SetBackground(0.1, 0.3, 0.1)
        rendererArray[1].SetViewport(0.5, 0, 1, 0.5)

        rendererArray[2].SetBackground(0.1, 0.1, 0.3)
        rendererArray[2].SetViewport(0, 0.5, 0.5, 1)

        rendererArray[3].AddActor(volumeActor)
        rendererArray[3].AddActor(outlineActor)
        rendererArray[3].SetBackground(0.1, 0.1, 0.1)
        rendererArray[3].SetViewport(0.5, 0.5, 1, 1)
        rendererArray[3].GetActiveCamera().Elevation(110)
        rendererArray[3].GetActiveCamera().SetViewUp(0, 0, -1)
        rendererArray[3].GetActiveCamera().Azimuth(45)
        rendererArray[3].GetActiveCamera().Dolly(1.15)
        rendererArray[3].ResetCamera()

        renderWindow.Render()

Looking forward to your comments. Thanks

2

Looking at the code, I don’t see anything obvious, but could you try to make a trame version of it (it should be simpler) that would allow the one looking at it to more easily reproduce locally what you may be running into.

3

For DICOM image visualization you probably don’t want to start from scratch, because even the most basic features takes year to develop and there are free (truly free, restriction-free) VTK-based solutions. For web-native solutions you can use OHIFv3 or Kitware’s VolView; for desktop-native you can use 3D Slicer. You can of course use all solutions on all platforms: package a web application as a desktop app and you can run a desktop app in your web browser.