volume not rendering using react

I’m trying to combine the time series example with volume rendering in react. But following code renders nothing on the screen.

The code downloads one frame of volume, and uses the vtkImageData-vtkVolumeMapper-vtkVolume combo for the rendering. The result is an empty render window with background and control panel.

When troubleshooting, I can see the data is loaded to the mapper (with correct bounds). But resizing window gave me errors like this:

Can any vtk-js and react expert spot any issues in my code? Thanks!


import { useState, useRef, useEffect } from 'react';

// Load the rendering pieces we want to use (for both WebGL and WebGPU)
import '@kitware/vtk.js/Rendering/Profiles/Geometry';

// Force DataAccessHelper to have access to various data source
import vtkHttpDataAccessHelper from '@kitware/vtk.js/IO/Core/DataAccessHelper/HttpDataAccessHelper';
import '@kitware/vtk.js/IO/Core/DataAccessHelper/HtmlDataAccessHelper';
import '@kitware/vtk.js/IO/Core/DataAccessHelper/JSZipDataAccessHelper';

import vtkFullScreenRenderWindow from '@kitware/vtk.js/Rendering/Misc/FullScreenRenderWindow';
import vtkVolume from '@kitware/vtk.js/Rendering/Core/Volume';
import vtkVolumeMapper from '@kitware/vtk.js/Rendering/Core/VolumeMapper';
import vtkColorTransferFunction from '@kitware/vtk.js/Rendering/Core/ColorTransferFunction';
import vtkPiecewiseFunction from '@kitware/vtk.js/Common/DataModel/PiecewiseFunction';

import vtkHttpDataSetReader from '@kitware/vtk.js/IO/Core/HttpDataSetReader';
import vtkXMLImageDataReader from '@kitware/vtk.js/IO/XML/XMLImageDataReader';
import vtkImageData from '@kitware/vtk.js/Common/DataModel/ImageData';

import styles from '../../app.module.css'

export default function Example() {
  console.log("Render App");
  const vtkContainerRef = useRef(null);
  const context = useRef(null); // vtk related objects
  const hasDownloadingStarted = useRef(false);
  const timeData = useRef([]);
  const [currentTP, setCurrentTP] = useState(1);

  //const BASE_URL = 'http://192.168.50.37:8000'
  const BASE_URL = 'http://10.102.180.67:8000'

  const { fetchBinary } = vtkHttpDataAccessHelper;

  function setVisibleDataset(ds) {
    console.log("[setVisibleDataset] timeData.length=", timeData.current.length);
    if (context.current) {
      const { renderWindow, mapper, renderer, actor } = context.current;
      mapper.setInputData(ds);
      console.log("-- input data set", ds);

      mapper.setSampleDistance(1.1);
      mapper.setPreferSizeOverAccuracy(true);
      mapper.setBlendModeToComposite();
      mapper.setIpScalarRange(0.0, 1.0);

      const  opacityFunction = vtkPiecewiseFunction.newInstance();
      opacityFunction.addPoint(-3024, 0.1);
      opacityFunction.addPoint(-637.62, 0.1);
      opacityFunction.addPoint(700, 0.5);
      opacityFunction.addPoint(3071, 0.9);
      actor.getProperty().setScalarOpacity(0, opacityFunction);

      const colorTransferFunction = vtkColorTransferFunction.newInstance();
      colorTransferFunction.addRGBPoint(0, 0, 0, 0);
      colorTransferFunction.addRGBPoint(1, 1, 1, 1);
      actor.getProperty().setRGBTransferFunction(0, colorTransferFunction);

      actor.getProperty().setScalarOpacityUnitDistance(0, 3.0);
      actor.getProperty().setInterpolationTypeToLinear();
      actor.getProperty().setShade(true);
      actor.getProperty().setAmbient(0.1);
      actor.getProperty().setDiffuse(0.9);
      actor.getProperty().setSpecular(0.2);
      actor.getProperty().setSpecularPower(10.0);

      renderer.resetCamera();
      renderer.getActiveCamera().elevation(-70);
      renderWindow.render();
    }
  }

  function downloadData() {
    console.log("[downloadData] started");
    const files = [
      'dist/img3d_bavcta008_baseline_00.vti'
    ];
    return Promise.all(
      files.map((fn) => 
        fetchBinary(`${BASE_URL}/${fn}`).then((binary) => {
          const reader = vtkXMLImageDataReader.newInstance();
          reader.parseAsArrayBuffer(binary);
          return reader.getOutputData(0);
        })
      )
    )
  };

  function downloadSample() {
    console.log("[downloadSample] started");
    const files = [
      'sample/headsq.vti'
    ];
    const httpReader = vtkHttpDataSetReader.newInstance({ fetchGzip: true });
    return Promise.all(
      files.map((fn) => 
        httpReader.setUrl(`${BASE_URL}/${fn}`).then(() => {
          return httpReader.getOutputData(0);
        })
      )
    );
  }

  /* Initialize renderWindow, renderer, mapper and actor */
  useEffect(() => {
    if (!context.current) {
      const fullScreenRenderWindow = vtkFullScreenRenderWindow.newInstance({
        rootContainer: vtkContainerRef.current, // html element containing this window
        background: [0.1, 0.1, 0.1],
      });

      const mapper = vtkVolumeMapper.newInstance();
      mapper.setInputData(vtkImageData.newInstance());

      const renderWindow = fullScreenRenderWindow.getRenderWindow();
      const renderer = fullScreenRenderWindow.getRenderer();
      const interactor = renderWindow.getInteractor();
      interactor.setDesiredUpdateRate(15.0);

      const actor = vtkVolume.newInstance();
      actor.setMapper(mapper);
      renderer.addVolume(actor);

      if (!hasDownloadingStarted.current) {
        hasDownloadingStarted.current = true;
        downloadSample().then((downloadedData) => {
          console.log("Data Downloaded: ", downloadedData);
          timeData.current = downloadedData;
          setVisibleDataset(timeData.current[currentTP - 1]);
        });
      }
      
      context.current = {
        fullScreenRenderWindow,
        renderWindow,
        renderer,
        actor,
        mapper
      };

      window.vtkContext = context.current;
    }

    return () => {
      if (context.current) {
        const { 
          fullScreenRenderWindow, actor, mapper, renderer
        } = context.current;

        actor.delete();
        mapper.delete();
        renderer.delete();
        fullScreenRenderWindow.delete();
        context.current = null;
      }
    };
  }, [vtkContainerRef]);

  function onRenderClicked() {
    if (context.current) {
      console.log("[onRenderClicked] timeData: ", timeData);
      const {renderWindow } = context.current;
      setVisibleDataset(timeData.current[currentTP - 1]);
      renderWindow.render();
    }
  }

  return (
    <div>
      <div ref={vtkContainerRef} />
      <div className={styles.control_panel}>
        <button onClick={onRenderClicked}>
          Manual Render
        </button>
      </div>
    </div>
  );
}

More information:

“dependencies”: {
@kitware/vtk.js”: “^25.13.1”,
“localforage”: “^1.10.0”,
“match-sorter”: “^6.3.1”,
“react”: “^18.2.0”,
“react-dom”: “^18.2.0”,
“react-router-dom”: “^6.4.3”,
“sort-by”: “^1.2.0”
},

The render window can render a simple cone without problem using vtkConeSource-vtkMapper-vtkActor combo. Issue only occurs when trying to render volume.

I just realized that I probably should have imported Volume rendering profile instead of Geometry.

So changed
import '@kitware/vtk.js/Rendering/Profiles/Geometry';
to
import '@kitware/vtk.js/Rendering/Profiles/Volume';

And after the change, now I’m getting this error:

I kind of solved it by changing the data reading method.

In the code sample I was reading data into an array of vtkImageData and set the image to mapper when a time frame became “active”.

I changed it to reading binaries using fetchBinary from vtkHttpDataAccessHelper, and store them into an array. When a time frame become “active”, I set the corresponding binary using parseAsArrayBuffer to a reader connected with mapper. Then the it started to render.

And I suspect parsing binary every time when a frame becomes active might bring extra performance cost. So I might need to have an array of readers instead. Any idea why the mapper’s setInputData is not working in this case?

Have you verified that you are passing in vtkImageData into mapper.setInputData?

Yes, I checked the data in console just before passing it to the mapper. It’s a vtkImageData with correct dimension, spacing and data.

Oh, it’s your usage of mapper.setInputData(vtkImageData.newInstance()). A new instance of vtkImageData has no pixel storage allocated for it. You’re better off deferring adding your actor to the scene until your data is available.

Thanks for the reply! The newInstance was a placeholder indeed, but are setting new image to the mapper expected to overwrite the existing data? Or should I call a function like update() to refresh the rendering pipeline?

I did check using console using mapper.getInputData() and the data looks correctly replaced, but was still not rendered.

Hmm… it seems like you didn’t set it properly. Here’s a simplified version of how I did it for a simple volume. Also, I created my own custom proxy class to help me with this and you should look into it if you can.

//Load your data first, then pass it as an ArrayBuffer and setup the pipeline
function setupPipeline(content: ArrayBuffer) {
    const reader = vtkXMLImageDataReader.newInstance();
    const piecewiseFunction = vtkPiecewiseFunction.newInstance();
    const colorTransferFunction = vtkColorTransferFunction.newInstance();
    const volumeMapper = vtkVolumeMapper.newInstance();
    const volume = vtkVolume.newInstance();
    const cropFilter = vtkImageCropFilter.newInstance();

    reader.parseAsArrayBuffer(content);
    cropFilter.setInputConnection(this.reader.getOutputPort());
    volumeMapper.setInputConnection(this.cropFilter.getOutputPort());
    volume.setMapper(this.volumeMapper);

    const dataArray: vtkDataArray = this.reader.getOutputData().getPointData().getScalars();

    colorTransferFunction.applyColorMap(vtkColorMaps.getPresetByName('jet'));
    colorTransferFunction.setMappingRange(...dataArray.getRange());
    colorTransferFunction.updateRange();
    volume.getProperty().setRGBTransferFunction(0, this.colorTransferFunction);
    volume.getProperty().setScalarOpacity(0, this.piecewiseFunction);

    return ({ reader, piecewiseFunction, colorTransferFunction, volumeMapper, volume, cropFilter});
}

    // In your onMount useEffect (with [ ] empty array as dependency), 
    // Instantiate your renderer, renderwindow, interactor and setContainer onto a HTMLDivElement with useRef 

//Run this function to manually render
function main() {
    const data = load(file); //Load file here
    // You may store the below objects you have created in some state so you can access them later
    const { reader, piecewiseFunction, colorTransferFunction, volumeMapper, volume, cropFilter } = setupPipeline(data);
    renderer.addVolume(volume);
    renderer.resetCamera();
    renderWindow.render();
}

Hi @Kalsyc , Thanks so much for the reply and sharing your code! And connecting reader’s OutputPort to mapper’s InputConnection worked in my case.

But if we want to render a time series of volumes, and want to change them frequently, like every 50 ms, is it possible for us to keep only one rendering pipeline and only change data to the pipeline for memory saving purpose. Currently I have 20 time points in my series. To make it work, I have 20 rendering pipelines setup and enable/disable visibility to create a “4d playing” feature. Is there any more efficient way to do this?

Thanks!

So this will go back to my original question. In this vanilla js time series example: vtk.js following code seems to work

function setVisibleDataset(ds) {
  mapper.setInputData(ds);
  renderer.resetCamera();
  renderWindow.render();
}

But it’s not working for my rewrite in React for volumes.

Maybe my experience with Angular may be helpful for React.
In my use of Angular, I sometimes found missing .d.ts for certain classes in vtk.js (macro?)
As a stop-gap, I would wrap my vtk.js functionality in a javascript class, and add a typescript declaration (a .d.ts file) to gain access to the javascript class/methods from the Angular typescript code.

As far as I know, you can use the same reader since I am assuming that all the volumes carry the same dataset (aka they were parsed from the same file). I think having the same mapper could work as well.

Could you share more snippets of your code with volumes?

One other thing you could check: you have a call to renderer.getActiveCamera().elevation(-70);. Comment that out and see if the dataset shows up in the view, as resetCamera() should bring it into view.

It indeed worked. I now can confirm my original problem was caused by adding a vtkImageData.newInstance() placeholder to the mapper. My solution is deferring adding image data until real data is loaded. Thanks so much for all the suggestions!

I did a recheck of my code again and can confirm setting the placeholder was the cause of my issue. An error occurred before I set new data to the mapper. That’s probably why setting new data was not working. It is now working fine. Thanks so much for the solution!