Hello
I’m trying to create a parallel structured grid but the extents for the pieces are wrong. This is how it looks like:
<?xml version="1.0"?>
<VTKFile type="PStructuredGrid" version="0.1" byte_order="LittleEndian" header_type="UInt32" compressor="vtkZLibDataCompressor">
<PStructuredGrid WholeExtent="0 15 0 15 0 0" GhostLevel="0">
<PPointData Scalars="data">
<PDataArray type="Float64" Name="data"/>
</PPointData>
<PPoints>
<PDataArray type="Float32" Name="Points" NumberOfComponents="3"/>
</PPoints>
<Piece Extent="0 7 0 15 0 0" Source="one_0.vts"/>
<Piece Extent="0 7 0 15 0 0" Source="one_1.vts"/>
</PStructuredGrid>
</VTKFile>
and the x-extents for the second piece are wrong. It should be 8 15 instead of 0 7.
The test code generating this is as follows:
#include <iostream>
#include <vector>
#include <mpi.h>
#include <vtkSmartPointer.h>
#include <vtkDataSet.h>
#include <vtkStructuredGrid.h>
#include <vtkXMLPStructuredGridWriter.h>
#include <vtkSOADataArrayTemplate.h>
#include <vtkAOSDataArrayTemplate.h>
#include <vtkPoints.h>
#include <vtkNew.h>
#include <vtkTrivialProducer.h>
#include <vtkMPIController.h>
#include <vtkPointData.h>
#include <vtkCellData.h>
#include <vtkDoubleArray.h>
#include <vtkUnstructuredGrid.h>
#include <vtkXMLPUnstructuredGridWriter.h>
#include <vtkCellArray.h>
#include <vtkDataSet.h>
int main(int argc, char** argv)
{
MPI_Init(NULL, NULL);
// Get the number of processes
int world_size, world_rank;
MPI_Comm_size(MPI_COMM_WORLD, &world_size);
MPI_Comm_rank(MPI_COMM_WORLD, &world_rank);
std::cout << "size and rank: " << world_size << ", " << world_rank << "!" << std::endl;
// Create fake VTK data for 2 ranks only
int globalExtents[6];
globalExtents[0] = 0;
globalExtents[1] = 15;
globalExtents[2] = 0;
globalExtents[3] = 15;
globalExtents[4] = 0;
globalExtents[5] = 0;
int localDims[3];
localDims[0] = 8;
localDims[1] = 16;
localDims[2] = 1;
int localExtents[6];
int offset = world_rank*8;
// x
localExtents[0] = offset;
localExtents[1] = localExtents[0]+localDims[0]-1;
// y
localExtents[2] = 0;
localExtents[3] = localDims[1]-1;
// z
localExtents[4] = 0;
localExtents[5] = 0;
std::vector<double> _xArray, _yArray;
double pointOffset = 1.0/15;
for (int i=0; i<localDims[0]; i++){
_xArray.push_back(world_rank*(pointOffset*8) + i*pointOffset);
std::cout << world_rank*(pointOffset*8) + i*pointOffset << ", ";
}
std::cout << "\ny-axis\n";
for (int i=0; i<localDims[1]; i++){
_yArray.push_back(i*pointOffset);
std::cout << i*pointOffset << ", ";
}
std::cout << "\n";
std::vector<double> data;
for (int i=0; i<localDims[0]*localDims[1]*localDims[2]; i++)
data.push_back(world_rank);
std::cout << world_rank << " ~ global extents: " << globalExtents[0] << ", " << globalExtents[1] << " "
<< globalExtents[2] << ", " << globalExtents[3] << " "
<< globalExtents[4] << ", " << globalExtents[5] << std::endl;
std::cout << world_rank << " ~ local extents: " << localExtents[0] << ", " << localExtents[1] << " "
<< localExtents[2] << ", " << localExtents[3] << " "
<< localExtents[4] << ", " << localExtents[5] << std::endl;
std::cout << world_rank << " ~ local dims: " << localDims[0] << ", " << localDims[1] << ", " << localDims[2] << std::endl;
MPI_Barrier(MPI_COMM_WORLD);
vtkSmartPointer<vtkStructuredGrid> strucGrid = vtkSmartPointer<vtkStructuredGrid>::New();
vtkSmartPointer<vtkPoints> pnts = vtkSmartPointer<vtkPoints>::New();
static auto contr = vtkSmartPointer<vtkMPIController>::New();
if (contr != NULL)
contr->Initialize(NULL, NULL, 1);
strucGrid->SetExtent(globalExtents);
strucGrid->SetDimensions(localDims);
//pnts->Allocate(localDims[0] * localDims[1] * localDims[2]);
for (int j = 0; j < localDims[1]; j++)
for (int i = 0; i < localDims[0]; i++)
pnts->InsertNextPoint(_xArray[i], _yArray[j], 0);
strucGrid->SetPoints(pnts);
vtkSmartPointer<vtkDoubleArray> values = vtkDoubleArray::New();;
values->SetName("data");
values->SetArray(&data[0], localDims[0]*localDims[1]*localDims[2], 1);
strucGrid->GetPointData()->SetScalars(values);
MPI_Barrier(MPI_COMM_WORLD);
vtkSmartPointer<vtkXMLPStructuredGridWriter> writer = vtkSmartPointer<vtkXMLPStructuredGridWriter>::New();
writer->SetNumberOfPieces(world_size);
writer->SetStartPiece(world_rank);
writer->SetEndPiece(world_rank);
int nranks = contr->GetNumberOfProcesses();
int rank = contr->GetLocalProcessId();
std::cout << "vtk mpi :" << rank << ", " << nranks << "!!!" << std::endl;
vtkNew<vtkTrivialProducer> tp;
tp->SetOutput(strucGrid);
tp->SetWholeExtent(globalExtents[0], globalExtents[1],
globalExtents[2], globalExtents[3],
globalExtents[4], globalExtents[5]);
writer->SetInputConnection(tp->GetOutputPort());
writer->SetController(contr);
writer->SetFileName("one.pvts");
writer->Update();
writer->SetInputData(strucGrid);
writer->Write();
MPI_Barrier(MPI_COMM_WORLD);
MPI_Finalize();
return 0;
}
what am I missing here?
The code can be run as follows: mpirun -np 2 ./test (this is only a test code that I used to replicate the issue I have and so it only works with 2 ranks)
cmake_minimum_required(VERSION 3.12)
project(test_parallel_vtk)
find_package(VTK REQUIRED)
if(NOT VTK_FOUND)
message(FATAL_ERROR "VTK requested, but not found")
return()
endif()
message("VTK found")
add_executable(test main.cpp)
target_link_libraries(test PRIVATE
VTK::CommonCore
VTK::CommonDataModel
VTK::ParallelCore
VTK::ParallelMPI
VTK::IOXML
VTK::IOParallelXML
VTK::IOCore
VTK::IOParallel
VTK::IOMPIParallel
VTK::mpi)