For now I am trying to do the same only in world space with the filter, not using UserMatrix.
Look I also tried this approach , same results, wrong coordinates after calculating rotation for 2 circle blue points :
vtkMatrix4x4* transformMatrix = loadWorkVolumeTransform(transformFilePath);
vtkNew<vtkConeSource> vtkConeSource;
vtkConeSource->SetRadius(0.6);
vtkConeSource->SetCenter(0, 0, 0);
vtkConeSource->Update();
double originalBounds[6] = { 0 };
vtkConeSource->GetOutput()->GetBounds(originalBounds);
vtkNew<vtkTransform> transform3D;
transform3D->SetMatrix(transformMatrix);
transform3D->Update();
vtkNew <vtkTransformPolyDataFilter> transformFilter3D;
transformFilter3D->SetInputData(vtkConeSource->GetOutput());
transformFilter3D->SetTransform(transform3D);
transformFilter3D->Update();
appender_result->AddInputConnection(transformFilter3D->GetOutputPort());
appender_result->Update();
double originalMin[3] = { originalBounds[0],originalBounds[2],originalBounds[4]};
double originalMax[3] = { originalBounds[1],originalBounds[3],originalBounds[5]};
double center[3] = { originalMin[0]+(originalMax[0] - originalMin[0]) / 2,originalMin[1]+(originalMax[1] - originalMin[1])/2,originalMin[2]+(originalMax[2] - originalMin[2])/2 };
vtkSmartPointer<vtkPoints> boundsPoints = vtkSmartPointer<vtkPoints>::New();
vtkSmartPointer<vtkPolyData> boundsPolydata = vtkSmartPointer<vtkPolyData>::New();
boundsPoints->InsertNextPoint(originalMin);
boundsPoints->InsertNextPoint(originalMax);
boundsPolydata->SetPoints(boundsPoints);
vtkNew<vtkTransform> transform;
transform->SetMatrix(transformMatrix);
transform->Update();
vtkTransformPolyDataFilter* transformFilter = vtkTransformPolyDataFilter::New();
transformFilter->SetInputData(boundsPolydata);
transformFilter->SetTransform(transform);
transformFilter->Update();
double transformedMin[3];
transformFilter->GetOutput()->GetPoint(0, transformedMin);
double transformedMax[3];
transformFilter->GetOutput()->GetPoint(1, transformedMax);
drawCircle1(transformedMin, color_blue);
drawCircle1(transformedMax, color_blue);
vtkSmartPointer<vtkPolyDataMapper> polyDataMapper_result = vtkSmartPointer<vtkPolyDataMapper>::New();
vtkSmartPointer<vtkActor> actor_result = vtkSmartPointer<vtkActor>::New();
actor_result->SetMapper(polyDataMapper_result);
polyDataMapper_result->SetInputData(appender_result->GetOutput());
drawBoundingBox(actor_result->GetBounds());
vtkRendered->AddActor(actor_result);
