How to color polydata using a vector of HSV data?

How do I color a polydata surface of N points using an Nx3 array where the columns correspond to H,S,V values? I have tried using mapper.SetColorModeToDefault() but it doesn’t seem to do what I thought it did. I will be fine with using RGB values if necessary, I can just pre-convert my HSV values using the python colorsys module. My current code is:

#!/usr/bin/env python

# noinspection PyUnresolvedReferences
import vtkmodules.vtkInteractionStyle
# noinspection PyUnresolvedReferences
import vtkmodules.vtkRenderingOpenGL2
from vtkmodules.vtkCommonDataModel import vtkPolyData
from vtkmodules.vtkCommonColor import vtkNamedColors
from vtkmodules.vtkFiltersCore import vtkElevationFilter
from vtkmodules.vtkFiltersSources import vtkSphereSource
from vtkmodules.vtkRenderingCore import (
    vtkActor,
    vtkDataSetMapper,
    vtkRenderWindow,
    vtkRenderWindowInteractor,
    vtkRenderer
)
from vtk.util import numpy_support as ns
from vtk import VTK_UNSIGNED_CHAR
import numpy as np


def main():

    renderer = vtkRenderer()
    renWin = vtkRenderWindow()
    renWin.AddRenderer(renderer)
    iren = vtkRenderWindowInteractor()
    iren.SetRenderWindow(renWin)

    sphere = vtkSphereSource()
    sphere.SetPhiResolution(12)
    sphere.SetThetaResolution(12)
    sphere.Update()
    
    polydata = vtkPolyData()
    polydata.ShallowCopy(sphere.GetOutput())
    
    N = polydata.GetNumberOfPoints()
    
    # Create N-by-3 array, where the columns should correspond to H,S and V values at each point
    my_data_array = (np.random.random((N,3))*256).astype(np.uint8)

    vtk_array = ns.numpy_to_vtk(num_array=my_data_array, deep=True, array_type=VTK_UNSIGNED_CHAR)
    vtk_array.SetName("HSV")
    polydata.GetPointData().AddArray(vtk_array)

    mapper = vtkDataSetMapper()
    mapper.SelectColorArray("HSV")
    mapper.SetInputData(polydata);
    mapper.SetColorModeToDefault()
    mapper.ScalarVisibilityOff()
    
    actor = vtkActor()
    actor.SetMapper(mapper)

    renderer.AddActor(actor)
    renWin.SetSize(640, 480)
    renWin.SetWindowName('ColoredSphere')

    renWin.Render()

    # Interact with the data.
    iren.Start()


if __name__ == '__main__':
    main()

You won’t be able to color by HSV values directly, but if you can color by RGB values by calling

mapper.SetColorModeToDirectScalars()

and set your color array to the array of converted RGB values.