Building an annotation software using VTK's interactor - issue with coordinate skipping

Here’s what I’m talking about:

Animation

The third one is potentially very problematic - what I do after the points have been dragged is to connect the coordinates and use that to create a region of interest. If the coordinates don’t save correctly (and they don’t if the visualization looks like this), then my ROI is wrong.

I’m using VTK 9.3.0. MWE follows

from vtkmodules.all import *
colors = vtkNamedColors()


class AnnotatorView:
    def __init__(self) -> None:
        self.renderer = vtkRenderer()
        self.render_window = vtkRenderWindow()
        self.render_window.AddRenderer(self.renderer)
        self.render_window.SetWindowName('Annotator')
        self.render_window.SetSize(800, 800)

        self.interactor = vtkRenderWindowInteractor()
        self.interactor.SetRenderWindow(self.render_window)

        self.create_background()

    def create_background(self, width=800, height=800):
        image_data = vtkImageData()
        image_data.SetDimensions(width, height, 1)
        image_data.AllocateScalars(VTK_UNSIGNED_CHAR, 1)

        for y in range(height):
            for x in range(width):
                gradient = y / 800.0
                image_data.SetScalarComponentFromFloat(x, y, 0, 0, gradient * 255)

        actor = vtkImageActor()
        actor.GetMapper().SetInputData(image_data)
        self.renderer.AddActor(actor)

    def add_point_actor(self, image_coordinates, point_size=2):
        """ Add a sphere at the given image coordinates """
        source = vtkSphereSource()
        source.SetCenter(image_coordinates[0], image_coordinates[1], 0)
        source.SetRadius(point_size)

        mapper = vtkPolyDataMapper()
        mapper.SetInputConnection(source.GetOutputPort())

        # Create an panel_actor
        actor = vtkActor()
        actor.SetMapper(mapper)
        actor.GetProperty().SetColor(colors.GetColor3d("Red"))
        actor.GetProperty().SetDiffuse(1)
        actor.GetProperty().SetSpecular(1)
        self.renderer.AddActor(actor)
        return actor   


class AnnotatorInteractorStyle(vtkInteractorStyleImage):
    def __init__(self, view, parent=None):
        self.is_dragging = False
        self.is_annotating = False
        self.dragged_points = []
        self.view = view

        self.AddObserver('LeftButtonPressEvent', self.left_button_press_event)
        self.AddObserver('KeyPressEvent', self.key_press_event)
        self.AddObserver('LeftButtonReleaseEvent', self.left_button_release_event)
        self.AddObserver("MouseMoveEvent", self.on_mouse_move_event)

    def key_press_event(self, obj, event):
        key = self.view.interactor.GetKeySym()
        if key == "a":
            self.is_annotating = True
        if key == "s":
            self.is_annotating = False

    def left_button_press_event(self, obj, event):
        if self.is_annotating:
            self.is_dragging = True

    def left_button_release_event(self, obj, event):
        if self.is_annotating:
            self.is_dragging = False

    def on_mouse_move_event(self, obj, event):
        if self.is_dragging:
            (mouseX, mouseY) = self.GetInteractor().GetEventPosition()
            # print(f'event position: {mouseX} , {mouseY}', flush=True)
            picker = vtkPointPicker()
            picker.Pick(mouseX, mouseY, 0.0, self.view.renderer)
            coords = picker.GetPickPosition()
            # print(f"image coords: {coords}")

            self.dragged_points.append(coords)
            self.view.add_point_actor(coords)
            self.view.render_window.Render()


if __name__ == "__main__":
    print("START")
    view = AnnotatorView()

    interactor_style = AnnotatorInteractorStyle(view)
    view.interactor.SetInteractorStyle(interactor_style)

    view.renderer.ResetCamera()
    view.render_window.Render()
    view.interactor.Initialize()
    view.interactor.Start()

    print("DONE")

Is this a VTK issue or is my code just not optimized?

(Our typical background images are 800*600 or smaller)