I am looking to implement a simple 3D triangle creator in JavaFX, for that I need to be able to convert the 2D mouse position to 3D.

After searching for a while, it seems like the only way to do this is to use the internal CameraHelper class, specifically it’s pickProjectPlane(camera, x, y) method

This method however does not seem to work as I’d expect with my setup, where i have a SubScene for the 3D scene

Here is some example code, my goal is to have a box rendered at the click position

public class JavaFX3DPickingTest extends Application {

    private SubScene scene3D;
    private OrbitCamera orbitCamera;
    private Group sceneGroup = new Group();

    @Override
    public void start(Stage stage) {
        AnchorPane pane = new AnchorPane();
        create3DSubScene(800, 600);
        pane.getChildren().add(scene3D);
        // for reference
        Box referenceBox = new Box(1, 1, 1);
        sceneGroup.getChildren().add(referenceBox);

        Scene scene = new Scene(pane, 800, 600, true);
        // even if the listener is added to scene3D, it produces the same result
        scene.setOnMousePressed(event -> {
            // only pick when alt is down
            if (!event.isAltDown()) {
                return;
            }
            double x = event.getSceneX();
            double y = event.getSceneY();
            // these values are too huge, clearly they need to be scaled, but I don't know by what factor exactly
            Point3D pos = CameraHelper.pickProjectPlane(orbitCamera.camera, x, y);
            System.out.println("Pos: " + pos);
            Box box = new Box(1, 1, 1);
            box.setTranslateX(pos.getX());
            box.setTranslateY(pos.getY());
            box.setTranslateZ(pos.getZ());
            sceneGroup.getChildren().add(box);
        });
        stage.setScene(scene);
        stage.show();
    }

    private void create3DSubScene(double width, double height) {
        scene3D = new SubScene(sceneGroup, width, height, true, SceneAntialiasing.BALANCED);
        scene3D.setFill(Color.rgb(25, 25, 25));
        orbitCamera = new OrbitCamera(scene3D, sceneGroup);
    }

    private static class OrbitCamera {

        private final SubScene subScene;
        private final Group root3D;

        private final double MAX_ZOOM = 300.0;

        public OrbitCamera(SubScene subScene, Group root) {
            this.subScene = subScene;
            this.root3D = root;
            init();
        }

        private void init() {
            camera.setNearClip(0.1D);
            camera.setFarClip(MAX_ZOOM * 1.15D);
            root3D.getChildren().add(camera); // do we really need to do this

            updateCamera();

            subScene.setCamera(camera);
            subScene.setOnScroll(event -> {
                double zoomFactor = 1.05;
                double deltaY = event.getDeltaY();

                if (deltaY < 0) {
                    zoomFactor = 2.0 - zoomFactor;
                }
                double z = cameraDistance / zoomFactor;
                z = Math.max(1, Math.min(MAX_ZOOM, z));
                cameraDistance = z;
                updateCamera();
            });

            subScene.setOnMousePressed(event -> {
                if (event.isAltDown()) {
                    return;
                }
                mousePosX = event.getSceneX();
                mousePosY = event.getSceneY();
            });

            subScene.setOnMouseDragged(event -> {
                if (event.isAltDown()) {
                    return;
                }
                double modifier = 1.0;
                double modifierFactor = 0.3;

                if (event.isControlDown()) modifier = 0.1;
                if (event.isSecondaryButtonDown()) modifier = 0.035;

                double sceneX = event.getSceneX();
                double sceneY = event.getSceneY();
                double mouseDeltaX = sceneX - mousePosX;
                double mouseDeltaY = sceneY - mousePosY;
                mousePosX = sceneX;
                mousePosY = sceneY;

                if (event.isSecondaryButtonDown()) {
                    double dx = -mouseDeltaX * modifierFactor * modifier * 2.0;
                    double dy = -mouseDeltaY * modifierFactor * modifier * 2.0;
                    translation.setX(translation.getX() + dx);
                    translation.setY(translation.getY() + dy);
                } else if (event.isPrimaryButtonDown()) {
                    double dx = mouseDeltaX * modifierFactor * modifier * 2.0;
                    double dy = -mouseDeltaY * modifierFactor * modifier * 2.0;
                    yaw += dx;
                    pitch += dy;
                }

                updateCamera();
            });
        }

        public void updateCamera() {
            Affine affine = new Affine();
            Rotate rotateY = new Rotate(yaw, Rotate.Y_AXIS);
            Rotate rotateX = new Rotate(pitch, Rotate.X_AXIS);
            affine.append(rotateY);
            affine.append(rotateX);
            affine.append(new Translate(translation.getX(), translation.getY(), -cameraDistance));
            position = affine.transform(0, 0, 0);
            camera.getTransforms().setAll(affine);
        }


        private final PerspectiveCamera camera = new PerspectiveCamera(true);
        private double cameraDistance = 30;
        private Point3D position;
        private final Translate translation = new Translate(0.1889, -3.4019, -cameraDistance);
        private double yaw = 1.0;
        private double pitch = -19.4;
        private double mousePosX = 0;
        private double mousePosY = 0;
    }

}