[Issue] Touchscreens Not Working

[RMichaelPickering]

What platform are you experiencing this issue on?

Windows x64

What version of UWB are you using?

2.2.8

What Unity version are you running?

6000.3.7f1

Describe what the issue you are experiencing is.

Touchscreen not working with New Input System due to missing UI pointer location for 'Click' events. I've tracked down the root cause and applied the below hotfix to the RawImageUwbClientManager (see notes in comments) though it seems inelegant:

// UnityWebBrowser (UWB)
// Copyright (c) 2021-2022 Voltstro-Studios
//
// This project is under the MIT license. See the LICENSE.md file for more details.

using System;
using System.Collections;
using UnityEngine;
using UnityEngine.EventSystems;
using VoltstroStudios.UnityWebBrowser.Helper;
using VoltstroStudios.UnityWebBrowser.Input;
using VoltstroStudios.UnityWebBrowser.Shared;
using VoltstroStudios.UnityWebBrowser.Shared.Events;

namespace VoltstroStudios.UnityWebBrowser.Core
{
///

/// Input handler for .
///

public abstract class RawImageUwbClientInputHandler : RawImageUwbClientManager,
IPointerEnterHandler,
IPointerExitHandler,
IPointerDownHandler,
IPointerUpHandler
{
///

/// The to use
///

[Tooltip("The input handler to use")] public WebBrowserInputHandler inputHandler;

    /// <summary>
    ///     Disable usage of mouse
    /// </summary>
    [Tooltip("Disable usage of mouse")] public bool disableMouseInputs;

    /// <summary>
    ///     Disable usage of keyboard
    /// </summary>
    [Tooltip("Disable usage of keyboard")] public bool disableKeyboardInputs;

    private Coroutine keyboardAndMouseHandlerCoroutine;
    private Vector2 lastSuccessfulMousePositionSent;

    /* 20260627: RMichaelPickering - HOTFIX: Replace method to use GetPointerPosition instead of GetMousePosition.
     * This is to support touchscreen input!
     */
    public void OnPointerDown(PointerEventData eventData)
    {
        if (disableMouseInputs)
            return;

        if (browserClient is { IsConnected: false })
            return;

        MouseClickType clickType = eventData.button switch
        {
            PointerEventData.InputButton.Left => MouseClickType.Left,
            PointerEventData.InputButton.Right => MouseClickType.Right,
            PointerEventData.InputButton.Middle => MouseClickType.Middle,
            _ => throw new ArgumentOutOfRangeException()
        };

        int clickCount = eventData.clickCount > 0 ? eventData.clickCount : 1;

        /* HOTFIX: We assume that even for touch press/tap, eventData contains the correct position of the pointer,
         * so we can use that instead of relying on GetMousePosition which does not work for touch input.
         * I retested to confirm that the eventData.position will also be correct for mouse pointer devices.
         * So based on my testing, this HOTFIX works for touchscreen and mouse, at least on Windows devices.
         */
        if (GetPointerPosition(eventData.position, out Vector2 pos))
        {
            browserClient.SendMouseMove(pos);
            lastSuccessfulMousePositionSent = pos;
            browserClient.SendMouseClick(pos, clickCount, clickType, MouseEventType.Down);
        }
    }

    // ORIGINAL CODE, see HOTFIX note above

/* public void OnPointerDown(PointerEventData eventData)
{
if (disableMouseInputs)
return;

        if (browserClient is { IsConnected: false })
            return;

        MouseClickType clickType = eventData.button switch
        {
            PointerEventData.InputButton.Left => MouseClickType.Left,
            PointerEventData.InputButton.Right => MouseClickType.Right,
            PointerEventData.InputButton.Middle => MouseClickType.Middle,
            _ => throw new ArgumentOutOfRangeException()
        };
        
        int clickCount = eventData.clickCount > 0 ? eventData.clickCount : 1;

        if (GetMousePosition(out Vector2 pos))
            browserClient.SendMouseClick(pos, clickCount, clickType, MouseEventType.Down);
    }

*/
public void OnPointerEnter(PointerEventData eventData)
{
if (browserClient is { IsConnected: false })
return;

        keyboardAndMouseHandlerCoroutine = StartCoroutine(KeyboardAndMouseHandler());
    }

    public void OnPointerExit(PointerEventData eventData)
    {
        StopKeyboardAndMouseHandler();
    }

    /* 20260627: RMichaelPickering - HOTFIX: Replace method to use GetPointerPosition instead of GetMousePosition.
     * This is to support touchscreen input!
     */
    public void OnPointerUp(PointerEventData eventData)
    {
        if (disableMouseInputs)
            return;

        if (browserClient is { IsConnected: false })
            return;

        MouseClickType clickType = eventData.button switch
        {
            PointerEventData.InputButton.Left => MouseClickType.Left,
            PointerEventData.InputButton.Right => MouseClickType.Right,
            PointerEventData.InputButton.Middle => MouseClickType.Middle,
            _ => throw new ArgumentOutOfRangeException()
        };

        int clickCount = eventData.clickCount > 0 ? eventData.clickCount : 1;

        /* HOTFIX: We assume that even for touch press/tap, eventData contains the correct position of the pointer,
         * so we can use that instead of relying on GetMousePosition which does not work for touch input.
         * I retested to confirm that the eventData.position will also be correct for mouse pointer devices.
         * So based on my testing, this HOTFIX works for touchscreen and mouse, at least on Windows devices.
         */
        if (GetPointerPosition(eventData.position, out Vector2 pos))
        {
            browserClient.SendMouseClick(pos, clickCount, clickType, MouseEventType.Up);
        }
    }
    
    // ORIGINAL CODE, see HOTFIX note above
    /*
            public void OnPointerUp(PointerEventData eventData)
            {
                if (disableMouseInputs)
                    return;

                if (browserClient is { IsConnected: false })
                    return;

                MouseClickType clickType = eventData.button switch
                {
                    PointerEventData.InputButton.Left => MouseClickType.Left,
                    PointerEventData.InputButton.Right => MouseClickType.Right,
                    PointerEventData.InputButton.Middle => MouseClickType.Middle,
                    _ => throw new ArgumentOutOfRangeException()
                };

                int clickCount = eventData.clickCount > 0 ? eventData.clickCount : 1;

                if (GetMousePosition(out Vector2 pos))
                    browserClient.SendMouseClick(pos, clickCount, clickType, MouseEventType.Up);
            }
    */

    protected override void OnStart()
    {
        base.OnStart();
        if (inputHandler == null)
            throw new NullReferenceException("The input handler is null! You need to assign it in the editor!");
        
        browserClient.OnInputFocus += OnClientInput;
    }

    private void OnClientInput(bool focused)
    {
        if (browserClient is { IsConnected: false })
            return;
        
        if (focused)
        {
            if (!GetMousePosition(out Vector2 pos)) return;
            pos.x /= 1.5f;
            inputHandler.EnableIme(pos);
            return;
        }
        
        inputHandler.DisableIme();
    }

    protected override void OnDestroyed()
    {
        base.OnDestroyed();
        StopKeyboardAndMouseHandler();
    }

    /// <summary>
    ///     Gets the current mouse position on the image
    /// </summary>
    /// <param name="pos"></param>
    /// <returns>Returns true if the mouse is in the image.</returns>
    public bool GetMousePosition(out Vector2 pos)
    {
        Vector2 mousePos = inputHandler.GetCursorPos();

        if (WebBrowserUtils.GetScreenPointToLocalPositionDeltaOnImage(image, mousePos, out pos))
        {
            Texture imageTexture = image.texture;
            pos.x *= imageTexture.width;
            pos.y *= imageTexture.height;

            return true;
        }

        return false;
    }

    /* 20260627: HOTFIX: Replaced call to GetMousePosition to instead call this new method!
     * This is to support touchscreen input!
     * NOTE: Not to be a stickler, but the original Mouse version of this method had a semi-side-effect of
     * looking up the mouse position from, er, somewhere inside this module where it was being stored, I think.
     * Then it would check if that remembered position is inside an image that is being used to display the browser, I think.
     * Ultimately what's needed is a way to check if the user just clicked/tapped something in the embedded browser window
     * that is a control, and if so, we need to send the corresponding event to the browser so that it can handle it.
     * I see also that there's a keyboard/mouse handler coroutine that starts/stops when the pointer enters/exits the 
     * browser area, but in general that won't work for touchscreen input as when there's no touch, there's no pointer
     * location. I'm uncertain how big an issue this is, but probably to be fully general, there should be a touch
     * handler and/or a way to not need to enable/disable the input handlers based on pointer enter/exit events.
     * At least with respect to handling both touch and mouse the desired results seem to be achieved now!
     */
    public bool GetPointerPosition(Vector2 screenPos, out Vector2 pos)
    {
        if (WebBrowserUtils.GetScreenPointToLocalPositionDeltaOnImage(image, screenPos, out pos))
        {
            Texture imageTexture = image.texture;
            pos.x *= imageTexture.width;
            pos.y *= imageTexture.height;
            return true;
        }
        return false;
    }

    private void StopKeyboardAndMouseHandler()
    {
        if (keyboardAndMouseHandlerCoroutine != null)
        {
            StopCoroutine(keyboardAndMouseHandlerCoroutine);
            inputHandler.OnStop();
        }
    }

    private IEnumerator KeyboardAndMouseHandler()
    {
        inputHandler.OnStart();

        while (Application.isPlaying)
        {
            yield return 0;

            if (!browserClient.ReadySignalReceived || !browserClient.IsConnected 
                                                   || browserClient.HasDisposed)
                continue;

            if (disableMouseInputs && disableKeyboardInputs)
                continue;

            if (GetMousePosition(out Vector2 pos))
            {
                if (!disableMouseInputs)
                {
                    //Mouse position
                    if (lastSuccessfulMousePositionSent != pos)
                    {
                        browserClient.SendMouseMove(pos);
                        lastSuccessfulMousePositionSent = pos;
                    }

                    //Mouse scroll
                    float scroll = inputHandler.GetScroll();
                    scroll *= browserClient.BrowserTexture.height;

                    if (scroll != 0)
                        browserClient.SendMouseScroll(pos, (int)scroll);
                }

                if (!disableKeyboardInputs)
                {
                    //Input
                    WindowsKey[] keysDown = inputHandler.GetDownKeys();
                    WindowsKey[] keysUp = inputHandler.GetUpKeys();
                    string inputBuffer = inputHandler.GetFrameInputBuffer();

                    if (keysDown.Length > 0 || keysUp.Length > 0 || inputBuffer.Length > 0)
                        browserClient.SendKeyboardControls(keysDown, keysUp, inputBuffer.ToCharArray());
                }
            }
        }

        inputHandler.OnStop();
    }
}

}

Provide reproducible steps for this issue.

1. Setup UWB (I used the Basic version) with Unity's New Input System
2. Try touchscreen click on UWB browser UI element like a button or hotlink
3. It doesn't trigger the desired UI event in the browser

Any additional info you like to provide?

No response