Hand Tracking

.. video:: ../../_static/19_hand_tracking.webm
    :alt: Hand Tracking Demo with Vuer and Vision Pro
    :autoplay:
    :nocontrols:
    :loop:
    :muted:
    :poster: ../_static/19_hand_tracking.png
    :preload: auto
    :width: 100%

from vuer import Vuer, VuerSession
from vuer.schemas import Hands
from asyncio import sleep

app = Vuer()

@app.add_handler("HAND_MOVE")
async def handler(event, session):
    print(f"Movement Event: key-{event.key}", event.value)

@app.spawn(start=True)
async def main(session: VuerSession):
    # Important: You need to set the `stream` option to `True` to start
    # streaming the hand movement.

    session.upsert(
        Hands(
            stream=True,
            key="hands",
            # hideLeft=False,       # hides the hand, but still streams the data.
            # hideRight=False,      # hides the hand, but still streams the data.
            # disableLeft=False,    # disables the left data stream, also hides the hand.
            # disableRight=False,   # disables the right data stream, also hides the hand.
        ),
        to="bgChildren",
    )

    while True:
        await sleep(1)

/**
 * Left and right pose are relative to the wrist transformations.
 */
export type HandsData = {
  left?: Float32Array;       // 25 * 16 values.
  right?: Float32Array;      // 25 * 16 values.
  leftState: HandState;
  rightState: HandState;
};

export type HandState = {
  pinch: boolean;
  squeeze: boolean;
  tap: boolean;

  pinchValue: number;
  squeezeValue: number;
  tapValue: number;
}

                                  ⌈  a0 a4 a8 a12  ⌉
                                  |  a1 a5 a9 a13  |
                                  |  a2 a6 a10 a14 |
                                  ⌊  a3 a7 a11 a15 ⌋