Hand tracker: Complete code

All of the code for the project can be found below or on GitHub.

using UnityEngine;
using Auki.ConjureKit;
using UnityEngine.UI;
using Auki.ConjureKit.Manna;
using Auki.Ur;
using Auki.Util;
using UnityEngine.XR.ARFoundation;
using UnityEngine.XR.ARSubsystems;

public class ConjureKitManager : MonoBehaviour
{
    [SerializeField] private Camera arCamera;
    [SerializeField] private ARSession arSession;
    [SerializeField] private ARRaycastManager arRaycastManager;

    [SerializeField] private Text sessionState;
    [SerializeField] private Text sessionID;
    
    [SerializeField] private GameObject cube;
    [SerializeField] private Button spawnButton;

    [SerializeField] Button qrCodeButton;
    private bool qrCodeBool;
    
    private IConjureKit _conjureKit;
    private Manna _manna;
    
    private ARCameraManager arCameraManager;
    private Texture2D _videoTexture;
    
    [SerializeField] private Renderer fingertipLandmark;
    private HandTracker _handTracker;
    private bool landmarksVisualizeBool = true;
    
    [SerializeField] private AROcclusionManager arOcclusionManager;
    private bool occlusionBool = true;

    void Start()
    {
        arCameraManager = arCamera.GetComponent<ARCameraManager>();
        
        _conjureKit = new ConjureKit(
            arCamera.transform,
            "YOUR_APP_KEY",
            "YOUR_APP_SECRET");

        _manna = new Manna(_conjureKit);
        
        _conjureKit.OnStateChanged += state =>
        {
            if (state == State.JoinedSession)
            {
                Debug.Log("State.JoinedSession  " + Time.realtimeSinceStartup);
            }

            if (state == State.Calibrated)
            {
                Debug.Log("State.Calibrated  " + Time.realtimeSinceStartup);
            }

            sessionState.text = state.ToString();
            ToggleControlsState(state == State.Calibrated);
        };

        _conjureKit.OnJoined += session =>
        {
            Debug.Log("OnJoined " + Time.realtimeSinceStartup);
            sessionID.text = session.Id.ToString();
        };

        _conjureKit.OnLeft += session =>
        {
            sessionID.text = "";
        };

        _conjureKit.OnEntityAdded += CreateCube;
        _conjureKit.Connect();
        
        _handTracker = HandTracker.GetInstance();
        _handTracker.SetARSystem(arSession, arCamera, arRaycastManager);
        
        _handTracker.OnUpdate += (landmarks, translations, isRightHand, score) =>
        {
            if (score[0] > 0)
            {
                var handPosition = new Vector3(
                    translations[0],
                    translations[1],
                    translations[2]);

                var pointerLandmarkIndex = 8 * 3; // Index fingertip
                var pointerLandMarkPosition = new Vector3(
                    landmarks[pointerLandmarkIndex + 0],
                    landmarks[pointerLandmarkIndex + 1],
                    landmarks[pointerLandmarkIndex + 2]);

                fingertipLandmark.enabled = true;
                fingertipLandmark.transform.localPosition = handPosition + pointerLandMarkPosition;
            }
            else
            {
                fingertipLandmark.enabled = false;
            }
        };
        
        _handTracker.Start();
        _handTracker.ShowHandMesh();
    }
    
    private void Update()
    {
        FeedMannaWithVideoFrames();
        _handTracker.Update();
    }
    
    private void FeedMannaWithVideoFrames()
    {
        var imageAcquired = arCameraManager.TryAcquireLatestCpuImage(out var cpuImage);
        if (!imageAcquired)
        {
            AukiDebug.LogInfo("Couldn't acquire CPU image");
            return;
        }

        if (_videoTexture == null) _videoTexture = new Texture2D(cpuImage.width, cpuImage.height, TextureFormat.R8, false);

        var conversionParams = new XRCpuImage.ConversionParams(cpuImage, TextureFormat.R8);
        cpuImage.ConvertAsync(
            conversionParams,
            (status, @params, buffer) =>
            {
                _videoTexture.SetPixelData(buffer, 0, 0);
                _videoTexture.Apply();
                cpuImage.Dispose();

                _manna.ProcessVideoFrameTexture(
                    _videoTexture,
                    arCamera.projectionMatrix,
                    arCamera.worldToCameraMatrix
                );
            }
        );
    }
    private void ToggleControlsState(bool interactable)
    {
        if (spawnButton) spawnButton.interactable = interactable;
        if (qrCodeButton) qrCodeButton.interactable = interactable;
    }
    
    public void ToggleLighthouse()
    {
        qrCodeBool = !qrCodeBool;
        _manna.SetLighthouseVisible(qrCodeBool);
    }
    
    public void ToggleHandLandmarks()
    {
        landmarksVisualizeBool = !landmarksVisualizeBool;

        if (landmarksVisualizeBool)
        {
            _handTracker.ShowHandMesh();
        }
        else
        {
            _handTracker.HideHandMesh();
        }
    }
    
    public void ToggleOcclusion()
    {
        occlusionBool = !occlusionBool;

        arOcclusionManager.requestedHumanDepthMode = occlusionBool ? HumanSegmentationDepthMode.Fastest : HumanSegmentationDepthMode.Disabled;
        arOcclusionManager.requestedHumanStencilMode = occlusionBool ? HumanSegmentationStencilMode.Fastest : HumanSegmentationStencilMode.Disabled;
        arOcclusionManager.requestedEnvironmentDepthMode = occlusionBool ? EnvironmentDepthMode.Fastest : EnvironmentDepthMode.Disabled;
    }

    public void CreateCubeEntity()
    {
        if (_conjureKit.GetState() != State.Calibrated)
            return;

        Vector3 position = arCamera.transform.position + arCamera.transform.forward * 0.5f;
        Quaternion rotation = Quaternion.Euler(0, arCamera.transform.eulerAngles.y, 0);

        Pose entityPos = new Pose(position, rotation);

        _conjureKit.GetSession().AddEntity(
            entityPos,
            onComplete: entity => CreateCube(entity),
            onError: error => Debug.Log(error));
    }

    private void CreateCube(Entity entity)
    {
        if (entity.Flag == EntityFlag.EntityFlagParticipantEntity) return;

        var pose = _conjureKit.GetSession().GetEntityPose(entity);
        Instantiate(cube, pose.position, pose.rotation);
    }
}

The full code for this tutorial can be found on GitHub on the ^tutorial/handtracker^ branch.

The complete project with all parts and the latest packages is on the master branch of the same repo.

Need a refresher on the essentials?

Check out more lessons, DIY kits and essentials reading material at the developer learning centre homepage.

Want to build on the posemesh and need a helping hand?

Apply for a grant of AUKI tokens to get your project off the ground, and work directly with the Auki Labs team to get your creation to market. Successful applicants may be granted up to 100k USD worth of AUKI tokens, and development and marketing support from the Auki Labs team.