AR pet: Complete code

All of the code for the project can be found below or on GitHub.

^Main.cs^:

using System.Collections.Generic;
using Auki.ConjureKit;
using Auki.ConjureKit.Manna;
using Auki.Ur;
using Auki.Util;
using UnityEngine;
using UnityEngine.UI;
using UnityEngine.XR.ARFoundation;
using UnityEngine.XR.ARSubsystems;

namespace AukiHandTrackerSample
{
    /// <summary>
    /// This sample shows how to use Ur hand tracker for the most common cases:
    /// - Showing hand calibration state.
    /// - Calibrating hand tracker for specific hand size.
    /// - Visualizing the hand landmarks
    /// </summary>
    public class Main : MonoBehaviour
    {
        private const int NumberOfTrackedHands = 1;
        
        /// References needed to initialize the hand tracker
        [SerializeField] private Camera arCamera;
        [SerializeField] private ARSession arSession;
        [SerializeField] private ARRaycastManager arRaycastManager;
        
        [SerializeField] private Text sessionState;
        [SerializeField] private Text sessionID;
    
        [SerializeField] private GameObject raccoon;
        [SerializeField] private Button spawnButton;
    
        private bool qrCodeBool;
        [SerializeField] Button qrCodeButton;
    
        private IConjureKit _conjureKit;
        private Manna _manna;
    
        private ARCameraManager arCameraManager;
        private Texture2D _videoTexture;
        
        [SerializeField] private HandLandmark handLandmarkPrefab;
        
        private HandTracker _handTracker;
        private List<HandLandmark> _handLandmarks;

        private Vector3[] handLandmarksPositions;
        [SerializeField] private Renderer fingertipLandmark;

        public bool hasPlayedDead = false;
        private GameObject raccoonObject;
        private Animator raccoonAnimator;

        private void Start()
        {
            arCameraManager = arCamera.GetComponent<ARCameraManager>();
        
            _conjureKit = new ConjureKit(
                arCamera.transform,
                "YOUR_APP_KEY",
                "YOUR_APP_SECRET");
    
            _manna = new Manna(_conjureKit);
        
            _conjureKit.OnStateChanged += state =>
            {
                sessionState.text = state.ToString();
                ToggleControlsState(state == State.Calibrated);
            };
    
            _conjureKit.OnJoined += session =>
            {
                sessionID.text = session.Id.ToString();
            };
    
            _conjureKit.OnLeft += session =>
            {
                sessionID.text = "";
            };
    
            _conjureKit.OnEntityAdded += CreateRaccoon;
            _conjureKit.Connect();
            
            
            _handTracker = HandTracker.GetInstance();

            // Initialize the hand tracker
            _handTracker.SetARSystem(arSession, arCamera, arRaycastManager);

            // Initialize a list of HandLandmarks to display landmark index and position
            _handLandmarks = new List<HandLandmark>(NumberOfTrackedHands * HandTracker.LandmarksCount);
            for (int i = 0; i < HandTracker.LandmarksCount; i++)
            {
                _handLandmarks.Add(Instantiate(handLandmarkPrefab));
                _handLandmarks[i].SetText(i.ToString());
                _handLandmarks[i].transform.SetParent(arCamera.transform);
            }

            handLandmarksPositions = new Vector3[HandTracker.LandmarksCount];

            _handTracker.OnUpdate += (landmarks, translations, isRightHand, score) =>
            {
                for (int h = 0; h < NumberOfTrackedHands; ++h)
                {
                    if (score[h] > 0)
                    {
                        var handPosition = new Vector3(
                            translations[h * 3 + 0], 
                            translations[h * 3 + 1],
                            translations[h * 3 + 2]);

                        var handLandmarkIndex = h * HandTracker.LandmarksCount * 3;
                        fingertipLandmark.transform.localPosition = handPosition + handLandmarksPositions[8];
                        
                        for (int l = 0; l < HandTracker.LandmarksCount; ++l)
                        {
                            handLandmarksPositions[l] = new Vector3(
                                landmarks[handLandmarkIndex + (l * 3) + 0],
                                landmarks[handLandmarkIndex + (l * 3) + 1],
                                landmarks[handLandmarkIndex + (l * 3) + 2]);
                    
                            // Update the landmarks position 
                            _handLandmarks[l].transform.localPosition = handPosition + handLandmarksPositions[l];
                        }
                        
                        var indexPalmDistance = Vector3.Distance(handLandmarksPositions[8], handLandmarksPositions[0]);
                        var middlePalmDistance = Vector3.Distance(handLandmarksPositions[12], handLandmarksPositions[0]);
                        var ringPalmDistance = Vector3.Distance(handLandmarksPositions[16], handLandmarksPositions[0]);
                        var pinkyPalmDistance = Vector3.Distance(handLandmarksPositions[20], handLandmarksPositions[0]);
                        
                        if (indexPalmDistance > 0.1f && middlePalmDistance < 0.08f && ringPalmDistance < 0.08f && pinkyPalmDistance < 0.08f)
                        {
                            if (!hasPlayedDead)
                            {
                                PlayDead();
                            }
                        }
                    }
                }
            };

            fingertipLandmark.transform.parent = arCamera.transform;
            _handTracker.Start(NumberOfTrackedHands);
        }

        private void Update()
        {
            _handTracker.Update();
            FeedMannaWithVideoFrames();
        }
        
        private void FeedMannaWithVideoFrames()
        {
            var imageAcquired = arCameraManager.TryAcquireLatestCpuImage(out var cpuImage);
            if (!imageAcquired)
            {
                AukiDebug.LogInfo("Couldn't acquire CPU image");
                return;
            }

            if (_videoTexture == null) _videoTexture = new Texture2D(cpuImage.width, cpuImage.height, TextureFormat.R8, false);

            var conversionParams = new XRCpuImage.ConversionParams(cpuImage, TextureFormat.R8);
            cpuImage.ConvertAsync(
                conversionParams,
                (status, @params, buffer) =>
                {
                    _videoTexture.SetPixelData(buffer, 0, 0);
                    _videoTexture.Apply();
                    cpuImage.Dispose();

                    _manna.ProcessVideoFrameTexture(
                        _videoTexture,
                        arCamera.projectionMatrix,
                        arCamera.worldToCameraMatrix
                    );
                }
            );
        }
        
        private void ToggleControlsState(bool interactable)
        {
            if (spawnButton) spawnButton.interactable = interactable;
            if (qrCodeButton) qrCodeButton.interactable = interactable;
        }

        public void ToggleLighthouse()
        {
            qrCodeBool = !qrCodeBool;
            _manna.SetLighthouseVisible(qrCodeBool);
        }

        public void CreateRaccoonEntity()
        {
            if (_conjureKit.GetState() != State.Calibrated)
                return;
            
            Ray ray = arCamera.ScreenPointToRay(new Vector3(Screen.width / 2, Screen.height / 2));
            List<ARRaycastHit> hits = new List<ARRaycastHit>();
            
            if (arRaycastManager.Raycast(ray, hits, TrackableType.PlaneWithinPolygon))
            {
                Quaternion rotation = Quaternion.Euler(0, 180, 0);
                Pose hitPose = new Pose(hits[0].pose.position, rotation);

                _conjureKit.GetSession().AddEntity(
                    hitPose,
                    onComplete: entity => CreateRaccoon(entity),
                    onError: error => Debug.Log(error));
            }
        }

        private void CreateRaccoon(Entity entity)
        {
            if (entity.Flag == EntityFlag.EntityFlagParticipantEntity) return;

            var pose = _conjureKit.GetSession().GetEntityPose(entity);
            Instantiate(raccoon, pose.position, pose.rotation);
            GameObject.Find("SpawnButton").SetActive(false); // Remove spawn button
        }

        public void PlayDead()
        {
            hasPlayedDead = true;
            raccoonObject = GameObject.Find("Raccoon Cub PA(Clone)");
            raccoonAnimator = raccoonObject.GetComponent<Animator>();
            raccoonAnimator.SetTrigger("PlayDead");
        }
        
        public void GetUp()
        {
            hasPlayedDead = false;
            raccoonObject = GameObject.Find("Raccoon Cub PA(Clone)");
            raccoonAnimator = raccoonObject.GetComponent<Animator>();
            raccoonAnimator.SetTrigger("GetUp");
        }
    }
}

^TouchableByHand.cs^:

using System.Collections;
using System.Collections.Generic;
using UnityEngine;

namespace AukiHandTrackerSample
{
    public class TouchableByHand : MonoBehaviour
    {
        public Main mainScript;
        private GameObject raccoonObject;
        private Animator raccoonAnimator;

        private void Awake()
        {
            mainScript = FindObjectOfType<Main>();
        }

        private void OnTriggerEnter(Collider other)
        {
            if (mainScript == null)
            {
                mainScript = FindObjectOfType<Main>();
                if (mainScript == null)
                {
                    Debug.LogError("Main script not found");
                    return;
                }
            }

            if (mainScript.hasPlayedDead)
            {
                mainScript.GetUp();
            }
        }
    }
}

The full project can be found on GitHub.

Need a refresher on the essentials?

Check out more lessons, DIY kits and essentials reading material at the developer learning centre homepage.

ポーズメッシュを構築するのにサポートが必要ですか?

プロジェクトをスタートさせるためにAUKIトークンの助成金を申請し、Auki Labsチームと直接連携して、あなたのクリエイションをマーケットへ。選ばれた申請者は最大10万米ドル相当のAUKIトークンの助成を受け、アウキラボチームによる開発、マーケティング支援を受けることができます。