Skip to content

Commit

Permalink
eye tracking attempt...
Browse files Browse the repository at this point in the history
  • Loading branch information
@zakaton
zakaton committed Jan 12, 2024
1 parent ec40b73 commit dc0f1c6
Show file tree
Hide file tree
Showing 5 changed files with 168 additions and 76 deletions.
46 changes: 39 additions & 7 deletions examples/ar-session/eye-tracking/index.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
<script src="../../src/aframe/aframe-orbit-controls-component.min.js"></script>

<script type="module" src="./script.js"></script>
<script type="module" src="../src/lighting.js"></script>
</head>
<style>
body {
Expand Down Expand Up @@ -58,8 +59,26 @@
top: 1rem;
left: 1rem;
}
#eyeTrackingOverlay {
top: 0;
left: 0;
background-color: transparent;
width: 100%;
height: 100%;
}
#eyeTrackingCursor {
width: 1rem;
height: 1rem;
background-color: red;
border-radius: 50%;
display: none;
position: relative;
}
</style>
<body>
<div id="eyeTrackingOverlay" class="overlay">
<div id="eyeTrackingCursor"></div>
</div>
<div id="navOverlay" class="overlay">
<div>
<small><a href="/">home</a></small>
Expand All @@ -79,6 +98,16 @@
<button id="toggleDebug" disabled>show debug</button>
<button id="toggleShowCamera" disabled>show camera</button>
</div>
<div>
<button id="resetEyeCalibration" disabled>reset eye-tracking calibration</button>
<button id="toggleEyeCalibration" disabled>calibrate eye-tracking</button>
</div>
<div>face <span id="faceSpan"></span></div>
<div>lookAtPoint <span id="lookAtPointSpan"></span></div>
<div>lookAtPointEntity <span id="lookAtPointEntitySpan"></span></div>
<div>lookAtPointInWorld <span id="lookAtPointInWorldSpan"></span></div>
<div>lookatPointInCamera <span id="lookAtPointInCameraSpan"></span></div>
<div>eyeTrackingPoint <span id="eyeTrackingPointSpan"></span></div>
</div>
</div>

Expand All @@ -97,15 +126,18 @@
position="-0.5 1 1"
></a-entity>

<a-camera
id="camera"
look-controls="enabled: false;"
wasd-controls="enabled: false;"
position="0 0 0"
></a-camera>
<a-camera id="camera" look-controls="enabled: false;" wasd-controls="enabled: false;" position="0 0 0">
<a-ring
color="black"
position="0 0 -0.01"
radius-inner="0.0001"
radius-outer="0.0002"
material="shader: flat;"
></a-ring>
</a-camera>

<a-entity id="face" position="0 0 -0.4">
<a-sphere radius="0.04" color="green"></a-sphere>
<a-sphere radius="0.03" color="green" visible="true"></a-sphere>

<a-entity id="leftEye" position="-0.03 0.04 00.02">
<a-sphere radius="0.01" color="white">
Expand Down
160 changes: 109 additions & 51 deletions examples/ar-session/eye-tracking/script.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,8 @@ ARSessionManager.addEventListener("isRunning", (event) => {
const isRunning = event.message.isRunning;
console.log("isRunning", isRunning);

eyeTrackingCursor.style.display = isRunning ? "block" : "none";

runButton.disabled = isRunning;
pauseButton.disabled = !isRunning;
});
Expand All @@ -65,7 +67,7 @@ const toggleDebugButton = document.getElementById("toggleDebug");
toggleDebugButton.addEventListener("click", () => {
const newIsDebugEnabled = !isDebugEnabled;
ARSessionManager.setDebugOptions({
showFeaturePoints: newIsDebugEnabled,
showAnchorOrigins: newIsDebugEnabled,
});
});
toggleDebugButton.disabled = !ARSessionManager.isSupported;
Expand All @@ -74,7 +76,7 @@ ARSessionManager.addEventListener("debugOptions", (event) => {
/** @type {ARSDebugOptions} */
const debugOptions = event.message.debugOptions;
console.log("debugOptions", debugOptions);
isDebugEnabled = debugOptions.showFeaturePoints;
isDebugEnabled = debugOptions.showAnchorOrigins;
toggleDebugButton.innerText = isDebugEnabled ? "hide debug" : "show debug";
});

Expand All @@ -92,16 +94,86 @@ ARSessionManager.addEventListener("showCamera", (event) => {
toggleShowCameraButton.innerText = showCamera ? "hide camera" : "show camera";
});

/** @typedef {import("../../src/three/three.module.min.js").Vector2} Vector2 */
/** @typedef {import("../../src/three/three.module.min.js").Vector3} Vector3 */
/** @typedef {import("../../src/three/three.module.min.js").Quaternion} Quaternion */
/** @typedef {import("../../src/three/three.module.min.js").Euler} Euler */
/** @typedef {import("../../src/three/three.module.min.js").Matrix4} Matrix4 */
/** @typedef {import("../../src/three/three.module.min.js").Box2} Box2 */

/** @type {Matrix4} */
const cameraMatrix = new THREE.Matrix4();
const cameraMatrixInverse = new THREE.Matrix4();
const aframeCamera = document.getElementById("camera");
var threeCamera;
aframeCamera.addEventListener("loaded", () => {
threeCamera = aframeCamera?.components?.camera?.camera;
console.log("threeCamera", threeCamera);
});
var latestFocalLength;
ARSessionManager.addEventListener("camera", (event) => {
/** @type {import("../../../src/ARSessionManager.js").ARSCamera} */
const camera = event.message.camera;

//aframeCamera.object3D.position.set(...camera.position);
//cameraMatrix.setPosition(...camera.position);

if (ARSessionManager.cameraMode == "ar") {
const quaternion = new THREE.Quaternion(...camera.quaternion);
aframeCamera.object3D.quaternion.copy(quaternion);
cameraMatrix.makeRotationFromQuaternion(quaternion);
} else {
/** @type {Euler} */
const euler = new THREE.Euler(...camera.eulerAngles);
euler.z += Math.PI / 2;
aframeCamera.object3D.rotation.copy(euler);
cameraMatrix.makeRotationFromEuler(euler);
}
cameraMatrixInverse.copy(cameraMatrix).invert();

if (threeCamera) {
if (latestFocalLength != camera.focalLength) {
threeCamera.setFocalLength(camera.focalLength);
latestFocalLength = camera.focalLength;
}
}

scene.renderer.toneMappingExposure = camera.exposureOffset;
});

const leftEyeEntity = document.getElementById("leftEye");
const rightEyeEntity = document.getElementById("rightEye");
const faceEntity = document.getElementById("face");
const lookAtPointEntity = document.getElementById("lookAtPoint");
var eyeBlinkThreshold = 0.5;
/** @type {Matrix4} */
const faceMatrix = new THREE.Matrix4();
/** @type {Matrix4} */
const faceMatrixInverse = new THREE.Matrix4();

/** @typedef {import("../../src/three/three.module.min.js").Vector3} Vector3 */
/** @typedef {import("../../src/three/three.module.min.js").Quaternion} Quaternion */
/** @typedef {import("../../src/three/three.module.min.js").Euler} Euler */
const eyeTrackingCursor = document.getElementById("eyeTrackingCursor");
/** @type {Vector2} */
const eyeTrackingPoint = new THREE.Vector2();
/** @type {Box2} */
const eyeTrackingRange = new THREE.Box2();

var eyeBlinkThreshold = 0.5;
/** @type {HTMLButtonElement} */
const resetEyeCalibrationButton = document.getElementById("resetEyeCalibration");
resetEyeCalibrationButton.addEventListener("click", () => {
eyeTrackingRange.makeEmpty();
});
resetEyeCalibrationButton.disabled = !ARSessionManager.isSupported;

var calibratingEyeTrackingRange = false;
/** @type {HTMLButtonElement} */
const toggleEyeCalibrationButton = document.getElementById("toggleEyeCalibration");
toggleEyeCalibrationButton.addEventListener("click", () => {
calibratingEyeTrackingRange = !calibratingEyeTrackingRange;
toggleEyeCalibrationButton.innerText = calibratingEyeTrackingRange
? "stop calibrating eye-tracking"
: "calibrate eye-tracking";
});
toggleEyeCalibrationButton.disabled = !ARSessionManager.isSupported;

/** @typedef {import("../../../src/ARSessionManager.js").ARSFaceAnchor} ARSFaceAnchor */
ARSessionManager.addEventListener("faceAnchors", (event) => {
Expand All @@ -114,6 +186,10 @@ ARSessionManager.addEventListener("faceAnchors", (event) => {
/** @type {Quaternion} */
const newQuaternion = new THREE.Quaternion(...faceAnchor.quaternion);

faceMatrix.makeRotationFromQuaternion(newQuaternion);
faceMatrix.setPosition(newPosition);
faceMatrixInverse.copy(faceMatrix).invert();

faceEntity.object3D.position.lerp(newPosition, 0.5);
faceEntity.object3D.quaternion.slerp(newQuaternion, 0.5);

Expand All @@ -132,65 +208,47 @@ ARSessionManager.addEventListener("faceAnchors", (event) => {
rightEyeEntity.object3D.quaternion.slerp(newRightEyeQuaternion, 0.5);

/** @type {Vector3} */
const newLookAtPointPosition = new THREE.Vector3(...faceAnchor.lookAtPoint);
newLookAtPointPosition.multiplyScalar(0.3);
lookAtPointEntity.object3D.position.lerp(newLookAtPointPosition, 0.5);
const lookAtPoint = new THREE.Vector3(...faceAnchor.lookAtPoint);
const newLookAtPointEntityPosition = lookAtPoint.clone().normalize().multiplyScalar(0.3);
lookAtPointEntity.object3D.position.lerp(newLookAtPointEntityPosition, 0.5);

const isLeftEyeClosed = faceAnchor.blendShapes.eyeBlinkLeft > eyeBlinkThreshold;
const showLeftEye = !isLeftEyeClosed;
if (leftEyeEntity.object3D.visible != showLeftEye) {
leftEyeEntity.object3D.visible = showLeftEye;
}

const isRightEyeClosed = faceAnchor.blendShapes.eyeBlinkRight > eyeBlinkThreshold;
const showRightEye = !isRightEyeClosed;
if (rightEyeEntity.object3D.visible != showRightEye) {
rightEyeEntity.object3D.visible = showRightEye;
}
}
});

const aframeCamera = document.getElementById("camera");
var latestFocalLength;
ARSessionManager.addEventListener("camera", (event) => {
/** @type {import("../../../src/ARSessionManager.js").ARSCamera} */
const camera = event.message.camera;

//aframeCamera.object3D.position.set(...camera.position);

if (ARSessionManager.cameraMode == "ar") {
aframeCamera.object3D.quaternion.set(...camera.quaternion);
} else {
/** @type {Euler} */
const euler = new THREE.Euler(...camera.eulerAngles);
euler.z += Math.PI / 2;
aframeCamera.object3D.rotation.copy(euler);
}

const threeCamera = aframeCamera?.components?.camera?.camera;
if (threeCamera) {
if (latestFocalLength != camera.focalLength) {
threeCamera.setFocalLength(camera.focalLength);
latestFocalLength = camera.focalLength;
const lookAtPointInWorld = lookAtPoint.clone().applyMatrix4(faceMatrix);
const lookAtPointEntityInWorld = new THREE.Vector3();
lookAtPointEntity.object3D.getWorldPosition(lookAtPointEntityInWorld);
const lookAtPointInCamera = lookAtPointInWorld.clone().applyMatrix4(cameraMatrixInverse);
/** @type {Vector2} */
const lookAtPointInCameraPoint = new THREE.Vector2(lookAtPointInCamera.x, lookAtPointInCamera.y);
if (calibratingEyeTrackingRange && !isLeftEyeClosed && !isRightEyeClosed) {
eyeTrackingRange.expandByPoint(lookAtPointInCameraPoint);
}
}
eyeTrackingRange.getParameter(lookAtPointInCameraPoint, eyeTrackingPoint);

scene.renderer.toneMappingExposure = camera.exposureOffset;
});
eyeTrackingCursor.style.left = `${eyeTrackingPoint.x * 100}%`;
eyeTrackingCursor.style.top = `${(1 - eyeTrackingPoint.y) * 100}%`;

/** @typedef {import("../../../src/ARSessionManager.js").ARSLightEstimate} ARSLightEstimate */

const ambientLight = document.getElementById("ambientLight");
const directionalLight = document.getElementById("directionalLight");

ARSessionManager.addEventListener("lightEstimate", (event) => {
/** @type {ARSLightEstimate} */
const lightEstimate = event.message.lightEstimate;
ambientLight.components.light.light.intensity = lightEstimate.ambientIntensity / 1000;
const lightColor = utils.colorTemperatureToRGB(lightEstimate.ambientColorTemperature);
ambientLight.components.light.light.color.setRGB(...lightColor);
directionalLight.components.light.light.color.setRGB(...lightColor);
if (lightEstimate.primaryLightDirection) {
directionalLight.components.light.light.intensity = lightEstimate.primaryLightIntensity / 1000;
directionalLight.object3D.position.set(...lightEstimate.primaryLightDirection.map((v) => -v));
faceSpan.innerText = JSON.stringify(newPosition.toArray().map((v) => v.toFixed(6)));
lookAtPointSpan.innerText = JSON.stringify(lookAtPoint.toArray().map((v) => v.toFixed(6)));
lookAtPointEntitySpan.innerText = JSON.stringify(lookAtPointEntityInWorld.toArray().map((v) => v.toFixed(6)));
lookAtPointInWorldSpan.innerText = JSON.stringify(lookAtPointInWorld.toArray().map((v) => v.toFixed(6)));
lookAtPointInCameraSpan.innerText = JSON.stringify(lookAtPointInCamera.toArray().map((v) => v.toFixed(6)));
eyeTrackingPointSpan.innerText = JSON.stringify(eyeTrackingPoint.toArray().map((v) => v.toFixed(6)));
}
});
const faceSpan = document.getElementById("faceSpan");
const lookAtPointSpan = document.getElementById("lookAtPointSpan");
const lookAtPointInCameraSpan = document.getElementById("lookAtPointInCameraSpan");
const lookAtPointInWorldSpan = document.getElementById("lookAtPointInWorldSpan");
const lookAtPointEntitySpan = document.getElementById("lookAtPointEntitySpan");
const eyeTrackingPointSpan = document.getElementById("eyeTrackingPointSpan");
1 change: 1 addition & 0 deletions examples/ar-session/face-tracking/index.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
<link rel="shortcut icon" type="image/x-icon" href="favicon.ico" />
<script src="../../src/aframe/aframe-v1.5.0.min.js"></script>
<script type="module" src="./script.js"></script>
<script type="module" src="../src/lighting.js"></script>
</head>
<style>
body {
Expand Down
18 changes: 0 additions & 18 deletions examples/ar-session/face-tracking/script.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -170,21 +170,3 @@ ARSessionManager.addEventListener("camera", (event) => {

scene.renderer.toneMappingExposure = camera.exposureOffset;
});

/** @typedef {import("../../../src/ARSessionManager.js").ARSLightEstimate} ARSLightEstimate */

const ambientLight = document.getElementById("ambientLight");
const directionalLight = document.getElementById("directionalLight");

ARSessionManager.addEventListener("lightEstimate", (event) => {
/** @type {ARSLightEstimate} */
const lightEstimate = event.message.lightEstimate;
ambientLight.components.light.light.intensity = lightEstimate.ambientIntensity / 1000;
const lightColor = utils.colorTemperatureToRGB(lightEstimate.ambientColorTemperature);
ambientLight.components.light.light.color.setRGB(...lightColor);
directionalLight.components.light.light.color.setRGB(...lightColor);
if (lightEstimate.primaryLightDirection) {
directionalLight.components.light.light.intensity = lightEstimate.primaryLightIntensity / 1000;
directionalLight.object3D.position.set(...lightEstimate.primaryLightDirection.map((v) => -v));
}
});
19 changes: 19 additions & 0 deletions examples/ar-session/src/lighting.js
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
import { ARSessionManager, utils } from "../../../src/NativeWebKit.js";

/** @typedef {import("../../../src/ARSessionManager.js").ARSLightEstimate} ARSLightEstimate */

const ambientLight = document.getElementById("ambientLight");
const directionalLight = document.getElementById("directionalLight");

ARSessionManager.addEventListener("lightEstimate", (event) => {
/** @type {ARSLightEstimate} */
const lightEstimate = event.message.lightEstimate;
ambientLight.components.light.light.intensity = lightEstimate.ambientIntensity / 1000;
const lightColor = utils.colorTemperatureToRGB(lightEstimate.ambientColorTemperature);
ambientLight.components.light.light.color.setRGB(...lightColor);
directionalLight.components.light.light.color.setRGB(...lightColor);
if (lightEstimate.primaryLightDirection) {
directionalLight.components.light.light.intensity = lightEstimate.primaryLightIntensity / 1000;
directionalLight.object3D.position.set(...lightEstimate.primaryLightDirection.map((v) => -v));
}
});

0 comments on commit dc0f1c6

Please sign in to comment.