Control Cesium Camera with device orientation

[zeller.alex...@gmail.com]

1. A concise explanation of the problem you're experiencing.

Hey Guys,

I´m trying to control the position of the camera with the HTML5 device orientation (similar to https://groups.google.com/forum/#!topic/cesium-dev/cr2P2wfOwl4). The idea is that a user can move around his smartphone and
see for example terrain as if the camera would point out of the back of the device. I´ve searched around the web for quite a bit and found some useful sites (https://dev.opera.com/articles/w3c-device-orientation-usage/, http://www.users.on.net/~wallala/Cesium/Apps/Soarer/) but unfortunately I can not get it to work properly.

With the below code I can move the camera around by moving the device but the movements are not correct. Since I am stuck and not sure where exactly my problem is I thought I post this question and see if you guys can help me.

////////

var viewer = new Cesium.Viewer('cesiumContainer');
var scene = viewer.scene;
var camera = viewer.camera;

var terrainProvider = Cesium.createWorldTerrain();

function flyToLocation() {

// Create callback for browser's geolocation
function fly(position) {
viewer.camera.flyTo({
destination : Cesium.Cartesian3.fromDegrees(position.coords.longitude, position.coords.latitude, 10.0)
});
}

// Ask browser for location, and fly there.
navigator.geolocation.getCurrentPosition(fly);
};

var currentScreenOrientation = window.orientation || 0; // active default

if (window.DeviceOrientationEvent) {
window.addEventListener('deviceorientation', onDeviceOrientationChanged, false);
}


/**
* Rotation Matrix functions
* Convert Yaw/alpha/Z, Pitch/beta/X, Roll/gamma/Y to and from rotation matrix, apply transformations
*
* @author Rich Tibbett <https://dev.opera.com/articles/w3c-device-orientation-usage/>, Nghia Ho <http://nghiaho.com/?page_id=846>, Derek Wee
* @copyright
* @version 0.1
* @license
*/

var degtorad = Math.PI / 180; // Degree-to-Radian conversion
var worldOrientation = 90;
var screenOrientation = 90;

//R.1: Converting deviceorientation angles to a Rotation Matrix representation
function getBaseRotationMatrix( alpha, beta, gamma ) {
var _x = beta ? beta * degtorad : 0; // beta value
var _y = gamma ? gamma * degtorad : 0; // gamma value
var _z = alpha ? alpha * degtorad : 0; // alpha value

var cX = Math.cos( _x );
var cY = Math.cos( _y );
var cZ = Math.cos( _z );
var sX = Math.sin( _x );
var sY = Math.sin( _y );
var sZ = Math.sin( _z );

//
// ZXY-ordered rotation matrix construction.
//

var m11 = cZ * cY - sZ * sX * sY;
var m12 = - cX * sZ;
var m13 = cY * sZ * sX + cZ * sY;

var m21 = cY * sZ + cZ * sX * sY;
var m22 = cZ * cX;
var m23 = sZ * sY - cZ * cY * sX;

var m31 = - cX * sY;
var m32 = sX;
var m33 = cX * cY;

return [
m11, m12, m13,
m21, m22, m23,
m31, m32, m33
];
};

//R.2: Fixing our rotation matrix frame relative to the current screen orientation
function getScreenTransformationMatrix( screenOrientation ) {
var orientationAngle = screenOrientation ? screenOrientation * degtorad : 0;

var cA = Math.cos( orientationAngle );
var sA = Math.sin( orientationAngle );

// Construct our screen transformation matrix
var r_s = [
cA, -sA, 0,
sA, cA, 0,
0, 0, 1
];

return r_s;
}

//R.3: Fix our rotation matrix frame relative to our application’s world orientation (rotation around x-axis)
function getWorldTransformationMatrix() {
var x = 90 * degtorad;

var cA = Math.cos( x );
var sA = Math.sin( x );

// Construct our world transformation matrix
var r_w = [
1, 0, 0,
0, cA, -sA,
0, sA, cA
];

return r_w;
}

//R.4: Computing our final rotation matrix representation
function matrixMultiply( a, b ) {
var final = [];

final[0] = a[0] * b[0] + a[1] * b[3] + a[2] * b[6];
final[1] = a[0] * b[1] + a[1] * b[4] + a[2] * b[7];
final[2] = a[0] * b[2] + a[1] * b[5] + a[2] * b[8];

final[3] = a[3] * b[0] + a[4] * b[3] + a[5] * b[6];
final[4] = a[3] * b[1] + a[4] * b[4] + a[5] * b[7];
final[5] = a[3] * b[2] + a[4] * b[5] + a[5] * b[8];

final[6] = a[6] * b[0] + a[7] * b[3] + a[8] * b[6];
final[7] = a[6] * b[1] + a[7] * b[4] + a[8] * b[7];
final[8] = a[6] * b[2] + a[7] * b[5] + a[8] * b[8];

return final;
}

//Returns a 3 x 3 rotation matrix as an array
function computeMatrix(alpha, beta, gamma, currentScreenOrientation) {
var rotationMatrix = getBaseRotationMatrix(
alpha,
beta,
gamma
); // R

var screenTransform = getScreenTransformationMatrix( currentScreenOrientation ); // r_s

var screenAdjustedMatrix = matrixMultiply( rotationMatrix, screenTransform ); // R_s

var worldTransform = getWorldTransformationMatrix(); // r_w

var finalMatrix = matrixMultiply( screenAdjustedMatrix, worldTransform ); // R_w

return finalMatrix; // [ m11, m12, m13, m21, m22, m23, m31, m32, m33 ]
}

//Decomposes a 3 x 3 rotation matrix into an array containing Yaw/alpha/Z Pitch/beta/X Roll/gamma/Y
function getYawPitchRoll(rotationMatrix) {
var rm11 = rotationMatrix[0]; var rm12 = rotationMatrix[1]; var rm13 = rotationMatrix[2];
var rm21 = rotationMatrix[3]; var rm22 = rotationMatrix[4]; var rm23 = rotationMatrix[5];
var rm31 = rotationMatrix[6]; var rm32 = rotationMatrix[7]; var rm33 = rotationMatrix[8];

var yaw = Math.atan2(rm21, rm11);
var pitch = Math.atan2(rm32, rm33);
var roll = Math.atan2(rm31, Math.sqrt(Math.pow(rm32,2) + Math.pow(rm33,2)));

return [yaw, pitch, roll]; //[yaw, pitch, roll]
}


function onDeviceOrientationChanged(eventData) {
var beta = eventData.beta;
var gamma = eventData.gamma;
var alpha = eventData.alpha;

var matrix = computeMatrix(alpha, beta, gamma, screenOrientation)

var matrix = getYawPitchRoll(matrix);

console.log(matrix);


var right = Cesium.Matrix3.getRow(matrix, 0, new Cesium.Cartesian3);
var up = Cesium.Matrix3.getRow(matrix, 1, new Cesium.Cartesian3);
var direction = Cesium.Matrix3.getRow(matrix, 2, new Cesium.Cartesian3);


viewer.camera.setView({
orientation: {
heading : matrix[0],
pitch : matrix[1],
roll : matrix[2]
}
});

};

/////

Some notes: As you can see the code to get the rotation matrix is not by me. To test I use the Google Chrome Developer Tools Sensors to simulate the movement of the device. The Cesium version is 1.44.

Thanks for your help!

[Gabby Getz]

Hey there,

Since you already have the rotation matrix, I don't think the interim steps converting to heading, pitch, and roll are necessary.

Instead you can use Camera.setView with the endTransform option instead of orientation. You can construct the Matrix4 with Matrix4.fromRotationTranslation since you have both the translation (position) and the rotation matrix.

Otherwise to troubleshoot, make sure the matrix you are creating is passing the elements in the correct order (Cesium expects column-major order).

Sounds like an interesting project! Let us know if you have a demo once you get things working.

Thanks,

Gabby

[zeller.alex...@gmail.com]

Hey Gabby,

thanks for your help. I followed your advice and tried to set the view with the endTransform option. Unfortunately now the camera won´t move at all with the change of the device orientation. Again not sure what I am doing wrong since there are no error messages. Heres a code snippet:

function onDeviceOrientationChanged(eventData) {
var beta = eventData.beta;
var gamma = eventData.gamma;
var alpha = eventData.alpha;

var matrix = computeMatrix(alpha, beta, gamma, screenOrientation)

var rotationmatrix = Cesium.Matrix3.fromArray(matrix);

var matrix4 = new Cesium.Matrix4.fromRotationTranslation(rotationmatrix, destination, matrix4);

console.log(matrix4);

viewer.camera.setView({
endTransform: matrix4
});

};

Thanks again. Sorry if I´m doing some beginner´s mistake but im pretty new to Cesium.

[zeller.alex...@gmail.com]

One mistake I corrected was the order of the matrix you already mentioned. The code from Rich Tibbett creates the rotation matrix array in a row-major order. I fixed it by using

var rotationmatrix = new Cesium.Matrix3.fromRowMajorArray(matrix);

instead of

var rotationmatrix = Cesium.Matrix3.fromArray(matrix);

but this does not change the problem of the non-moving camera.

[zeller.alex...@gmail.com]

I experienced some strange behaviour. When I load the site the camera flies as expected to my location. The camera controls with the mouse work as expected with pan, rotate and zoom. If I then simulate a movement of the device via the Chrome developer tools sensor the camera controls don´t work the same way anymore. I cannot drag and by clicking and moving the mouse I rotate the camera. If I rotated the camera before I simulated movement the camera even changes position slightly.

I hope this does not sound confusing. If anyone has some ideas of what I am doing what or what I could try I would really appreciate it. Thank you.

[zeller.alex...@gmail.com]

Heres a Sandcastle-Link. Funnily enough the behaviour here es also a little bit different but I guess that because of the Sandcastle itself.

https://cesiumjs.org/Cesium/Build/Apps/Sandcastle/#c=jVj9Uts4EH8VDf80UMfGcaAcUNpc4KbMQGECLR9NpyNsJXFxrIwkJ6HXztxr3Ovdk9zqK5Hi0BYGbO3ub3el3ZVWnmKGpjmZEYZeo5LMUJfwvBqHHxWt8SJVwy4tBc5Lwl5sHvTLKWB4SkoCEI0N1dCwUjwmDC95egxMzc4IF3mJRU5L32QXMwFvuEwam0p6UJWpEhsUT9f0jKYK1NhEfwNXyL8oQl1GsCBgtCgecPqIBpShB0ZnHJzlaEhoYXBS3tXYmFCey4HVJzx/Q2W0YVh94bq9X3c5HDA6PiZDRghfaA5TSlnGw4KWw1xUGQlQjQUaNSfeDrc3tbUfav7iR780k+zwRzspNUE7pwDhMpOTQWJEGAmleImn+RALykJn8vAuuhVjpBSXxoMGwKQdZWVlhRfBSjXmKoV5lRcsh3cbuVleZnQWUof4/TvaPkDgL4ZVnhII9QBXhYxUCcTrd6dXqHN2dnFzhe4uPqDrC/Th6gTIJ+jy3cX7E0noXry/7l2cKWL35Or0wznqds5Peh308fTkpl/mA9Qwho/JNE+J49PJFN5MMI0MzjJFPcu5gASFfM4UynH6RYBoWdPVHeFySLIADXDByWKZZNZFW1v9cgv1qJn0ORYsnyObWlwyoV6mhAl0h2cRLiYjHN0H6DIX6Sh6IAJHtwHgiyIa4vEYR3dIUB1IyCHErOKxUgwhnkxkgBkuOcR+jK0VaegtrsQIEqKXpyN0nT+AdoEOR0JM+H4UwWRDOpHJnNJxBLmapwXh0SxJm3odms5CNCuOhyQ6CtD74SjH6B3VikBPKQkjqrS8mYDUlzx7vdfeBdljSLtHdEOI8ialkyeWD0cCqSGsAZcz2Q5jNS7AZMmJZEbL7WAIuYozyChYyVF4eYoiFO/pPNIF1RS02cMZlBlK1cJyVc4SPaOsyPy8/GN7sUXVk1YxZTb2wnjfhikvh6iWFxCQISyWCk0t2IxMGOFW1NmpoMz+xJxYeS3eQCoHAiRjHyAVdGQSVTr6ZQ6eSR5Cb8xza7ku+6amFGOKi4oscE+A09remOcanGb4wG8AVD4BUD/XADXDAg00vbWBSilvSNf1XqV4dz7vyeXd+7xvDo8vdPK8XNHJ73yeq5Pf+zyjU22ZZuO8v72DHM8gSbPVwpK5xAWrVODCBcroHscxKAevt+S0mtLWlnQU/t0dLIVaINSUiwL0e4eeSPCdomrcS61Lg61YK3bErMSqjVZr4citQ5UWtHLpnnVU4l0LSew46ChNpFLu6EuUx7dKi1HAiKhYiT7Jd1iOAMEPzNg8k0DRW4beMvSWoSeGnhh6kkjyZ7uRygJs7aO/8rksPlqxWnQGDI5hKDR5QMJRAmUIR5w9j0xtI6de/SLUB9a1t2faYqzvC04tOho7cgOQy1STf7OGtlJATsV0vMyvGXDyuePl83pJ0/XY3FVrZ1bDPyLMQlrl7AsH7SqYaUfFpMn1c1tFzAwMzxC37UA9UGxj6KQH6HWDmqig/mZEpZQ83XLdd/z3z79c7+ju3FFjoQgzWsExOW/iec43/YjfSNzagDvRnasjwInV83GaPx+Y+fOR0N7/IhAzG4jYXV5vxXUYZIQcog5R2lkbhZkbhbY828aTStjqGkDrWtQi8uwxpvnn0Lvl0HvA+QVnl1slWh3MwrqhCJ+2P8tzRT620IN8vIRRrEeJHrX0aFcBDSz2YLEHa3uwVy6s5cFaHmzHg+35biYamHhutj03d9a42fZgsQdrezDPzR0P1vJgOx5sxc1dDdz13Hzlubm3xs1XHiz2YG0P5rm558FaHmzHg+2t5J7CO9mnqBy6pgQqJaklHQaWLGWGn5yMS1W6ElOy9X4pePYq4qQl89ou2RytbcekuLYg35QR+aLsyJdN1f70llsD908TrfhnR8yz1yatGvbMVeWd7GsFd5Rs4fpqCfpzC2o+GaddzTNvS9Re/2SbNK7NlgpUZJ/1aJ3jwapR69asnjIaofifkGowVHchWwuk+grVVMiOAql2QvUSslH5bC+tNnmev8A1iLz+HWOBnTRRnfRrtGCFkrDY7HW/7LIVZcHXbbHLV5QDZD9KqH7KLtqv87rWSuhzZzWnFxqdLyZaq/720KOzc/yVso4srIaWXmr6XS3r0csZtQFax7aNB9qGCn6hvyT4dgP3409gVW46S+d9geFEyM9Q9hsMKZd5tW/B6juJvZ8Le0eXvxvBxiEXTwU56pfy9HybjycU7uQVKxphGAkynoCTcB1+qNJHIsKUc6nnMLKgwyyfIrjo9jdWPoL1N1BaYM6BM6iK4ir/RvobR4cRyHuwgsKltRxewB2zgM0OREbx0ZkmhmF4GMGwjhKUFg+YORr/Bw

[zeller.alex...@gmail.com]

If anyone is interested I´ve managed to get it working for the most part. Heres the link to the gist: https://gist.github.com/AlexZeller/d9627fa25901580d1cc02dcefacfccd4

You can copy the code into an empty Sandcastle and it should work. If you view it on a smartphone you then should be able the move around the device and in theory the camera should move as if it would point out of the back.

Ih haven´t tested if the orientation is correct, and if the terrain that appears on the screen really is behind the phone. And theres still a problem if your phone rotates the screen.

[Gabby Getz]

Hi,

For the camera controls not working after you set the matrix, what I usually suggest is the following line to restore camera controls.

viewer.camera.lookAtTransform(Cesium.Matrix4.IDENTITY);

However, I think that will override your device orientation. Any reason why you still need the mouse controls if you are using the device to control orientation?

You can also override the default camera controls with your own camera controls.

For the slightly different behavior in sandcastle, are you using the latest version of Cesium for local development? Otherwise, it may have to do with the way the sandcastle app is set up, with panels for the live coding and a frame to run cesium itself.

Thanks! This looks like a cool use of device orientation and can have a lot of AR applications!

Gabby