Implement pitch and roll reorientation support (#563)

app/CMakeLists.txt

@@ -22,6 +22,7 @@ add_library( # Sets the name of the library.
              src/main/cpp/BrowserWorld.cpp
              src/main/cpp/Controller.cpp
              src/main/cpp/ControllerContainer.cpp
+             src/main/cpp/DeviceUtils.cpp
              src/main/cpp/ElbowModel.cpp
              src/main/cpp/FadeBlitter.cpp
              src/main/cpp/GestureDelegate.cpp

app/src/googlevr/cpp/DeviceDelegateGoogleVR.cpp

@@ -4,6 +4,7 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 #include "DeviceDelegateGoogleVR.h"
+#include "DeviceUtils.h"
 #include "ElbowModel.h"
 #include "GestureDelegate.h"
 
@@ -86,6 +87,8 @@ struct DeviceDelegateGoogleVR::State {
   std::array<Controller, kMaxControllerCount> controllers;
   ImmersiveDisplayPtr immersiveDisplay;
   bool lastSubmitDiscarded;
+  vrb::Matrix reorientMatrix;
+  bool recentered;
 
   State()
       : gvr(nullptr)
@@ -102,10 +105,12 @@ struct DeviceDelegateGoogleVR::State {
       , far(100.f)
       , sixDofHead(false)
       , lastSubmitDiscarded(false)
+      , recentered(false)
   {
     frameBufferSize = {0,0};
     maxRenderSize = {0,0};
     gestures = GestureDelegate::Create();
+    reorientMatrix = vrb::Matrix::Identity();
   }
 
   gvr_context* GetContext() { return gvr; }
@@ -155,6 +160,7 @@ struct DeviceDelegateGoogleVR::State {
   SetRenderMode(const device::RenderMode aMode) {
     if (aMode != renderMode) {
       renderMode = aMode;
+      reorientMatrix = vrb::Matrix::Identity();
       CreateSwapChain();
     }
   }
@@ -275,6 +281,9 @@ struct DeviceDelegateGoogleVR::State {
         controllerDelegate->SetEnabled(index, true);
         controllerDelegate->SetVisible(index, true);
       }
+
+      recentered = recentered || gvr_controller_state_get_recentered(controllerState);
+
       gvr_quatf ori = gvr_controller_state_get_orientation(controllerState);
       vrb::Quaternion quat(ori.qx, ori.qy, ori.qz, ori.qw);
       controller.transform = vrb::Matrix::Rotation(vrb::Quaternion(ori.qx, ori.qy, ori.qz, ori.qw));
@@ -404,6 +413,11 @@ DeviceDelegateGoogleVR::GetHeadTransform() const {
   return m.cameras[0]->GetHeadTransform();
 }
 
+const vrb::Matrix&
+DeviceDelegateGoogleVR::GetReorientTransform() const {
+  return m.reorientMatrix;
+}
+
 void
 DeviceDelegateGoogleVR::SetClearColor(const vrb::Color& aColor) {
   m.clearColor = aColor;
@@ -460,9 +474,13 @@ DeviceDelegateGoogleVR::StartFrame() {
   m.headMatrix = vrb::Matrix::FromRowMajor(m.gvrHeadMatrix.m);
   m.headMatrix = m.headMatrix.Inverse();
   if (m.renderMode == device::RenderMode::StandAlone) {
+    if (m.recentered) {
+      m.reorientMatrix = DeviceUtils::CalculateReorientationMatrix(m.headMatrix, kAverageHeight);
+    }
     m.headMatrix.TranslateInPlace(kAverageHeight);
   }
   m.UpdateCameras();
+  m.recentered = false;
 
   if (!m.lastSubmitDiscarded) {
     m.frame = GVR_CHECK(gvr_swap_chain_acquire_frame(m.swapChain));
@@ -555,6 +573,7 @@ DeviceDelegateGoogleVR::Resume() {
     VRB_LOG("Resume GVR controller");
     GVR_CHECK(gvr_controller_resume(m.controllerContext));
   }
+  m.reorientMatrix = vrb::Matrix::Identity();
 }
 
 DeviceDelegateGoogleVR::DeviceDelegateGoogleVR(State& aState) : m(aState) {}

app/src/googlevr/cpp/DeviceDelegateGoogleVR.h

@@ -26,6 +26,7 @@ class DeviceDelegateGoogleVR : public DeviceDelegate {
   GestureDelegateConstPtr GetGestureDelegate() override;
   vrb::CameraPtr GetCamera(const device::Eye aWhich) override;
   const vrb::Matrix& GetHeadTransform() const override;
+  const vrb::Matrix& GetReorientTransform() const override;
   void SetClearColor(const vrb::Color& aColor) override;
   void SetClipPlanes(const float aNear, const float aFar) override;
   void SetControllerDelegate(ControllerDelegatePtr& aController) override;

app/src/main/cpp/BrowserWorld.cpp

@@ -136,8 +136,8 @@ struct BrowserWorld::State {
   CreationContextPtr create;
   ModelLoaderAndroidPtr loader;
   GroupPtr rootOpaqueParent;
-  GroupPtr rootOpaque;
-  GroupPtr rootTransparent;
+  TransformPtr rootOpaque;
+  TransformPtr rootTransparent;
   GroupPtr rootController;
   LightPtr light;
   ControllerContainerPtr controllers;
@@ -167,8 +167,8 @@ struct BrowserWorld::State {
     context = RenderContext::Create();
     create = context->GetRenderThreadCreationContext();
     loader = ModelLoaderAndroid::Create(context);
-    rootOpaque = Group::Create(create);
-    rootTransparent = Group::Create(create);
+    rootOpaque = Transform::Create(create);
+    rootTransparent = Transform::Create(create);
     rootController = Group::Create(create);
     light = Light::Create(create);
     rootOpaqueParent = Group::Create(create);
@@ -862,6 +862,9 @@ BrowserWorld::DrawWorld() {
   });
   m.device->StartFrame();
 
+  m.rootOpaque->SetTransform(m.device->GetReorientTransform());
+  m.rootTransparent->SetTransform(m.device->GetReorientTransform());
+
   m.device->BindEye(device::Eye::Left);
   m.drawList->Reset();
   m.rootOpaqueParent->Cull(*m.cullVisitor, *m.drawList);

app/src/main/cpp/DeviceDelegate.h

@@ -44,6 +44,7 @@ class DeviceDelegate {
   virtual GestureDelegateConstPtr GetGestureDelegate() = 0;
   virtual vrb::CameraPtr GetCamera(const device::Eye aWhich) = 0;
   virtual const vrb::Matrix& GetHeadTransform() const = 0;
+  virtual const vrb::Matrix& GetReorientTransform() const = 0;
   virtual void SetClearColor(const vrb::Color& aColor) = 0;
   virtual void SetClipPlanes(const float aNear, const float aFar) = 0;
   virtual void SetControllerDelegate(ControllerDelegatePtr& aController) = 0;

app/src/main/cpp/DeviceUtils.cpp

@@ -0,0 +1,53 @@
+/* -*- Mode: C++; tab-width: 20; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "DeviceUtils.h"
+#include "vrb/Matrix.h"
+#include "vrb/Quaternion.h"
+
+namespace crow {
+
+vrb::Matrix
+DeviceUtils::CalculateReorientationMatrix(const vrb::Matrix& aHeadTransform, const vrb::Vector& aHeightPosition) {
+  const float kPitchUpThreshold = 0.2f;
+  const float kPitchDownThreshold = 0.5f;
+  const float kRollThreshold = 0.35f;
+
+  float rx, ry, rz;
+  vrb::Quaternion quat(aHeadTransform);
+  quat.ToEulerAngles(rx, ry, rz);
+
+  // Use some thresholds to use default roll and yaw values when the rotation is not enough.
+  // This makes easier setting a default orientation easier for the user.
+  if (rx > 0 && rx < kPitchDownThreshold) {
+    rx = 0.0f;
+  } else if (rx < 0 && fabsf(rx) < kPitchUpThreshold) {
+    rx = 0.0f;
+  } else {
+    rx -= 0.05f; // It feels better with some extra margin
+  }
+
+  if (fabsf(rz) < kRollThreshold) {
+    rz = 0.0f;
+  }
+
+  if (rx == 0.0f && rz == 0.0f) {
+    return vrb::Matrix::Identity();
+  }
+
+  quat.SetFromEulerAngles(rx, ry, rz);
+  vrb::Matrix result = vrb::Matrix::Rotation(quat.Inverse());
+
+  // Rotate UI reorientation matrix from origin so user height translation doesn't affect the sphere.
+  result.PreMultiplyInPlace(vrb::Matrix::Position(aHeightPosition));
+  result.PostMultiplyInPlace(vrb::Matrix::Position(-aHeightPosition));
+
+  return result;
+
+}
+
+
+}
+

app/src/main/cpp/DeviceUtils.h

@@ -0,0 +1,23 @@
+/* -*- Mode: C++; tab-width: 20; indent-tabs-mode: nil; c-basic-offset: 2 -*-
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef DEVICE_UTILS_DOT_H
+#define DEVICE_UTILS_DOT_H
+
+#include "vrb/MacroUtils.h"
+#include "vrb/Forward.h"
+
+namespace crow {
+
+class DeviceUtils {
+public:
+  static vrb::Matrix CalculateReorientationMatrix(const vrb::Matrix& aHeadTransform, const vrb::Vector& aHeightPosition);
+private:
+  VRB_NO_DEFAULTS(DeviceUtils)
+};
+
+}
+
+#endif //  DEVICE_UTILS_DOT_H

app/src/noapi/cpp/DeviceDelegateNoAPI.cpp

@@ -36,6 +36,7 @@ struct DeviceDelegateNoAPI::State {
   bool clicked;
   float width, height;
   float near, far;
+  vrb::Matrix reorientMatrix;
   State()
       : renderMode(device::RenderMode::StandAlone)
       , heading(0.0f)
@@ -46,6 +47,7 @@ struct DeviceDelegateNoAPI::State {
       , height(100.0f)
       , near(0.1f)
       , far(1000.0f)
+      , reorientMatrix(vrb::Matrix::Identity())
   {
   }
 
@@ -122,6 +124,11 @@ DeviceDelegateNoAPI::GetHeadTransform() const {
   return m.camera->GetTransform();
 }
 
+const vrb::Matrix&
+DeviceDelegateNoAPI::GetReorientTransform() const {
+  return m.reorientMatrix;
+}
+
 void
 DeviceDelegateNoAPI::SetClearColor(const vrb::Color& aColor) {
   m.clearColor = aColor;

app/src/noapi/cpp/DeviceDelegateNoAPI.h

@@ -26,6 +26,7 @@ class DeviceDelegateNoAPI : public DeviceDelegate {
   GestureDelegateConstPtr GetGestureDelegate() override;
   vrb::CameraPtr GetCamera(const device::Eye) override;
   const vrb::Matrix& GetHeadTransform() const override;
+  const vrb::Matrix& GetReorientTransform() const override;
   void SetClearColor(const vrb::Color& aColor) override;
   void SetClipPlanes(const float aNear, const float aFar) override;
   void SetControllerDelegate(ControllerDelegatePtr& aController) override;

app/src/oculusvr/cpp/DeviceDelegateOculusVR.cpp

@@ -4,6 +4,7 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 #include "DeviceDelegateOculusVR.h"
+#include "DeviceUtils.h"
 #include "ElbowModel.h"
 #include "BrowserEGLContext.h"
 
@@ -117,6 +118,8 @@ struct DeviceDelegateOculusVR::State {
   ElbowModel::HandEnum hand = ElbowModel::HandEnum::Right;
   ControllerDelegatePtr controller;
   ImmersiveDisplayPtr immersiveDisplay;
+  int reorientCount = -1;
+  vrb::Matrix reorientMatrix = vrb::Matrix::Identity();
 
 
   void UpdatePerspective() {
@@ -163,6 +166,8 @@ struct DeviceDelegateOculusVR::State {
     }
     UpdatePerspective();
 
+    reorientCount = vrapi_GetSystemStatusInt(&java, VRAPI_SYS_STATUS_RECENTER_COUNT);
+
     // Send the remote back button java events to the apps
     vrapi_SetPropertyInt(&java, VRAPI_BLOCK_REMOTE_BUTTONS_WHEN_NOT_EMULATING_HMT, 0);
     // Reorient the headset after controller recenter.
@@ -269,6 +274,7 @@ struct DeviceDelegateOculusVR::State {
     controllerState.Header.ControllerType = ovrControllerType_TrackedRemote;
     vrapi_GetCurrentInputState(ovr, controllerID, &controllerState.Header);
 
+    reorientCount = controllerState.RecenterCount;
     const bool triggerPressed = (controllerState.Buttons & ovrButton_A) != 0;
     const bool trackpadPressed = (controllerState.Buttons & ovrButton_Enter) != 0;
     const bool trackpadTouched = (bool) controllerState.TrackpadStatus;
@@ -315,6 +321,9 @@ DeviceDelegateOculusVR::SetRenderMode(const device::RenderMode aMode) {
   for (int i = 0; i < VRAPI_EYE_COUNT; ++i) {
     m.eyeSwapChains[i]->Init(render, m.renderMode, m.renderWidth, m.renderHeight);
   }
+
+  // Reset reorient when exiting or entering immersive
+  m.reorientMatrix = vrb::Matrix::Identity();
 }
 
 device::RenderMode
@@ -352,6 +361,11 @@ DeviceDelegateOculusVR::GetHeadTransform() const {
   return m.cameras[0]->GetHeadTransform();
 }
 
+const vrb::Matrix&
+DeviceDelegateOculusVR::GetReorientTransform() const {
+  return m.reorientMatrix;
+}
+
 void
 DeviceDelegateOculusVR::SetClearColor(const vrb::Color& aColor) {
   m.clearColor = aColor;
@@ -416,7 +430,6 @@ DeviceDelegateOculusVR::StartFrame() {
   ovrMatrix4f matrix = vrapi_GetTransformFromPose(&m.predictedTracking.HeadPose.Pose);
   vrb::Matrix head = vrb::Matrix::FromRowMajor(matrix.M[0]);
 
-
   static const vrb::Vector kAverageHeight(0.0f, 1.7f, 0.0f);
   if (m.renderMode == device::RenderMode::StandAlone) {
     head.TranslateInPlace(kAverageHeight);
@@ -435,8 +448,13 @@ DeviceDelegateOculusVR::StartFrame() {
     m.immersiveDisplay->SetCapabilityFlags(caps);
   }
 
-
+  int lastReorientCount = m.reorientCount;
   m.UpdateControllers(head);
+  bool reoriented = lastReorientCount != m.reorientCount && lastReorientCount > 0 && m.reorientCount > 0;
+  if (reoriented && m.renderMode == device::RenderMode::StandAlone) {
+    m.reorientMatrix = DeviceUtils::CalculateReorientationMatrix(head, kAverageHeight);
+  }
+
   VRB_GL_CHECK(glClearColor(m.clearColor.Red(), m.clearColor.Green(), m.clearColor.Blue(), m.clearColor.Alpha()));
 }
 
@@ -543,6 +561,8 @@ DeviceDelegateOculusVR::EnterVR(const crow::BrowserEGLContext& aEGLContext) {
     vrapi_SetPerfThread(m.ovr, VRAPI_PERF_THREAD_TYPE_RENDERER, gettid());
   }
 
+  // Reset reorientation after Enter VR
+  m.reorientMatrix = vrb::Matrix::Identity();
   vrapi_SetRemoteEmulation(m.ovr, false);
 }
 

app/src/oculusvr/cpp/DeviceDelegateOculusVR.h

@@ -29,6 +29,7 @@ class DeviceDelegateOculusVR : public DeviceDelegate {
   GestureDelegateConstPtr GetGestureDelegate() override { return nullptr; }
   vrb::CameraPtr GetCamera(const device::Eye aWhich) override;
   const vrb::Matrix& GetHeadTransform() const override;
+  const vrb::Matrix& GetReorientTransform() const override;
   void SetClearColor(const vrb::Color& aColor) override;
   void SetClipPlanes(const float aNear, const float aFar) override;
   void SetControllerDelegate(ControllerDelegatePtr& aController) override;

app/src/svr/cpp/DeviceDelegateSVR.cpp

@@ -118,6 +118,7 @@ struct DeviceDelegateSVR::State {
   bool controllerCreated = false;
   ControllerDelegatePtr controller;
   ImmersiveDisplayPtr immersiveDisplay;
+  vrb::Matrix reorientMatrix = vrb::Matrix::Identity();
 
   void UpdatePerspective(const svrDeviceInfo& aInfo) {
     const float fovx = aInfo.targetFovXRad * 0.5f;
@@ -358,6 +359,11 @@ DeviceDelegateSVR::GetHeadTransform() const {
   return m.cameras[0]->GetHeadTransform();
 }
 
+const vrb::Matrix&
+DeviceDelegateSVR::GetReorientTransform() const {
+  return m.reorientMatrix;
+}
+
 void
 DeviceDelegateSVR::SetClearColor(const vrb::Color& aColor) {
   m.clearColor = aColor;

app/src/svr/cpp/DeviceDelegateSVR.h

@@ -29,6 +29,7 @@ class DeviceDelegateSVR : public DeviceDelegate {
   GestureDelegateConstPtr GetGestureDelegate() override { return nullptr; }
   vrb::CameraPtr GetCamera(const device::Eye aWhich) override;
   const vrb::Matrix& GetHeadTransform() const override;
+  const vrb::Matrix& GetReorientTransform() const override;
   void SetClearColor(const vrb::Color& aColor) override;
   void SetClipPlanes(const float aNear, const float aFar) override;
   void SetControllerDelegate(ControllerDelegatePtr& aController) override;