Mesh Debugging implemented with feedback

qrenderdoc/Widgets/ComputeDebugSelector.cpp

@@ -65,6 +65,21 @@ ComputeDebugSelector::~ComputeDebugSelector()
   delete ui;
 }
 
+void ComputeDebugSelector::SetDefaultDispatch(const rdcfixedarray<uint32_t, 3> &group,
+                                              const rdcfixedarray<uint32_t, 3> &thread)
+{
+  QSignalBlocker blockers[6] = {QSignalBlocker(ui->groupX),  QSignalBlocker(ui->groupY),
+                                QSignalBlocker(ui->groupZ),  QSignalBlocker(ui->threadX),
+                                QSignalBlocker(ui->threadY), QSignalBlocker(ui->threadZ)};
+
+  ui->groupX->setValue(group[0]);
+  ui->groupY->setValue(group[1]);
+  ui->groupZ->setValue(group[2]);
+  ui->threadX->setValue(thread[0]);
+  ui->threadY->setValue(thread[1]);
+  ui->threadZ->setValue(thread[2]);
+}
+
 void ComputeDebugSelector::SetThreadBounds(const rdcfixedarray<uint32_t, 3> &group,
                                            const rdcfixedarray<uint32_t, 3> &thread)
 {

qrenderdoc/Widgets/ComputeDebugSelector.h

@@ -40,6 +40,8 @@ class ComputeDebugSelector : public QDialog
   explicit ComputeDebugSelector(QWidget *parent = 0);
   ~ComputeDebugSelector();
 
+  void SetDefaultDispatch(const rdcfixedarray<uint32_t, 3> &group,
+                          const rdcfixedarray<uint32_t, 3> &thread);
   void SetThreadBounds(const rdcfixedarray<uint32_t, 3> &group,
                        const rdcfixedarray<uint32_t, 3> &thread);
 

qrenderdoc/Windows/BufferViewer.cpp

@@ -39,6 +39,7 @@
 #include "Code/QRDUtils.h"
 #include "Code/Resources.h"
 #include "Widgets/CollapseGroupBox.h"
+#include "Widgets/ComputeDebugSelector.h"
 #include "Widgets/Extended/RDLabel.h"
 #include "Widgets/Extended/RDSplitter.h"
 #include "Windows/Dialogs/AxisMappingDialog.h"
@@ -2361,6 +2362,8 @@ BufferViewer::BufferViewer(ICaptureContext &ctx, bool meshview, QWidget *parent)
   m_ModelOut1 = new BufferItemModel(ui->out1Table, false, meshview, this);
   m_ModelOut2 = new BufferItemModel(ui->out2Table, false, meshview, this);
 
+  m_MeshDebugSelector = new ComputeDebugSelector(this);
+
   // we keep the old UI names for serialised layouts compatibility
   QString containerNames[] = {
       lit("vsinData"),
@@ -2440,6 +2443,9 @@ BufferViewer::BufferViewer(ICaptureContext &ctx, bool meshview, QWidget *parent)
   m_DebugVert = new QAction(tr("&Debug this Vertex"), this);
   m_DebugVert->setIcon(Icons::wrench());
 
+  m_DebugMeshThread = new QAction(tr("&Debug Mesh Thread"), this);
+  m_DebugMeshThread->setIcon(Icons::wrench());
+
   m_FilterMesh = new QAction(tr("&Filter to this Meshlet"), this);
   m_FilterMesh->setIcon(Icons::filter());
 
@@ -2458,6 +2464,8 @@ BufferViewer::BufferViewer(ICaptureContext &ctx, bool meshview, QWidget *parent)
   QObject::connect(m_ExportBytes, &QAction::triggered,
                    [this] { exportData(BufferExport(BufferExport::RawBytes)); });
   QObject::connect(m_DebugVert, &QAction::triggered, this, &BufferViewer::debugVertex);
+  QObject::connect(m_DebugMeshThread, &QAction::triggered, this,
+                   &BufferViewer::debugMeshThread);
   QObject::connect(m_RemoveFilter, &QAction::triggered,
                    [this]() { SetMeshFilter(MeshFilter::None); });
   QObject::connect(m_FilterMesh, &QAction::triggered, [this]() {
@@ -2624,6 +2632,9 @@ BufferViewer::BufferViewer(ICaptureContext &ctx, bool meshview, QWidget *parent)
   ui->fixedVars->setTooltipElidedItems(false);
   ui->fixedVars->installEventFilter(this);
 
+  QObject::connect(m_MeshDebugSelector, &ComputeDebugSelector::beginDebug, this,
+                   &BufferViewer::meshDebugSelector_beginDebug);
+
   Reset();
 
   m_Ctx.AddCaptureViewer(this);
@@ -3093,6 +3104,37 @@ void BufferViewer::stageRowMenu(MeshDataStage stage, QMenu *menu, const QPoint &
       menu->addAction(m_RemoveFilter);
       menu->addAction(m_FilterMesh);
       menu->addAction(m_GotoTask);
+
+      const ShaderReflection *shaderDetails =
+          m_Ctx.CurPipelineState().GetShaderReflection(ShaderStage::Mesh);
+
+      m_DebugMeshThread->setEnabled(false);
+
+      if(!m_Ctx.APIProps().shaderDebugging)
+      {
+        m_DebugMeshThread->setToolTip(tr("This API does not support shader debugging"));
+      }
+      else if(!m_Ctx.CurAction() ||
+              !(m_Ctx.CurAction()->flags & (ActionFlags::Drawcall | ActionFlags::MeshDispatch)))
+      {
+        m_DebugMeshThread->setToolTip(tr("No draw call selected"));
+      }
+      else if(!shaderDetails)
+      {
+        m_DebugMeshThread->setToolTip(tr("No mesh shader bound"));
+      }
+      else if(!shaderDetails->debugInfo.debuggable)
+      {
+        m_DebugMeshThread->setToolTip(
+            tr("This shader doesn't support debugging: %1").arg(shaderDetails->debugInfo.debugStatus));
+      }
+      else
+      {
+        m_DebugMeshThread->setEnabled(true);
+        m_DebugMeshThread->setToolTip(QString());
+      }
+      menu->addAction(m_DebugMeshThread);
+
       menu->addSeparator();
 
       m_GotoTask->setEnabled(m_Ctx.CurPipelineState().GetShaderReflection(ShaderStage::Task));
@@ -6638,6 +6680,133 @@ void BufferViewer::debugVertex()
   m_Ctx.AddDockWindow(s->Widget(), DockReference::AddTo, this);
 }
 
+void BufferViewer::debugMeshThread()
+{
+  if(!m_Ctx.IsCaptureLoaded())
+    return;
+
+  const ActionDescription *action = m_Ctx.CurAction();
+  if(!action)
+    return;
+
+  if(!m_CurView)
+    return;
+
+  QModelIndex idx = m_CurView->selectionModel()->currentIndex();
+
+  if(!idx.isValid())
+  {
+    GUIInvoke::call(this, [this]() {
+      RDDialog::critical(this, tr("Error debugging"),
+                         tr("Error debugging meshlet - make sure a valid meshlet is selected"));
+    });
+    return;
+  }
+
+  uint32_t taskIndex = 0, meshletIndex = 0;
+  GetIndicesForMeshRow((uint32_t)idx.row(), taskIndex, meshletIndex);
+
+  const ShaderReflection *shaderDetails =
+      m_Ctx.CurPipelineState().GetShaderReflection(ShaderStage::Mesh);
+
+  if(!shaderDetails)
+    return;
+
+  rdcfixedarray<uint32_t, 3> threadGroupSize = action->dispatchThreadsDimension[0] == 0 ? shaderDetails->dispatchThreadsDimension : action->dispatchThreadsDimension;
+  m_MeshDebugSelector->SetThreadBounds(action->dispatchDimension, threadGroupSize);
+
+  // Calculate 3d group id from 1d meshlet index and dispatch dimensions
+  // Imagine 8x2x4 with idx 60
+  // 8x2 = 16
+  // 8x2x4 = 64
+  // 60 % x = 4
+  // 60 % (x * y) = 12 / x = 1
+  // 60 / (x * y) = 3
+  // index 60 is id (4,1,3)
+  // 4 + (8 * 1) + (16 * 3) = 60
+  uint32_t xDim = action->dispatchDimension[0];
+  uint32_t yDim = action->dispatchDimension[1];
+  uint32_t zDim = action->dispatchDimension[2];
+  rdcfixedarray<uint32_t, 3> meshletGroup = {
+      meshletIndex % xDim, 
+      (meshletIndex % (xDim * yDim)) / xDim, 
+      meshletIndex / (xDim * yDim),
+  };
+  m_MeshDebugSelector->SetDefaultDispatch(meshletGroup, {0, 0, 0});
+
+  RDDialog::show(m_MeshDebugSelector);
+}
+
+void BufferViewer::meshDebugSelector_beginDebug(const rdcfixedarray<uint32_t, 3> &group,
+                                                   const rdcfixedarray<uint32_t, 3> &thread)
+{
+  const ActionDescription *action = m_Ctx.CurAction();
+
+  if(!action)
+    return;
+
+  const ShaderReflection *shaderDetails =
+      m_Ctx.CurPipelineState().GetShaderReflection(ShaderStage::Mesh);
+
+  if(!shaderDetails)
+    return;
+
+  struct threadSelect
+  {
+    rdcfixedarray<uint32_t, 3> g;
+    rdcfixedarray<uint32_t, 3> t;
+  } debugThread = {
+      // g[]
+      {group[0], group[1], group[2]},
+      // t[]
+      {thread[0], thread[1], thread[2]},
+  };
+
+  bool done = false;
+  ShaderDebugTrace *trace = NULL;
+
+  m_Ctx.Replay().AsyncInvoke([&trace, &done, debugThread](IReplayController *r) {
+    trace = r->DebugMeshThread(debugThread.g, debugThread.t);
+
+    if(trace->debugger == NULL)
+    {
+      r->FreeTrace(trace);
+      trace = NULL;
+    }
+
+    done = true;
+  });
+
+  QString debugContext = lit("Mesh Group [%1,%2,%3] Thread [%4,%5,%6]")
+                             .arg(group[0])
+                             .arg(group[1])
+                             .arg(group[2])
+                             .arg(thread[0])
+                             .arg(thread[1])
+                             .arg(thread[2]);
+
+  // wait a short while before displaying the progress dialog (which won't show if we're already
+  // done by the time we reach it)
+  for(int i = 0; !done && i < 100; i++)
+    QThread::msleep(5);
+
+  ShowProgressDialog(this, tr("Debugging %1").arg(debugContext), [&done]() { return done; });
+
+  if(!trace)
+  {
+    RDDialog::critical(
+        this, tr("Error debugging"),
+        tr("Error debugging thread - make sure a valid group and thread is selected"));
+    return;
+  }
+
+  // viewer takes ownership of the trace
+  IShaderViewer *s = m_Ctx.DebugShader(
+      shaderDetails, m_Ctx.CurPipelineState().GetComputePipelineObject(), trace, debugContext);
+
+  m_Ctx.AddDockWindow(s->Widget(), DockReference::AddTo, this);
+}
+
 void BufferViewer::SyncViews(RDTableView *primary, bool selection, bool scroll)
 {
   if(!ui->syncViews->isChecked())

qrenderdoc/Windows/BufferViewer.h

@@ -34,6 +34,7 @@ namespace Ui
 class BufferViewer;
 }
 
+class ComputeDebugSelector;
 class RDSpinBox64;
 class QItemSelection;
 class QMenu;
@@ -164,12 +165,16 @@ private slots:
   void updateExportActionNames();
   void exportData(const BufferExport &params);
   void debugVertex();
+  void debugMeshThread();
+  void meshDebugSelector_beginDebug(const rdcfixedarray<uint32_t, 3> &group,
+                                       const rdcfixedarray<uint32_t, 3> &thread);
   void fixedVars_contextMenu(const QPoint &pos);
 
 private:
   bool eventFilter(QObject *watched, QEvent *event) override;
   Ui::BufferViewer *ui;
   ICaptureContext &m_Ctx;
+  ComputeDebugSelector *m_MeshDebugSelector;
 
   IReplayOutput *m_Output;
 
@@ -323,6 +328,7 @@ private slots:
   QAction *m_ExportCSV = NULL;
   QAction *m_ExportBytes = NULL;
   QAction *m_DebugVert = NULL;
+  QAction *m_DebugMeshThread = NULL;
   QAction *m_FilterMesh = NULL;
   QAction *m_RemoveFilter = NULL;
   QAction *m_GotoTask = NULL;

qrenderdoc/Windows/ShaderMessageViewer.cpp

@@ -266,6 +266,10 @@ ShaderMessageViewer::ShaderMessageViewer(ICaptureContext &ctx, ShaderStageMask s
           inputs.view = msg.location.pixel.view;
           trace = r->DebugPixel(msg.location.pixel.x, msg.location.pixel.y, inputs);
         }
+        else if(msg.stage == ShaderStage::Mesh)
+        {
+          trace = r->DebugMeshThread(msg.location.mesh.meshGroup, msg.location.mesh.thread);
+        }
 
         if(trace && trace->debugger == NULL)
         {

renderdoc/api/replay/renderdoc_replay.h

@@ -1021,6 +1021,17 @@ bucket when the pixel values are divided between ``minval`` and ``maxval``.
   virtual ShaderDebugTrace *DebugThread(const rdcfixedarray<uint32_t, 3> &groupid,
                                         const rdcfixedarray<uint32_t, 3> &threadid) = 0;
 
+  DOCUMENT(R"(Retrieve a debugging trace from running a mesh shader.
+
+:param Tuple[int,int,int] groupid: A list containing the 3D workgroup index.
+:param Tuple[int,int,int] threadid: A list containing the 3D thread index within the workgroup.
+:return: The resulting trace resulting from debugging. Destroy with
+  :meth:`FreeTrace`.
+:rtype: ShaderDebugTrace
+)");
+  virtual ShaderDebugTrace *DebugMeshThread(const rdcfixedarray<uint32_t, 3> &groupid,
+                                               const rdcfixedarray<uint32_t, 3> &threadid) = 0;
+
   DOCUMENT(R"(Continue a shader's debugging with a given shader debugger instance. This will run an
 implementation defined number of steps and then return those steps in a list. This may be a fixed
 number of steps or it may run for a fixed length of time and return as many steps as can be

renderdoc/core/image_viewer.cpp

@@ -357,6 +357,11 @@ class ImageViewer : public IReplayDriver
   {
     return new ShaderDebugTrace();
   }
+  ShaderDebugTrace *DebugMeshThread(uint32_t eventId, const rdcfixedarray<uint32_t, 3> &groupid,
+                                       const rdcfixedarray<uint32_t, 3> &threadid)
+  {
+    return new ShaderDebugTrace();
+  }
   rdcarray<ShaderDebugState> ContinueDebug(ShaderDebugger *debugger) { return {}; }
   void FreeDebugger(ShaderDebugger *debugger) { delete debugger; }
   void BuildTargetShader(ShaderEncoding sourceEncoding, const bytebuf &source, const rdcstr &entry,

renderdoc/core/replay_proxy.cpp

@@ -1684,6 +1684,44 @@ ShaderDebugTrace *ReplayProxy::DebugThread(uint32_t eventId,
   PROXY_FUNCTION(DebugThread, eventId, groupid, threadid);
 }
 
+template <typename ParamSerialiser, typename ReturnSerialiser>
+ShaderDebugTrace *ReplayProxy::Proxied_DebugMeshThread(ParamSerialiser &paramser,
+                                                          ReturnSerialiser &retser, uint32_t eventId,
+                                                          const rdcfixedarray<uint32_t, 3> &groupid,
+                                                          const rdcfixedarray<uint32_t, 3> &threadid)
+{
+  const ReplayProxyPacket expectedPacket = eReplayProxy_DebugMeshThread;
+  ReplayProxyPacket packet = eReplayProxy_DebugMeshThread;
+  ShaderDebugTrace *ret;
+
+  {
+    BEGIN_PARAMS();
+    SERIALISE_ELEMENT(eventId);
+    SERIALISE_ELEMENT(groupid);
+    SERIALISE_ELEMENT(threadid);
+    END_PARAMS();
+  }
+
+  {
+    REMOTE_EXECUTION();
+    if(paramser.IsReading() && !paramser.IsErrored() && !m_IsErrored)
+      ret = m_Remote->DebugMeshThread(eventId, groupid, threadid);
+    else
+      ret = new ShaderDebugTrace;
+  }
+
+  SERIALISE_RETURN(*ret);
+
+  return ret;
+}
+
+ShaderDebugTrace *ReplayProxy::DebugMeshThread(uint32_t eventId,
+                                                  const rdcfixedarray<uint32_t, 3> &groupid,
+                                                  const rdcfixedarray<uint32_t, 3> &threadid)
+{
+  PROXY_FUNCTION(DebugMeshThread, eventId, groupid, threadid);
+}
+
 template <typename ParamSerialiser, typename ReturnSerialiser>
 rdcarray<ShaderDebugState> ReplayProxy::Proxied_ContinueDebug(ParamSerialiser &paramser,
                                                               ReturnSerialiser &retser,

renderdoc/core/replay_proxy.h

@@ -89,6 +89,7 @@ enum ReplayProxyPacket
   eReplayProxy_DebugVertex,
   eReplayProxy_DebugPixel,
   eReplayProxy_DebugThread,
+  eReplayProxy_DebugMeshThread,
 
   eReplayProxy_RenderOverlay,
 
@@ -552,6 +553,9 @@ class ReplayProxy : public IReplayDriver
   IMPLEMENT_FUNCTION_PROXIED(ShaderDebugTrace *, DebugThread, uint32_t eventId,
                              const rdcfixedarray<uint32_t, 3> &groupid,
                              const rdcfixedarray<uint32_t, 3> &threadid);
+  IMPLEMENT_FUNCTION_PROXIED(ShaderDebugTrace *, DebugMeshThread, uint32_t eventId,
+                             const rdcfixedarray<uint32_t, 3> &groupid,
+                             const rdcfixedarray<uint32_t, 3> &threadid);
   IMPLEMENT_FUNCTION_PROXIED(rdcarray<ShaderDebugState>, ContinueDebug, ShaderDebugger *debugger);
   IMPLEMENT_FUNCTION_PROXIED(void, FreeDebugger, ShaderDebugger *debugger);
 

renderdoc/driver/d3d11/d3d11_replay.h

@@ -288,6 +288,8 @@ class D3D11Replay : public IReplayDriver
                                const DebugPixelInputs &inputs);
   ShaderDebugTrace *DebugThread(uint32_t eventId, const rdcfixedarray<uint32_t, 3> &groupid,
                                 const rdcfixedarray<uint32_t, 3> &threadid);
+  ShaderDebugTrace *DebugMeshThread(uint32_t eventId, const rdcfixedarray<uint32_t, 3> &groupid,
+                                       const rdcfixedarray<uint32_t, 3> &threadid);
   rdcarray<ShaderDebugState> ContinueDebug(ShaderDebugger *debugger);
   void FreeDebugger(ShaderDebugger *debugger);