Skip to content

Commit

Permalink
Update for ZED SDK 4.2 (#659)
Browse files Browse the repository at this point in the history
* adding 4.2 samples

* Fixing #645 yolo python color conversion
  • Loading branch information
adujardin authored Nov 6, 2024
1 parent 229593e commit 3472a79
Show file tree
Hide file tree
Showing 162 changed files with 16,998 additions and 6,770 deletions.
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@

---

:tada: The **ZED SDK 4.1** is released! We support the [**ZED X**](https://www.stereolabs.com/zed-x/) and [**ZED X Mini**](https://www.stereolabs.com/zed-x/) cameras, added the **Fusion API** for multi-camera Body Tracking, and more! Please check the [Release Notes](https://www.stereolabs.com/developers/release/) of the latest version for more details.
:tada: The **ZED SDK 4.2** is released! We support the [**ZED X**](https://www.stereolabs.com/zed-x/) and [**ZED X Mini**](https://www.stereolabs.com/zed-x/) cameras, added the **Fusion API** for multi-camera Body Tracking, and more! Please check the [Release Notes](https://www.stereolabs.com/developers/release/) of the latest version for more details.

## Overview

Expand Down
13 changes: 11 additions & 2 deletions body tracking/body tracking/cpp/include/GLViewer.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -127,10 +127,19 @@ const std::vector<std::pair<BODY_38_PARTS, BODY_38_PARTS>> BODY_BONES_FAST_RENDE
class Shader {
public:

Shader() {
}
Shader() : verterxId_(0), fragmentId_(0), programId_(0) {}
Shader(const GLchar* vs, const GLchar* fs);
~Shader();

// Delete the move constructor and move assignment operator
Shader(Shader&&) = delete;
Shader& operator=(Shader&&) = delete;

// Delete the copy constructor and copy assignment operator
Shader(const Shader&) = delete;
Shader& operator=(const Shader&) = delete;

void set(const GLchar* vs, const GLchar* fs);
GLuint getProgramId();

static const GLint ATTRIB_VERTICES_POS = 0;
Expand Down
19 changes: 11 additions & 8 deletions body tracking/body tracking/cpp/src/GLViewer.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -149,10 +149,10 @@ void GLViewer::init(int argc, char **argv) {
glHint(GL_LINE_SMOOTH_HINT, GL_NICEST);

// Compile and create the shader for 3D objects
shaderSK.it = Shader(SK_VERTEX_SHADER, SK_FRAGMENT_SHADER);
shaderSK.it.set(SK_VERTEX_SHADER, SK_FRAGMENT_SHADER);
shaderSK.MVP_Mat = glGetUniformLocation(shaderSK.it.getProgramId(), "u_mvpMatrix");

shaderLine.it = Shader(VERTEX_SHADER, FRAGMENT_SHADER);
shaderLine.it.set(VERTEX_SHADER, FRAGMENT_SHADER);
shaderLine.MVP_Mat = glGetUniformLocation(shaderLine.it.getProgramId(), "u_mvpMatrix");

// Create the camera
Expand Down Expand Up @@ -384,8 +384,7 @@ void GLViewer::idle() {
glutPostRedisplay();
}

Simple3DObject::Simple3DObject() {
}
Simple3DObject::Simple3DObject() : vaoID_(0) {}

Simple3DObject::~Simple3DObject() {
if (vaoID_ != 0) {
Expand Down Expand Up @@ -732,6 +731,10 @@ sl::Transform Simple3DObject::getModelMatrix() const {
}

Shader::Shader(const GLchar* vs, const GLchar* fs) {
set(vs, fs);
}

void Shader::set(const GLchar* vs, const GLchar* fs) {
if (!compile(verterxId_, GL_VERTEX_SHADER, vs)) {
std::cout << "ERROR: while compiling vertex shader" << std::endl;
}
Expand Down Expand Up @@ -767,12 +770,12 @@ Shader::Shader(const GLchar* vs, const GLchar* fs) {
}

Shader::~Shader() {
if (verterxId_ != 0)
if (verterxId_ != 0 && glIsShader(verterxId_))
glDeleteShader(verterxId_);
if (fragmentId_ != 0)
if (fragmentId_ != 0 && glIsShader(fragmentId_))
glDeleteShader(fragmentId_);
if (programId_ != 0)
glDeleteShader(programId_);
if (programId_ != 0 && glIsProgram(programId_))
glDeleteProgram(programId_);
}

GLuint Shader::getProgramId() {
Expand Down
1 change: 1 addition & 0 deletions body tracking/body tracking/csharp/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ add_executable(${PROJECT_NAME}
MainWindow.cs
GLViewer.cs
TrackingViewer.cs
Utils.cs
App.config
packages.config
Properties/AssemblyInfo.cs
Expand Down
5 changes: 2 additions & 3 deletions body tracking/body tracking/csharp/MainWindow.cs
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,6 @@ public MainWindow(string[] args)
isTrackingON = bt_params.enableObjectTracking;
bt_params.enableSegmentation = false;
bt_params.enableBodyFitting = true; // smooth skeletons moves
bt_params.imageSync = true; // the body tracking is synchronized to the image
bt_params.detectionModel = sl.BODY_TRACKING_MODEL.HUMAN_BODY_ACCURATE;
bt_params.bodyFormat = sl.BODY_FORMAT.BODY_38;

Expand Down Expand Up @@ -125,7 +124,7 @@ public MainWindow(string[] args)
bt_runtime_parameters.detectionConfidenceThreshold = 20;

window_name = "ZED| 2D View";
Cv2.NamedWindow(window_name, WindowMode.Normal);// Create Window
Cv2.NamedWindow(window_name);// Create Window

// Create OpenGL window
CreateWindow();
Expand Down Expand Up @@ -234,7 +233,7 @@ private void NativeWindow_Render(object sender, NativeWindowEventArgs e)
// Retrieve Objects
zedCamera.RetrieveBodies(ref bodies, ref bt_runtime_parameters);

TrackingViewer.render_2D(ref imageLeftOcv, imgScale, ref bodies, isTrackingON, bt_params.bodyFormat);
TrackingViewer.render_2D(ref imageLeftOcv, imgScale, ref bodies, isTrackingON, bt_params.bodyFormat, bt_params.enableSegmentation);

//Update GL View
viewer.update(pointCloud, bodies, camPose);
Expand Down
26 changes: 25 additions & 1 deletion body tracking/body tracking/csharp/TrackingViewer.cs
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
using sl;

using OpenCvSharp;
using System.Windows.Media.Media3D;


public class TrackingViewer
Expand Down Expand Up @@ -58,7 +59,7 @@ static sl.float2 getImagePosition(Vector2[] bounding_box_image, sl.float2 img_sc
return position;
}

public static void render_2D(ref OpenCvSharp.Mat left_display, sl.float2 img_scale, ref sl.Bodies bodies, bool showOnlyOK, sl.BODY_FORMAT body_format)
public static void render_2D(ref OpenCvSharp.Mat left_display, sl.float2 img_scale, ref sl.Bodies bodies, bool showOnlyOK, sl.BODY_FORMAT body_format, bool render_mask)
{
OpenCvSharp.Mat overlay = left_display.Clone();
OpenCvSharp.Rect roi_render = new OpenCvSharp.Rect(1, 1, left_display.Size().Width, left_display.Size().Height);
Expand Down Expand Up @@ -93,6 +94,29 @@ public static void render_2D(ref OpenCvSharp.Mat left_display, sl.float2 img_sca
}
}
}

if (render_mask)
{
Point top_left_corner = Utils.cvt(bod.boundingBox2D[0], img_scale);
Point top_right_corner = Utils.cvt(bod.boundingBox2D[1], img_scale);
Point bottom_left_corner = Utils.cvt(bod.boundingBox2D[3], img_scale);
var width = (int)Math.Abs(top_right_corner.X - top_left_corner.X);
var height = (int)Math.Abs(bottom_left_corner.Y - top_left_corner.Y);
// Scaled ROI
OpenCvSharp.Rect roi = new OpenCvSharp.Rect(top_left_corner.X, top_left_corner.Y, width, height);
sl.Mat mask = new sl.Mat(bod.mask);
if (mask.IsInit())
{
sl.Mat m = new sl.Mat();
mask.CopyTo(m);
OpenCvSharp.Mat tmp_mask = new OpenCvSharp.Mat(mask.GetHeight(), mask.GetWidth(), OpenCvSharp.MatType.CV_8UC1, m.GetPtr());
if (!tmp_mask.Empty())
{
var mask_resized = tmp_mask.Resize(roi.Size);
overlay.SubMat(roi).SetTo(base_color, mask_resized);
}
}
}
}
}

Expand Down
163 changes: 163 additions & 0 deletions body tracking/body tracking/csharp/Utils.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,163 @@
//======= Copyright (c) Stereolabs Corporation, All rights reserved. ===============
using System;
using System.Numerics;
using System.Runtime.InteropServices;
using System.Threading.Tasks;
using OpenCvSharp;

namespace sl
{
public class Utils
{
/// <summary>
/// Creates an OpenCV version of a ZED Mat.
/// </summary>
/// <param name="zedmat">Source ZED Mat.</param>
/// <param name="zedmattype">Type of ZED Mat - data type and channel number.
/// <returns></returns>
public static OpenCvSharp.Mat SLMat2CVMat(ref sl.Mat zedmat, MAT_TYPE zedmattype)
{
int cvmattype = SLMatType2CVMatType(zedmattype);
OpenCvSharp.Mat cvmat = new OpenCvSharp.Mat(zedmat.GetHeight(), zedmat.GetWidth(), cvmattype, zedmat.GetPtr());

return cvmat;
}

/// <summary>
/// Returns the OpenCV type that corresponds to a given ZED Mat type.
/// </summary>
private static int SLMatType2CVMatType(MAT_TYPE zedmattype)
{
switch (zedmattype)
{
case sl.MAT_TYPE.MAT_32F_C1:
return OpenCvSharp.MatType.CV_32FC1;
case sl.MAT_TYPE.MAT_32F_C2:
return OpenCvSharp.MatType.CV_32FC2;
case sl.MAT_TYPE.MAT_32F_C3:
return OpenCvSharp.MatType.CV_32FC3;
case sl.MAT_TYPE.MAT_32F_C4:
return OpenCvSharp.MatType.CV_32FC4;
case sl.MAT_TYPE.MAT_8U_C1:
return OpenCvSharp.MatType.CV_8UC1;
case sl.MAT_TYPE.MAT_8U_C2:
return OpenCvSharp.MatType.CV_8UC2;
case sl.MAT_TYPE.MAT_8U_C3:
return OpenCvSharp.MatType.CV_8UC3;
case sl.MAT_TYPE.MAT_8U_C4:
return OpenCvSharp.MatType.CV_8UC4;
default:
return -1;
}
}

public static ulong getMilliseconds(ulong ts_ns)
{
return ts_ns / 1000000;
}

public static void drawVerticalLine(ref OpenCvSharp.Mat left_display, Point start_pt, Point end_pt, Scalar clr, int thickness)
{
int n_steps = 7;
Point pt1, pt4;
pt1.X = ((n_steps - 1) * start_pt.X + end_pt.X) / n_steps;
pt1.Y = ((n_steps - 1) * start_pt.Y + end_pt.Y) / n_steps;

pt4.X = (start_pt.X + (n_steps - 1) * end_pt.X) / n_steps;
pt4.Y = (start_pt.Y + (n_steps - 1) * end_pt.Y) / n_steps;

Cv2.Line(left_display, start_pt, pt1, clr, thickness);
Cv2.Line(left_display, pt4, end_pt, clr, thickness);
}

public static Point cvt(Vector2 pt, sl.float2 scale)
{
return new Point(pt.X * scale.x, pt.Y * scale.y);
}

public static sl.float4 generateColorID(int idx)
{
int offset = Math.Max(0, idx % 5);
sl.float4 color = new float4();
color.x = id_colors[offset, 0];
color.y = id_colors[offset, 1];
color.z = id_colors[offset, 2];
color.w = 1.0f;
return color;
}

public static OpenCvSharp.Scalar generateColorID_u(int idx)
{
int offset = Math.Max(0, idx % 5);
OpenCvSharp.Scalar color = new OpenCvSharp.Scalar();
color[0] = id_colors[offset, 2] * 255;
color[1] = id_colors[offset, 1] * 255;
color[2] = id_colors[offset, 0] * 255;
color[3] = 1.0f * 255;
return color;
}

public static float[,] id_colors = new float[5, 3]{

{.231f, .909f, .69f},
{.098f, .686f, .816f},
{.412f, .4f, .804f},
{1, .725f, .0f},
{.989f, .388f, .419f}
};

public static float[,] class_colors = new float[6, 3]{
{ 44.0f, 117.0f, 255.0f}, // PEOPLE
{ 255.0f, 0.0f, 255.0f}, // VEHICLE
{ 0.0f, 0.0f, 255.0f},
{ 0.0f, 255.0f, 255.0f},
{ 0.0f, 255.0f, 0.0f},
{ 255.0f, 255.0f, 255.0f}
};

public static float4 generateColorClass(int idx)
{
idx = Math.Min(5, idx);
sl.float4 color = new float4();
color.x = class_colors[idx, 0];
color.y = class_colors[idx, 1];
color.z = class_colors[idx, 2];
color.w = 255.0f;
return color;
}

public static OpenCvSharp.Scalar generateColorClass_u(int idx)
{
idx = Math.Min(5, idx);
OpenCvSharp.Scalar color = new OpenCvSharp.Scalar();
color[0] = class_colors[idx, 0];
color[0] = class_colors[idx, 1];
color[0] = class_colors[idx, 2];
color[0] = 255.0f;
return color;
}

public static bool renderObject(ObjectData i, bool showOnlyOK)
{
if (showOnlyOK)
return (i.objectTrackingState == sl.OBJECT_TRACKING_STATE.OK);
else
return (i.objectTrackingState == sl.OBJECT_TRACKING_STATE.OK || i.objectTrackingState == sl.OBJECT_TRACKING_STATE.OFF);
}


public static byte _applyFading(double val, float current_alpha, double current_clr)
{
return (byte)(current_alpha * current_clr + (1.0 - current_alpha) * val);
}

public static Vec4b applyFading(Scalar val, float current_alpha, Scalar current_clr){
Vec4b out_ = new Vec4b();
out_[0] = _applyFading(val[0], current_alpha, current_clr[0]);
out_[1] = _applyFading(val[1], current_alpha, current_clr[1]);
out_[2] = _applyFading(val[2], current_alpha, current_clr[2]);
out_[3] = 255;
return out_;
}
}
}
13 changes: 11 additions & 2 deletions body tracking/export/JSON export/cpp/include/GLViewer.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -127,10 +127,19 @@ const std::vector<std::pair<BODY_38_PARTS, BODY_38_PARTS>> BODY_BONES_FAST_RENDE
class Shader {
public:

Shader() {
}
Shader() : verterxId_(0), fragmentId_(0), programId_(0) {}
Shader(const GLchar* vs, const GLchar* fs);
~Shader();

// Delete the move constructor and move assignment operator
Shader(Shader&&) = delete;
Shader& operator=(Shader&&) = delete;

// Delete the copy constructor and copy assignment operator
Shader(const Shader&) = delete;
Shader& operator=(const Shader&) = delete;

void set(const GLchar* vs, const GLchar* fs);
GLuint getProgramId();

static const GLint ATTRIB_VERTICES_POS = 0;
Expand Down
Loading

0 comments on commit 3472a79

Please sign in to comment.