From 9418a9a640054c5e1f6fd9cb5b6b2d9a2f3fbab4 Mon Sep 17 00:00:00 2001 From: tvlenin Date: Mon, 11 Nov 2019 11:12:53 -0600 Subject: [PATCH 001/188] Increase version number --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index cf188648..170c82d1 100644 --- a/configure.ac +++ b/configure.ac @@ -5,7 +5,7 @@ dnl please read gstreamer/docs/random/autotools before changing this file dnl initialize autoconf dnl releases only do -Wall, git and prerelease does -Werror too dnl use a three digit version number for releases, and four for git/pre -AC_INIT([GStreamer Inference],[0.6.0.1],[https://github.com/RidgeRun/gst-inference/issues],[gst-inference]) +AC_INIT([GStreamer Inference],[0.7.0.1],[https://github.com/RidgeRun/gst-inference/issues],[gst-inference]) AG_GST_INIT From c07941fb9f28ba72adb14aec9b30530692cbc01b Mon Sep 17 00:00:00 2001 From: Abraham Arias Date: Fri, 30 Aug 2019 11:44:14 -0600 Subject: [PATCH 002/188] Add support for pkg-config with pc.in Use the gst-inference.pc.in file as template to generate the .pc file with autotools variables. --- Makefile.am | 5 +++++ configure.ac | 7 +++++-- gst-inference.pc.in | 14 ++++++++++++++ 3 files changed, 24 insertions(+), 2 deletions(-) create mode 100644 gst-inference.pc.in diff --git a/Makefile.am b/Makefile.am index a33a6965..86e71efc 100644 --- a/Makefile.am +++ b/Makefile.am @@ -54,3 +54,8 @@ CRUFT_DIRS = include $(top_srcdir)/common/cruft.mak all-local: check-cruft + +pkgconfigdir = $(libdir)/pkgconfig +pkgconfig_DATA = gst-inference-@PACKAGE_VERSION@.pc + +EXTRA_DIST = gst-inference.pc.in \ No newline at end of file diff --git a/configure.ac b/configure.ac index 170c82d1..de793440 100644 --- a/configure.ac +++ b/configure.ac @@ -367,6 +367,9 @@ AC_SUBST(GST_PLUGIN_LDFLAGS) dnl *** output files *** +# Our Include path, to be shared in multiple makefiles +AC_SUBST([GSTINCLUDEDIR],[$includedir/gstreamer-1.0]) + dnl keep this alphabetic per directory, please AC_CONFIG_FILES( Makefile @@ -384,6 +387,7 @@ gst-libs/gst/opencv/Makefile gst-libs/gst/r2inference/Makefile m4/Makefile tests/check/Makefile +gst-inference-${PACKAGE_VERSION}.pc:gst-inference.pc.in tests/Makefile tests/examples/classification/Makefile tests/examples/detection/Makefile @@ -393,5 +397,4 @@ tests/files/Makefile ) AC_OUTPUT -AG_GST_OUTPUT_PLUGINS - +AG_GST_OUTPUT_PLUGINS \ No newline at end of file diff --git a/gst-inference.pc.in b/gst-inference.pc.in new file mode 100644 index 00000000..ba7cf1dd --- /dev/null +++ b/gst-inference.pc.in @@ -0,0 +1,14 @@ +prefix=@prefix@ +exec_prefix=@exec_prefix@ +libdir=@libdir@ +includedir=@GSTINCLUDEDIR@ + +Name: GstInference +Description: GStreamer framework for deep learning inference. +URL: https://github.com/RidgeRun/gst-inference/ +Version: @VERSION@ +Requires: gtk-doc gstreamer-1.0 gstreamer-plugins-base-1.0 \ +gstreamer-plugins-good-1.0 gstreamer-audio-1.0 gstreamer-video-1.0 \ +r2inference-0.0 +Libs: -L${libdir} -lgstinference-@PACKAGE_VERSION@ +Cflags: -I${includedir} From 3e225fa7563a767f3583945b18e2f777bcb5f02b Mon Sep 17 00:00:00 2001 From: Michael Gruner Date: Tue, 3 Sep 2019 19:04:13 -0600 Subject: [PATCH 003/188] Follow standard and use GStreamer API version on .pc --- Makefile.am | 4 ++-- configure.ac | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile.am b/Makefile.am index 86e71efc..4d7fe161 100644 --- a/Makefile.am +++ b/Makefile.am @@ -56,6 +56,6 @@ include $(top_srcdir)/common/cruft.mak all-local: check-cruft pkgconfigdir = $(libdir)/pkgconfig -pkgconfig_DATA = gst-inference-@PACKAGE_VERSION@.pc +pkgconfig_DATA = gst-inference-@GST_API_VERSION@.pc -EXTRA_DIST = gst-inference.pc.in \ No newline at end of file +EXTRA_DIST = gst-inference.pc.in diff --git a/configure.ac b/configure.ac index de793440..77359113 100644 --- a/configure.ac +++ b/configure.ac @@ -387,7 +387,7 @@ gst-libs/gst/opencv/Makefile gst-libs/gst/r2inference/Makefile m4/Makefile tests/check/Makefile -gst-inference-${PACKAGE_VERSION}.pc:gst-inference.pc.in +gst-inference-${GST_API_VERSION}.pc:gst-inference.pc.in tests/Makefile tests/examples/classification/Makefile tests/examples/detection/Makefile @@ -397,4 +397,4 @@ tests/files/Makefile ) AC_OUTPUT -AG_GST_OUTPUT_PLUGINS \ No newline at end of file +AG_GST_OUTPUT_PLUGINS From eb101cdf2ee8066530aad676749edfe16f0e3ef3 Mon Sep 17 00:00:00 2001 From: Michael Gruner Date: Tue, 3 Sep 2019 19:15:54 -0600 Subject: [PATCH 004/188] Remove uneeded dependencies --- gst-inference.pc.in | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/gst-inference.pc.in b/gst-inference.pc.in index ba7cf1dd..6a26b6f7 100644 --- a/gst-inference.pc.in +++ b/gst-inference.pc.in @@ -7,8 +7,6 @@ Name: GstInference Description: GStreamer framework for deep learning inference. URL: https://github.com/RidgeRun/gst-inference/ Version: @VERSION@ -Requires: gtk-doc gstreamer-1.0 gstreamer-plugins-base-1.0 \ -gstreamer-plugins-good-1.0 gstreamer-audio-1.0 gstreamer-video-1.0 \ -r2inference-0.0 +Requires: gstreamer-1.0 gstreamer-plugins-base-1.0 gstreamer-video-1.0 r2inference-0.0 Libs: -L${libdir} -lgstinference-@PACKAGE_VERSION@ Cflags: -I${includedir} From 941c51b09e9d252d82c66de8266a31341c0707b2 Mon Sep 17 00:00:00 2001 From: Michael Gruner Date: Tue, 3 Sep 2019 19:18:59 -0600 Subject: [PATCH 005/188] Use appropriate library version suffix --- gst-inference.pc.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gst-inference.pc.in b/gst-inference.pc.in index 6a26b6f7..a753a936 100644 --- a/gst-inference.pc.in +++ b/gst-inference.pc.in @@ -8,5 +8,5 @@ Description: GStreamer framework for deep learning inference. URL: https://github.com/RidgeRun/gst-inference/ Version: @VERSION@ Requires: gstreamer-1.0 gstreamer-plugins-base-1.0 gstreamer-video-1.0 r2inference-0.0 -Libs: -L${libdir} -lgstinference-@PACKAGE_VERSION@ +Libs: -L${libdir} -lgstinference-@GST_API_VERSION@ Cflags: -I${includedir} From 423fab52b93e400dabdfc1c79b5463a0a7da7dd0 Mon Sep 17 00:00:00 2001 From: Michael Gruner Date: Tue, 3 Sep 2019 19:22:37 -0600 Subject: [PATCH 006/188] Fix typo in gitignore --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 6bcf7c25..fb032452 100644 --- a/.gitignore +++ b/.gitignore @@ -26,7 +26,7 @@ m4/ .libs/ stamp-h* autoregen.sh -gstinference-*.pc +gst-inference-*.pc # Build outputs .deps/ From 3334ad15f05aae2938850a5b1641fa6ee4b39b55 Mon Sep 17 00:00:00 2001 From: Michael Gruner Date: Tue, 3 Sep 2019 21:32:08 -0600 Subject: [PATCH 007/188] inferencecrop: Add placeholder for detection crop --- Makefile.am | 1 + configure.ac | 2 + gst/Makefile.am | 1 + gst/inferencecrop/Makefile.am | 32 +++++++ gst/inferencecrop/gstdetectioncrop.cc | 118 ++++++++++++++++++++++++++ gst/inferencecrop/gstdetectioncrop.h | 34 ++++++++ 6 files changed, 188 insertions(+) create mode 100644 gst/Makefile.am create mode 100644 gst/inferencecrop/Makefile.am create mode 100644 gst/inferencecrop/gstdetectioncrop.cc create mode 100644 gst/inferencecrop/gstdetectioncrop.h diff --git a/Makefile.am b/Makefile.am index 4d7fe161..fcdc4dde 100644 --- a/Makefile.am +++ b/Makefile.am @@ -2,6 +2,7 @@ DISTCHECK_CONFIGURE_FLAGS=--enable-gtk-doc ALWAYS_SUBDIRS = \ gst-libs \ + gst \ ext \ tests \ common \ diff --git a/configure.ac b/configure.ac index 77359113..f85dc696 100644 --- a/configure.ac +++ b/configure.ac @@ -381,6 +381,8 @@ docs/version.entities ext/Makefile ext/opencv/Makefile ext/r2inference/Makefile +gst/Makefile +gst/inferencecrop/Makefile gst-libs/Makefile gst-libs/gst/Makefile gst-libs/gst/opencv/Makefile diff --git a/gst/Makefile.am b/gst/Makefile.am new file mode 100644 index 00000000..bdebbbf5 --- /dev/null +++ b/gst/Makefile.am @@ -0,0 +1 @@ +SUBDIRS = inferencecrop diff --git a/gst/inferencecrop/Makefile.am b/gst/inferencecrop/Makefile.am new file mode 100644 index 00000000..e532d25b --- /dev/null +++ b/gst/inferencecrop/Makefile.am @@ -0,0 +1,32 @@ +plugin_LTLIBRARIES = libgstinferencecrop.la + +libgstinferencecrop_la_SOURCES = \ + gstdetectioncrop.cc + + +libgstinferencecrop_la_CFLAGS = \ + $(GST_CFLAGS) \ + $(GST_BASE_CFLAGS) \ + $(GST_PLUGINS_BASE_CFLAGS) \ + -I$(top_srcdir)/gst-libs + +libgstinferencecrop_la_CXXFLAGS = \ + $(GST_CFLAGS) \ + $(GST_BASE_CFLAGS) \ + $(GST_PLUGINS_BASE_CFLAGS) \ + -I$(top_srcdir)/gst-libs + +libgstinferencecrop_la_LIBADD = \ + $(GST_LIBS) \ + $(GST_BASE_LIBS) \ + $(GST_PLUGINS_BASE_LIBS) \ + $(top_builddir)/gst-libs/gst/r2inference/libgstinference-@GST_API_VERSION@.la + +libgstinferencecrop_la_LDFLAGS = \ + $(GST_PLUGIN_LDFLAGS) + +libgstinferencecrop_la_LIBTOOLFLAGS = \ + $(GST_PLUGIN_LIBTOOLFLAGS) + +noinst_HEADERS = \ + gstdetectioncrop.h diff --git a/gst/inferencecrop/gstdetectioncrop.cc b/gst/inferencecrop/gstdetectioncrop.cc new file mode 100644 index 00000000..df841141 --- /dev/null +++ b/gst/inferencecrop/gstdetectioncrop.cc @@ -0,0 +1,118 @@ +/* + * GStreamer + * Copyright (C) 2019 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + */ + +/** + * SECTION:element-gstdetectioncrop + * + * The detectioncrop element parses a detectionmeta and crops the incomming + * image to the bounding box. + * + * + * Example launch line + * |[ + * gst-launch-1.0 -v videotestsrc ! detectioncrop ! xvimagesink + * ]| + * Process video frames from the camera using a detectioncrop model. + * + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "gstdetectioncrop.h" + +#include "gst/r2inference/gstinferencemeta.h" + +/* generic templates */ +static GstStaticPadTemplate sink_template = GST_STATIC_PAD_TEMPLATE ("sink", + GST_PAD_SINK, + GST_PAD_ALWAYS, + GST_STATIC_CAPS_ANY); + +static GstStaticPadTemplate src_template = GST_STATIC_PAD_TEMPLATE ("src", + GST_PAD_SRC, + GST_PAD_ALWAYS, + GST_STATIC_CAPS_ANY); + +GST_DEBUG_CATEGORY_STATIC (gst_detection_crop_debug_category); +#define GST_CAT_DEFAULT gst_detection_crop_debug_category + +enum +{ + PROP_0 +}; + +struct _GstDetectionCrop +{ + GstBin parent; +}; + +struct _GstDetectionCropClass +{ + GstBinClass parent; +}; + +/* class initialization */ + +G_DEFINE_TYPE_WITH_CODE (GstDetectionCrop, gst_detection_crop, + GST_TYPE_BIN, + GST_DEBUG_CATEGORY_INIT (gst_detection_crop_debug_category, "detectioncrop", + 0, "debug category for detectioncrop element")); + +static void +gst_detection_crop_class_init (GstDetectionCropClass * klass) +{ + GstElementClass *element_class = GST_ELEMENT_CLASS (klass); + + gst_element_class_add_pad_template (element_class, + gst_static_pad_template_get (&sink_template)); + gst_element_class_add_pad_template (element_class, + gst_static_pad_template_get (&src_template)); + + gst_element_class_set_static_metadata (GST_ELEMENT_CLASS (klass), + "detectioncrop", "Filter", + "Crops an incoming image based on an inference prediction bounding box", + " Michael Gruner "); +} + +static void +gst_detection_crop_init (GstDetectionCrop * detectioncrop) +{ +} + +static gboolean +plugin_init (GstPlugin * plugin) +{ + gboolean ret = TRUE; + + ret = + gst_element_register (plugin, "detectioncrop", GST_RANK_NONE, + GST_TYPE_DETECTION_CROP); + + return ret; +} + +GST_PLUGIN_DEFINE (GST_VERSION_MAJOR, + GST_VERSION_MINOR, + inferencecrop, + "Crops an incoming image based on an inference prediction bounding box", + plugin_init, VERSION, "LGPL", PACKAGE_NAME, GST_PACKAGE_ORIGIN) diff --git a/gst/inferencecrop/gstdetectioncrop.h b/gst/inferencecrop/gstdetectioncrop.h new file mode 100644 index 00000000..620b8712 --- /dev/null +++ b/gst/inferencecrop/gstdetectioncrop.h @@ -0,0 +1,34 @@ +/* + * GStreamer + * Copyright (C) 2019 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + */ + +#ifndef _GST_DETECTIONCROP_H_ +#define _GST_DETECTIONCROP_H_ + +#include + +G_BEGIN_DECLS + +#define GST_TYPE_DETECTION_CROP gst_detection_crop_get_type () +G_DECLARE_FINAL_TYPE (GstDetectionCrop, gst_detection_crop, GST, DETECTION_CROP, GstBin) + +G_END_DECLS + +#endif From 92ff7f58173d046ba7beee801e94774a5eab715a Mon Sep 17 00:00:00 2001 From: Michael Gruner Date: Tue, 3 Sep 2019 22:44:23 -0600 Subject: [PATCH 008/188] Abstract actual cropper element in a separate class --- gst/inferencecrop/Makefile.am | 11 ++- gst/inferencecrop/cropelement.cc | 103 ++++++++++++++++++++++++++ gst/inferencecrop/cropelement.h | 60 +++++++++++++++ gst/inferencecrop/gstdetectioncrop.cc | 20 ++++- gst/inferencecrop/videocrop.cc | 40 ++++++++++ gst/inferencecrop/videocrop.h | 44 +++++++++++ 6 files changed, 274 insertions(+), 4 deletions(-) create mode 100644 gst/inferencecrop/cropelement.cc create mode 100644 gst/inferencecrop/cropelement.h create mode 100644 gst/inferencecrop/videocrop.cc create mode 100644 gst/inferencecrop/videocrop.h diff --git a/gst/inferencecrop/Makefile.am b/gst/inferencecrop/Makefile.am index e532d25b..98cf6969 100644 --- a/gst/inferencecrop/Makefile.am +++ b/gst/inferencecrop/Makefile.am @@ -1,7 +1,9 @@ plugin_LTLIBRARIES = libgstinferencecrop.la libgstinferencecrop_la_SOURCES = \ - gstdetectioncrop.cc + gstdetectioncrop.cc \ + cropelement.cc \ + videocrop.cc libgstinferencecrop_la_CFLAGS = \ @@ -14,7 +16,8 @@ libgstinferencecrop_la_CXXFLAGS = \ $(GST_CFLAGS) \ $(GST_BASE_CFLAGS) \ $(GST_PLUGINS_BASE_CFLAGS) \ - -I$(top_srcdir)/gst-libs + -I$(top_srcdir)/gst-libs \ + -std=c++11 libgstinferencecrop_la_LIBADD = \ $(GST_LIBS) \ @@ -29,4 +32,6 @@ libgstinferencecrop_la_LIBTOOLFLAGS = \ $(GST_PLUGIN_LIBTOOLFLAGS) noinst_HEADERS = \ - gstdetectioncrop.h + gstdetectioncrop.h \ + cropelement.h \ + videocrop.h diff --git a/gst/inferencecrop/cropelement.cc b/gst/inferencecrop/cropelement.cc new file mode 100644 index 00000000..b2680d20 --- /dev/null +++ b/gst/inferencecrop/cropelement.cc @@ -0,0 +1,103 @@ +/* + * GStreamer + * Copyright (C) 2019 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + */ + +#include "cropelement.h" + +CropElement::CropElement () +{ + this->element = nullptr; + this->image_width = 0; + this->image_height = 0; + this->x = 0; + this->y = 0; + this->width = 0; + this->height = 0; +} + +bool +CropElement::Validate () +{ + GstElement * element; + + this->mutex.lock (); + if (nullptr == this->element) { + const std::string factory = this->GetFactory (); + this->element = gst_element_factory_make (factory.c_str (), nullptr); + } + element = this->element; + this->mutex.unlock (); + + return element != nullptr; +} + +GstElement * +CropElement::GetElement () +{ + GstElement * element; + + this->mutex.lock (); + element = this->element; + this->mutex.unlock (); + + return element; +} + +void +CropElement::SetImageSize (gint width, gint height) +{ + this->mutex.lock (); + this->image_width = width; + this->image_height = height; + + this->UpdateElement (this->element, + this->image_width, + this->image_height, + this->x, + this->y, + this->width, + this->height); + this->mutex.unlock (); +} + +void +CropElement::SetBoundingBox (gint x, gint y, gint width, gint height) +{ + this->mutex.lock (); + this->x = x; + this->y = y; + this->width = width; + this->height = height; + + this->UpdateElement (this->element, + this->image_width, + this->image_height, + this->x, + this->y, + this->width, + this->height); + this->mutex.unlock (); +} + +CropElement::~CropElement () +{ + gst_object_unref (this->element); + this->element = nullptr; +} diff --git a/gst/inferencecrop/cropelement.h b/gst/inferencecrop/cropelement.h new file mode 100644 index 00000000..ef3080d9 --- /dev/null +++ b/gst/inferencecrop/cropelement.h @@ -0,0 +1,60 @@ +/* + * GStreamer + * Copyright (C) 2019 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + */ + +#ifndef __CROP_ELEMENT_H__ +#define __CROP_ELEMENT_H__ + +#include +#include +#include + +class CropElement +{ +public: + CropElement (); + bool Validate (); + GstElement * GetElement (); + void SetImageSize (gint width, gint height); + void SetBoundingBox (gint x, gint y, gint width, gint height); + virtual ~CropElement (); + +protected: + virtual const std::string GetFactory () const = 0; + virtual void UpdateElement (GstElement * element, + gint image_width, + gint image_height, + gint x, + gint y, + gint width, + gint height) = 0; + +private: + GstElement * element; + gint image_width; + gint image_height; + gint x; + gint y; + gint width; + gint height; + std::mutex mutex; +}; + +#endif //__CROP_ELEMENT_H__ diff --git a/gst/inferencecrop/gstdetectioncrop.cc b/gst/inferencecrop/gstdetectioncrop.cc index df841141..37697b71 100644 --- a/gst/inferencecrop/gstdetectioncrop.cc +++ b/gst/inferencecrop/gstdetectioncrop.cc @@ -41,6 +41,7 @@ #include "gstdetectioncrop.h" #include "gst/r2inference/gstinferencemeta.h" +#include "videocrop.h" /* generic templates */ static GstStaticPadTemplate sink_template = GST_STATIC_PAD_TEMPLATE ("sink", @@ -56,6 +57,8 @@ static GstStaticPadTemplate src_template = GST_STATIC_PAD_TEMPLATE ("src", GST_DEBUG_CATEGORY_STATIC (gst_detection_crop_debug_category); #define GST_CAT_DEFAULT gst_detection_crop_debug_category +static void gst_detection_crop_finalize (GObject * object); + enum { PROP_0 @@ -64,6 +67,7 @@ enum struct _GstDetectionCrop { GstBin parent; + CropElement *element; }; struct _GstDetectionCropClass @@ -82,6 +86,7 @@ static void gst_detection_crop_class_init (GstDetectionCropClass * klass) { GstElementClass *element_class = GST_ELEMENT_CLASS (klass); + GObjectClass *object_class = G_OBJECT_CLASS (klass); gst_element_class_add_pad_template (element_class, gst_static_pad_template_get (&sink_template)); @@ -92,11 +97,24 @@ gst_detection_crop_class_init (GstDetectionCropClass * klass) "detectioncrop", "Filter", "Crops an incoming image based on an inference prediction bounding box", " Michael Gruner "); + + object_class->finalize = gst_detection_crop_finalize; +} + +static void +gst_detection_crop_init (GstDetectionCrop * self) +{ + self->element = new VideoCrop (); } static void -gst_detection_crop_init (GstDetectionCrop * detectioncrop) +gst_detection_crop_finalize (GObject * object) { + GstDetectionCrop *self = GST_DETECTION_CROP (object); + + delete (self->element); + + G_OBJECT_CLASS (gst_detection_crop_parent_class)->finalize (object); } static gboolean diff --git a/gst/inferencecrop/videocrop.cc b/gst/inferencecrop/videocrop.cc new file mode 100644 index 00000000..3ce6eee6 --- /dev/null +++ b/gst/inferencecrop/videocrop.cc @@ -0,0 +1,40 @@ +/* + * GStreamer + * Copyright (C) 2019 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + */ + +#include "videocrop.h" + +const std::string +VideoCrop::GetFactory () const +{ + return this->factory; +} + +void +VideoCrop::UpdateElement (GstElement * element, + gint image_width, + gint image_height, + gint x, + gint y, + gint width, + gint height) +{ + +} diff --git a/gst/inferencecrop/videocrop.h b/gst/inferencecrop/videocrop.h new file mode 100644 index 00000000..f484425e --- /dev/null +++ b/gst/inferencecrop/videocrop.h @@ -0,0 +1,44 @@ +/* + * GStreamer + * Copyright (C) 2019 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + */ + +#ifndef __VIDEO_CROP_H__ +#define __VIDEO_CROP_H__ + +#include "cropelement.h" + +#include + +class VideoCrop : public CropElement +{ +protected: + const std::string GetFactory () const override; + void UpdateElement (GstElement * element, + gint image_width, + gint image_height, + gint x, + gint y, + gint width, + gint height) override; +private: + const std::string factory = "videocrop"; +}; + +#endif //__VIDEO_CROP_H__ From c247f88692eb2e8136ce6eabc32842ac9536327e Mon Sep 17 00:00:00 2001 From: Michael Gruner Date: Tue, 3 Sep 2019 23:08:28 -0600 Subject: [PATCH 009/188] Fail if no cropping element was found --- gst/inferencecrop/cropelement.cc | 6 ++-- gst/inferencecrop/cropelement.h | 2 +- gst/inferencecrop/gstdetectioncrop.cc | 51 +++++++++++++++++++++++++++ gst/inferencecrop/videocrop.h | 4 ++- 4 files changed, 59 insertions(+), 4 deletions(-) diff --git a/gst/inferencecrop/cropelement.cc b/gst/inferencecrop/cropelement.cc index b2680d20..9373453d 100644 --- a/gst/inferencecrop/cropelement.cc +++ b/gst/inferencecrop/cropelement.cc @@ -98,6 +98,8 @@ CropElement::SetBoundingBox (gint x, gint y, gint width, gint height) CropElement::~CropElement () { - gst_object_unref (this->element); - this->element = nullptr; + if (nullptr != this->element) { + gst_object_unref (this->element); + this->element = nullptr; + } } diff --git a/gst/inferencecrop/cropelement.h b/gst/inferencecrop/cropelement.h index ef3080d9..2285cb16 100644 --- a/gst/inferencecrop/cropelement.h +++ b/gst/inferencecrop/cropelement.h @@ -35,9 +35,9 @@ class CropElement void SetImageSize (gint width, gint height); void SetBoundingBox (gint x, gint y, gint width, gint height); virtual ~CropElement (); + virtual const std::string GetFactory () const = 0; protected: - virtual const std::string GetFactory () const = 0; virtual void UpdateElement (GstElement * element, gint image_width, gint image_height, diff --git a/gst/inferencecrop/gstdetectioncrop.cc b/gst/inferencecrop/gstdetectioncrop.cc index 37697b71..e546e06a 100644 --- a/gst/inferencecrop/gstdetectioncrop.cc +++ b/gst/inferencecrop/gstdetectioncrop.cc @@ -58,6 +58,9 @@ GST_DEBUG_CATEGORY_STATIC (gst_detection_crop_debug_category); #define GST_CAT_DEFAULT gst_detection_crop_debug_category static void gst_detection_crop_finalize (GObject * object); +static GstStateChangeReturn gst_detection_crop_change_state (GstElement * + element, GstStateChange transition); +static gboolean gst_detection_crop_start (GstDetectionCrop * self); enum { @@ -98,6 +101,9 @@ gst_detection_crop_class_init (GstDetectionCropClass * klass) "Crops an incoming image based on an inference prediction bounding box", " Michael Gruner "); + element_class->change_state = + GST_DEBUG_FUNCPTR (gst_detection_crop_change_state); + object_class->finalize = gst_detection_crop_finalize; } @@ -117,6 +123,51 @@ gst_detection_crop_finalize (GObject * object) G_OBJECT_CLASS (gst_detection_crop_parent_class)->finalize (object); } +static gboolean +gst_detection_crop_start (GstDetectionCrop * self) +{ + gboolean ret; + + g_return_val_if_fail (self, FALSE); + + ret = self->element->Validate (); + if (FALSE == ret) { + const std::string factory = self->element->GetFactory (); + GST_ERROR_OBJECT (self, "Unable to find element %s", factory.c_str ()); + } + + return ret; +} + +static GstStateChangeReturn +gst_detection_crop_change_state (GstElement * element, + GstStateChange transition) +{ + GstStateChangeReturn ret; + GstDetectionCrop *self = GST_DETECTION_CROP (element); + + switch (transition) { + case GST_STATE_CHANGE_READY_TO_PAUSED: + if (FALSE == gst_detection_crop_start (self)) { + GST_ERROR_OBJECT (self, "Failed to start"); + ret = GST_STATE_CHANGE_FAILURE; + goto out; + } + default: + break; + } + ret = + GST_ELEMENT_CLASS (gst_detection_crop_parent_class)->change_state + (element, transition); + if (GST_STATE_CHANGE_FAILURE == ret) { + GST_ERROR_OBJECT (self, "Parent failed to change state"); + goto out; + } + +out: + return ret; +} + static gboolean plugin_init (GstPlugin * plugin) { diff --git a/gst/inferencecrop/videocrop.h b/gst/inferencecrop/videocrop.h index f484425e..8b39a5d4 100644 --- a/gst/inferencecrop/videocrop.h +++ b/gst/inferencecrop/videocrop.h @@ -28,8 +28,10 @@ class VideoCrop : public CropElement { -protected: +public: const std::string GetFactory () const override; + +protected: void UpdateElement (GstElement * element, gint image_width, gint image_height, From df1d0ecfe714274cea7b6ca4b8fef2081bf75a57 Mon Sep 17 00:00:00 2001 From: Michael Gruner Date: Tue, 3 Sep 2019 23:35:08 -0600 Subject: [PATCH 010/188] Link crop element within the bin --- gst/inferencecrop/cropelement.h | 2 ++ gst/inferencecrop/gstdetectioncrop.cc | 39 ++++++++++++++++++------ gst/inferencecrop/videocrop.cc | 44 ++++++++++++++++++++++----- gst/inferencecrop/videocrop.h | 3 ++ 4 files changed, 72 insertions(+), 16 deletions(-) diff --git a/gst/inferencecrop/cropelement.h b/gst/inferencecrop/cropelement.h index 2285cb16..666bb820 100644 --- a/gst/inferencecrop/cropelement.h +++ b/gst/inferencecrop/cropelement.h @@ -36,6 +36,8 @@ class CropElement void SetBoundingBox (gint x, gint y, gint width, gint height); virtual ~CropElement (); virtual const std::string GetFactory () const = 0; + virtual GstPad * GetSinkPad () = 0; + virtual GstPad * GetSrcPad () = 0; protected: virtual void UpdateElement (GstElement * element, diff --git a/gst/inferencecrop/gstdetectioncrop.cc b/gst/inferencecrop/gstdetectioncrop.cc index e546e06a..84d92c47 100644 --- a/gst/inferencecrop/gstdetectioncrop.cc +++ b/gst/inferencecrop/gstdetectioncrop.cc @@ -110,7 +110,36 @@ gst_detection_crop_class_init (GstDetectionCropClass * klass) static void gst_detection_crop_init (GstDetectionCrop * self) { + GstElement *element; + GstPad *sinkpad, *sinkgpad, *srcpad, *srcgpad; + self->element = new VideoCrop (); + + if (FALSE == self->element->Validate ()) { + const std::string factory = self->element->GetFactory (); + GST_ERROR_OBJECT (self, "Unable to find element %s", factory.c_str ()); + return; + } + + element = self->element->GetElement (); + gst_bin_add (GST_BIN (self), GST_ELEMENT (gst_object_ref (element))); + + sinkpad = self->element->GetSinkPad (); + g_return_if_fail (sinkpad); + + sinkgpad = gst_ghost_pad_new ("sink", sinkpad); + gst_pad_set_active (sinkgpad, TRUE); + gst_element_add_pad (GST_ELEMENT (self), sinkgpad); + + srcpad = self->element->GetSrcPad (); + g_return_if_fail (srcpad); + + srcgpad = gst_ghost_pad_new ("src", srcpad); + gst_pad_set_active (srcgpad, TRUE); + gst_element_add_pad (GST_ELEMENT (self), srcgpad); + + gst_object_unref (sinkpad); + gst_object_unref (srcpad); } static void @@ -126,17 +155,9 @@ gst_detection_crop_finalize (GObject * object) static gboolean gst_detection_crop_start (GstDetectionCrop * self) { - gboolean ret; - g_return_val_if_fail (self, FALSE); - ret = self->element->Validate (); - if (FALSE == ret) { - const std::string factory = self->element->GetFactory (); - GST_ERROR_OBJECT (self, "Unable to find element %s", factory.c_str ()); - } - - return ret; + return self->element->Validate (); } static GstStateChangeReturn diff --git a/gst/inferencecrop/videocrop.cc b/gst/inferencecrop/videocrop.cc index 3ce6eee6..4aa4d503 100644 --- a/gst/inferencecrop/videocrop.cc +++ b/gst/inferencecrop/videocrop.cc @@ -29,12 +29,42 @@ VideoCrop::GetFactory () const void VideoCrop::UpdateElement (GstElement * element, - gint image_width, - gint image_height, - gint x, - gint y, - gint width, - gint height) + gint image_width, + gint image_height, + gint x, + gint y, + gint width, + gint height) { - + gint top = y; + gint bottom = image_height - y - height; + gint left = x; + gint right = image_width - x - width; + + g_object_set (element, + "top", top, + "bottom", bottom, + "left", left, + "right", right, NULL); +} + +GstPad * +VideoCrop::GetSinkPad () +{ + return this->GetPad ("sink"); +} + +GstPad * +VideoCrop::GetSrcPad () +{ + return this->GetPad ("src"); +} + +GstPad * +VideoCrop::GetPad (const std::string &name) +{ + GstElement * element = this->GetElement (); + GstPad * pad = gst_element_get_static_pad (element, name.c_str()); + + return pad; } diff --git a/gst/inferencecrop/videocrop.h b/gst/inferencecrop/videocrop.h index 8b39a5d4..3889bdef 100644 --- a/gst/inferencecrop/videocrop.h +++ b/gst/inferencecrop/videocrop.h @@ -30,6 +30,8 @@ class VideoCrop : public CropElement { public: const std::string GetFactory () const override; + GstPad * GetSrcPad () override; + GstPad * GetSinkPad () override; protected: void UpdateElement (GstElement * element, @@ -40,6 +42,7 @@ class VideoCrop : public CropElement gint width, gint height) override; private: + GstPad * GetPad (const std::string &name); const std::string factory = "videocrop"; }; From d73f994b5f1508083aa65e166e5e54870f1d231d Mon Sep 17 00:00:00 2001 From: Michael Gruner Date: Tue, 3 Sep 2019 23:51:39 -0600 Subject: [PATCH 011/188] Capture caps when set --- gst/inferencecrop/cropelement.cc | 4 ++++ gst/inferencecrop/gstdetectioncrop.cc | 28 +++++++++++++++++++++++++++ gst/inferencecrop/videocrop.cc | 2 +- 3 files changed, 33 insertions(+), 1 deletion(-) diff --git a/gst/inferencecrop/cropelement.cc b/gst/inferencecrop/cropelement.cc index 9373453d..601a781f 100644 --- a/gst/inferencecrop/cropelement.cc +++ b/gst/inferencecrop/cropelement.cc @@ -66,6 +66,10 @@ CropElement::SetImageSize (gint width, gint height) this->mutex.lock (); this->image_width = width; this->image_height = height; + this->x = 0; + this->y = 0; + this->width = width; + this->height = height; this->UpdateElement (this->element, this->image_width, diff --git a/gst/inferencecrop/gstdetectioncrop.cc b/gst/inferencecrop/gstdetectioncrop.cc index 84d92c47..b5a84c82 100644 --- a/gst/inferencecrop/gstdetectioncrop.cc +++ b/gst/inferencecrop/gstdetectioncrop.cc @@ -61,6 +61,8 @@ static void gst_detection_crop_finalize (GObject * object); static GstStateChangeReturn gst_detection_crop_change_state (GstElement * element, GstStateChange transition); static gboolean gst_detection_crop_start (GstDetectionCrop * self); +static void gst_detection_crop_set_caps (GstPad * pad, GParamSpec * unused, + GstDetectionCrop * self); enum { @@ -131,6 +133,9 @@ gst_detection_crop_init (GstDetectionCrop * self) gst_pad_set_active (sinkgpad, TRUE); gst_element_add_pad (GST_ELEMENT (self), sinkgpad); + g_signal_connect (sinkgpad, "notify::caps", + G_CALLBACK (gst_detection_crop_set_caps), self); + srcpad = self->element->GetSrcPad (); g_return_if_fail (srcpad); @@ -189,6 +194,29 @@ gst_detection_crop_change_state (GstElement * element, return ret; } +static void +gst_detection_crop_set_caps (GstPad * pad, GParamSpec * unused, + GstDetectionCrop * self) +{ + GstCaps *caps; + GstStructure *st; + gint width; + gint height; + + g_object_get (pad, "caps", &caps, NULL); + if (NULL == caps) { + return; + } + + st = gst_caps_get_structure (caps, 0); + gst_structure_get_int (st, "width", &width); + gst_structure_get_int (st, "height", &height); + + GST_INFO_OBJECT (self, "Set new caps to %" GST_PTR_FORMAT, caps); + + self->element->SetImageSize (width, height); +} + static gboolean plugin_init (GstPlugin * plugin) { diff --git a/gst/inferencecrop/videocrop.cc b/gst/inferencecrop/videocrop.cc index 4aa4d503..2817c4a3 100644 --- a/gst/inferencecrop/videocrop.cc +++ b/gst/inferencecrop/videocrop.cc @@ -20,7 +20,7 @@ */ #include "videocrop.h" - +#include const std::string VideoCrop::GetFactory () const { From 90bd7b54b737b94bb79bdcb09f2df07d259dbafd Mon Sep 17 00:00:00 2001 From: Michael Gruner Date: Wed, 4 Sep 2019 00:20:11 -0600 Subject: [PATCH 012/188] Intercept detection meta and adjust crop accordingly --- gst/inferencecrop/cropelement.cc | 20 ++++++++---------- gst/inferencecrop/cropelement.h | 1 + gst/inferencecrop/gstdetectioncrop.cc | 29 +++++++++++++++++++++++++++ 3 files changed, 38 insertions(+), 12 deletions(-) diff --git a/gst/inferencecrop/cropelement.cc b/gst/inferencecrop/cropelement.cc index 601a781f..b6c9eede 100644 --- a/gst/inferencecrop/cropelement.cc +++ b/gst/inferencecrop/cropelement.cc @@ -60,25 +60,21 @@ CropElement::GetElement () return element; } +void +CropElement::Reset () +{ + this->SetBoundingBox(0, 0, this->image_width, this->image_height); +} + void CropElement::SetImageSize (gint width, gint height) { this->mutex.lock (); this->image_width = width; this->image_height = height; - this->x = 0; - this->y = 0; - this->width = width; - this->height = height; - - this->UpdateElement (this->element, - this->image_width, - this->image_height, - this->x, - this->y, - this->width, - this->height); this->mutex.unlock (); + + this->Reset (); } void diff --git a/gst/inferencecrop/cropelement.h b/gst/inferencecrop/cropelement.h index 666bb820..e48341d3 100644 --- a/gst/inferencecrop/cropelement.h +++ b/gst/inferencecrop/cropelement.h @@ -38,6 +38,7 @@ class CropElement virtual const std::string GetFactory () const = 0; virtual GstPad * GetSinkPad () = 0; virtual GstPad * GetSrcPad () = 0; + void Reset (); protected: virtual void UpdateElement (GstElement * element, diff --git a/gst/inferencecrop/gstdetectioncrop.cc b/gst/inferencecrop/gstdetectioncrop.cc index b5a84c82..99d9f97f 100644 --- a/gst/inferencecrop/gstdetectioncrop.cc +++ b/gst/inferencecrop/gstdetectioncrop.cc @@ -63,6 +63,8 @@ static GstStateChangeReturn gst_detection_crop_change_state (GstElement * static gboolean gst_detection_crop_start (GstDetectionCrop * self); static void gst_detection_crop_set_caps (GstPad * pad, GParamSpec * unused, GstDetectionCrop * self); +static GstPadProbeReturn gst_detection_crop_new_buffer (GstPad * pad, + GstPadProbeInfo * info, GstDetectionCrop * self); enum { @@ -135,6 +137,8 @@ gst_detection_crop_init (GstDetectionCrop * self) g_signal_connect (sinkgpad, "notify::caps", G_CALLBACK (gst_detection_crop_set_caps), self); + gst_pad_add_probe (sinkgpad, GST_PAD_PROBE_TYPE_BUFFER, + (GstPadProbeCallback) gst_detection_crop_new_buffer, self, NULL); srcpad = self->element->GetSrcPad (); g_return_if_fail (srcpad); @@ -217,6 +221,31 @@ gst_detection_crop_set_caps (GstPad * pad, GParamSpec * unused, self->element->SetImageSize (width, height); } +static GstPadProbeReturn +gst_detection_crop_new_buffer (GstPad * pad, GstPadProbeInfo * info, + GstDetectionCrop * self) +{ + GstBuffer *buffer; + GstDetectionMeta *meta; + + buffer = gst_pad_probe_info_get_buffer (info); + + meta = + (GstDetectionMeta *) gst_buffer_get_meta (buffer, + GST_DETECTION_META_API_TYPE); + if (NULL == meta || meta->num_boxes <= 0) { + GST_LOG_OBJECT (self, "No meta found, dropping buffer"); + return GST_PAD_PROBE_DROP; + } + + BBox box = meta->boxes[0]; + GST_LOG_OBJECT (self, "BBox: %fx%fx%fx%f", box.x, box.y, box.width, + box.height); + self->element->SetBoundingBox ((gint) box.x, (gint) box.y, (gint) box.width, + (gint) box.height); + return GST_PAD_PROBE_OK; +} + static gboolean plugin_init (GstPlugin * plugin) { From 3a3b0bc10e45b31ebc03d1102aca29533612e4d2 Mon Sep 17 00:00:00 2001 From: Michael Gruner Date: Wed, 4 Sep 2019 18:46:20 -0600 Subject: [PATCH 013/188] Install property to select the object to crop --- gst/inferencecrop/gstdetectioncrop.cc | 91 +++++++++++++++++++++++++-- 1 file changed, 86 insertions(+), 5 deletions(-) diff --git a/gst/inferencecrop/gstdetectioncrop.cc b/gst/inferencecrop/gstdetectioncrop.cc index 99d9f97f..7394171b 100644 --- a/gst/inferencecrop/gstdetectioncrop.cc +++ b/gst/inferencecrop/gstdetectioncrop.cc @@ -58,6 +58,10 @@ GST_DEBUG_CATEGORY_STATIC (gst_detection_crop_debug_category); #define GST_CAT_DEFAULT gst_detection_crop_debug_category static void gst_detection_crop_finalize (GObject * object); +static void gst_detection_crop_set_property (GObject * object, + guint property_id, const GValue * value, GParamSpec * pspec); +static void gst_detection_crop_get_property (GObject * object, + guint property_id, GValue * value, GParamSpec * pspec); static GstStateChangeReturn gst_detection_crop_change_state (GstElement * element, GstStateChange transition); static gboolean gst_detection_crop_start (GstDetectionCrop * self); @@ -66,15 +70,21 @@ static void gst_detection_crop_set_caps (GstPad * pad, GParamSpec * unused, static GstPadProbeReturn gst_detection_crop_new_buffer (GstPad * pad, GstPadProbeInfo * info, GstDetectionCrop * self); +#define PROP_CROP_INDEX_DEFAULT 0 +#define PROP_CROP_INDEX_MAX G_MAXUINT +#define PROP_CROP_INDEX_MIN 0 + enum { - PROP_0 + PROP_0, + PROP_CROP_INDEX, }; struct _GstDetectionCrop { GstBin parent; CropElement *element; + guint crop_index; }; struct _GstDetectionCropClass @@ -109,6 +119,13 @@ gst_detection_crop_class_init (GstDetectionCropClass * klass) GST_DEBUG_FUNCPTR (gst_detection_crop_change_state); object_class->finalize = gst_detection_crop_finalize; + object_class->set_property = gst_detection_crop_set_property; + object_class->get_property = gst_detection_crop_get_property; + + g_object_class_install_property (object_class, PROP_CROP_INDEX, + g_param_spec_uint ("crop-index", "Crop Index", "Index of the detected " + "object to crop in the prediction array", PROP_CROP_INDEX_MIN, + PROP_CROP_INDEX_MAX, PROP_CROP_INDEX_DEFAULT, G_PARAM_READWRITE)); } static void @@ -118,6 +135,7 @@ gst_detection_crop_init (GstDetectionCrop * self) GstPad *sinkpad, *sinkgpad, *srcpad, *srcgpad; self->element = new VideoCrop (); + self->crop_index = PROP_CROP_INDEX_DEFAULT; if (FALSE == self->element->Validate ()) { const std::string factory = self->element->GetFactory (); @@ -161,6 +179,46 @@ gst_detection_crop_finalize (GObject * object) G_OBJECT_CLASS (gst_detection_crop_parent_class)->finalize (object); } +static void +gst_detection_crop_set_property (GObject * object, guint property_id, + const GValue * value, GParamSpec * pspec) +{ + GstDetectionCrop *self = GST_DETECTION_CROP (object); + + GST_DEBUG_OBJECT (self, "set_property"); + + switch (property_id) { + case PROP_CROP_INDEX: + GST_OBJECT_LOCK (self); + self->crop_index = g_value_get_uint (value); + GST_OBJECT_UNLOCK (self); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); + break; + } +} + +static void +gst_detection_crop_get_property (GObject * object, guint property_id, + GValue * value, GParamSpec * pspec) +{ + GstDetectionCrop *self = GST_DETECTION_CROP (object); + + GST_DEBUG_OBJECT (self, "get_property"); + + switch (property_id) { + case PROP_CROP_INDEX: + GST_OBJECT_LOCK (self); + g_value_set_uint (value, self->crop_index); + GST_OBJECT_UNLOCK (self); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); + break; + } +} + static gboolean gst_detection_crop_start (GstDetectionCrop * self) { @@ -227,23 +285,46 @@ gst_detection_crop_new_buffer (GstPad * pad, GstPadProbeInfo * info, { GstBuffer *buffer; GstDetectionMeta *meta; + guint crop_index; + BBox box; + GstPadProbeReturn ret = GST_PAD_PROBE_DROP; + + GST_OBJECT_LOCK (self); + crop_index = self->crop_index; + GST_OBJECT_UNLOCK (self); buffer = gst_pad_probe_info_get_buffer (info); meta = (GstDetectionMeta *) gst_buffer_get_meta (buffer, GST_DETECTION_META_API_TYPE); - if (NULL == meta || meta->num_boxes <= 0) { + + if (NULL == meta) { GST_LOG_OBJECT (self, "No meta found, dropping buffer"); - return GST_PAD_PROBE_DROP; + goto out; + } + + if (meta->num_boxes <= 0) { + GST_LOG_OBJECT (self, "Meta has no valid objects"); + goto out; + } + + if (crop_index >= meta->num_boxes) { + GST_LOG_OBJECT (self, "Configured crop index is larger than " + "the amount of objects in the prediction"); + goto out; } - BBox box = meta->boxes[0]; + box = meta->boxes[crop_index]; GST_LOG_OBJECT (self, "BBox: %fx%fx%fx%f", box.x, box.y, box.width, box.height); self->element->SetBoundingBox ((gint) box.x, (gint) box.y, (gint) box.width, (gint) box.height); - return GST_PAD_PROBE_OK; + + ret = GST_PAD_PROBE_OK; + +out: + return ret; } static gboolean From 38230d2c7b4f3e767b8957371c8cce1a8553a6de Mon Sep 17 00:00:00 2001 From: Michael Gruner Date: Wed, 4 Sep 2019 18:46:32 -0600 Subject: [PATCH 014/188] Protect videocrop against negative values --- gst/inferencecrop/videocrop.cc | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/gst/inferencecrop/videocrop.cc b/gst/inferencecrop/videocrop.cc index 2817c4a3..92873141 100644 --- a/gst/inferencecrop/videocrop.cc +++ b/gst/inferencecrop/videocrop.cc @@ -41,6 +41,22 @@ VideoCrop::UpdateElement (GstElement * element, gint left = x; gint right = image_width - x - width; + if (top < 0) { + top = 0; + } + + if (bottom < 0) { + bottom = 0; + } + + if (left < 0) { + left = 0; + } + + if (right < 0) { + right = 0; + } + g_object_set (element, "top", top, "bottom", bottom, From fdd9b3856fbb7c3344f0afda16ddb0dbbed27fe8 Mon Sep 17 00:00:00 2001 From: Michael Gruner Date: Fri, 6 Sep 2019 11:25:56 -0600 Subject: [PATCH 015/188] Install missing public header --- gst-libs/gst/r2inference/Makefile.am | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/gst-libs/gst/r2inference/Makefile.am b/gst-libs/gst/r2inference/Makefile.am index 7616d45f..5e27624c 100644 --- a/gst-libs/gst/r2inference/Makefile.am +++ b/gst-libs/gst/r2inference/Makefile.am @@ -45,4 +45,10 @@ gstinferenceinclude_HEADERS= \ gsttensorflow.h \ gstinferencepreprocess.h \ gstinferencepostprocess.h \ - gstinferencedebug.h + gstinferencedebug.h \ + gstinferencemeta.h + + + + + From 1bc63916e8d795ea7c9f0800c511d45dbabbeb8b Mon Sep 17 00:00:00 2001 From: Michael Gruner Date: Thu, 12 Sep 2019 18:27:53 -0600 Subject: [PATCH 016/188] Allow the user to configure the class to crop on the detectioncrop --- gst/inferencecrop/gstdetectioncrop.cc | 102 ++++++++++++++++++++++++-- 1 file changed, 97 insertions(+), 5 deletions(-) diff --git a/gst/inferencecrop/gstdetectioncrop.cc b/gst/inferencecrop/gstdetectioncrop.cc index 7394171b..6e6964c2 100644 --- a/gst/inferencecrop/gstdetectioncrop.cc +++ b/gst/inferencecrop/gstdetectioncrop.cc @@ -69,15 +69,26 @@ static void gst_detection_crop_set_caps (GstPad * pad, GParamSpec * unused, GstDetectionCrop * self); static GstPadProbeReturn gst_detection_crop_new_buffer (GstPad * pad, GstPadProbeInfo * info, GstDetectionCrop * self); +static gint gst_detection_crop_find_by_index (GstDetectionCrop * self, + guint crop_index, GstDetectionMeta * meta); +static gint gst_detection_crop_find_by_class (GstDetectionCrop * self, + gint crop_class, GstDetectionMeta * meta); +static gint gst_detection_crop_find_index (GstDetectionCrop * self, + guint crop_index, gint crop_class, GstDetectionMeta * meta); #define PROP_CROP_INDEX_DEFAULT 0 #define PROP_CROP_INDEX_MAX G_MAXUINT #define PROP_CROP_INDEX_MIN 0 +#define PROP_CROP_CLASS_DEFAULT -1 +#define PROP_CROP_CLASS_MAX G_MAXINT +#define PROP_CROP_CLASS_MIN -1 + enum { PROP_0, PROP_CROP_INDEX, + PROP_CROP_CLASS, }; struct _GstDetectionCrop @@ -85,6 +96,7 @@ struct _GstDetectionCrop GstBin parent; CropElement *element; guint crop_index; + gint crop_class; }; struct _GstDetectionCropClass @@ -124,8 +136,17 @@ gst_detection_crop_class_init (GstDetectionCropClass * klass) g_object_class_install_property (object_class, PROP_CROP_INDEX, g_param_spec_uint ("crop-index", "Crop Index", "Index of the detected " - "object to crop in the prediction array", PROP_CROP_INDEX_MIN, + "object to crop in the prediction array. This will be ignored if " + "crop-class is set to a non-negative value", PROP_CROP_INDEX_MIN, PROP_CROP_INDEX_MAX, PROP_CROP_INDEX_DEFAULT, G_PARAM_READWRITE)); + + g_object_class_install_property (object_class, PROP_CROP_CLASS, + g_param_spec_int ("crop-class", "Crop Class", + "Object class to crop look for " + "cropping. If set to -1, crop-index will be used. If set to a non-negative " + "value, the detections will be iterated until a valid class is found and then " + "used that one for cropping.", PROP_CROP_CLASS_MIN, + PROP_CROP_CLASS_MAX, PROP_CROP_CLASS_DEFAULT, G_PARAM_READWRITE)); } static void @@ -136,6 +157,7 @@ gst_detection_crop_init (GstDetectionCrop * self) self->element = new VideoCrop (); self->crop_index = PROP_CROP_INDEX_DEFAULT; + self->crop_class = PROP_CROP_CLASS_DEFAULT; if (FALSE == self->element->Validate ()) { const std::string factory = self->element->GetFactory (); @@ -193,6 +215,11 @@ gst_detection_crop_set_property (GObject * object, guint property_id, self->crop_index = g_value_get_uint (value); GST_OBJECT_UNLOCK (self); break; + case PROP_CROP_CLASS: + GST_OBJECT_LOCK (self); + self->crop_class = g_value_get_int (value); + GST_OBJECT_UNLOCK (self); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); break; @@ -213,6 +240,11 @@ gst_detection_crop_get_property (GObject * object, guint property_id, g_value_set_uint (value, self->crop_index); GST_OBJECT_UNLOCK (self); break; + case PROP_CROP_CLASS: + GST_OBJECT_LOCK (self); + g_value_set_int (value, self->crop_class); + GST_OBJECT_UNLOCK (self); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); break; @@ -279,6 +311,63 @@ gst_detection_crop_set_caps (GstPad * pad, GParamSpec * unused, self->element->SetImageSize (width, height); } +static gint +gst_detection_crop_find_by_index (GstDetectionCrop * self, guint crop_index, + GstDetectionMeta * meta) +{ + gint ret = crop_index; + + g_return_val_if_fail (self, -1); + g_return_val_if_fail (meta, -1); + + if (crop_index >= meta->num_boxes) { + GST_LOG_OBJECT (self, "Configured crop index is larger than " + "the amount of objects in the prediction"); + + ret = -1; + } + + return ret; +} + +static gint +gst_detection_crop_find_by_class (GstDetectionCrop * self, gint crop_class, + GstDetectionMeta * meta) +{ + gint i; + gint ret = -1; + + g_return_val_if_fail (self, -1); + g_return_val_if_fail (meta, -1); + + for (i = 0; i < meta->num_boxes; ++i) { + if (meta->boxes[i].label == crop_class) { + ret = i; + break; + } + } + + if (-1 == ret) { + GST_LOG_OBJECT (self, "No valid class detected"); + } + + return ret; +} + +static gint +gst_detection_crop_find_index (GstDetectionCrop * self, guint crop_index, + gint crop_class, GstDetectionMeta * meta) +{ + g_return_val_if_fail (self, -1); + g_return_val_if_fail (meta, -1); + + if (-1 == crop_class) { + return gst_detection_crop_find_by_index (self, crop_index, meta); + } else { + return gst_detection_crop_find_by_class (self, crop_class, meta); + } +} + static GstPadProbeReturn gst_detection_crop_new_buffer (GstPad * pad, GstPadProbeInfo * info, GstDetectionCrop * self) @@ -286,11 +375,14 @@ gst_detection_crop_new_buffer (GstPad * pad, GstPadProbeInfo * info, GstBuffer *buffer; GstDetectionMeta *meta; guint crop_index; + gint crop_class; + gint requested_index; BBox box; GstPadProbeReturn ret = GST_PAD_PROBE_DROP; GST_OBJECT_LOCK (self); crop_index = self->crop_index; + crop_class = self->crop_class; GST_OBJECT_UNLOCK (self); buffer = gst_pad_probe_info_get_buffer (info); @@ -309,13 +401,13 @@ gst_detection_crop_new_buffer (GstPad * pad, GstPadProbeInfo * info, goto out; } - if (crop_index >= meta->num_boxes) { - GST_LOG_OBJECT (self, "Configured crop index is larger than " - "the amount of objects in the prediction"); + requested_index = + gst_detection_crop_find_index (self, crop_index, crop_class, meta); + if (-1 == requested_index) { goto out; } - box = meta->boxes[crop_index]; + box = meta->boxes[requested_index]; GST_LOG_OBJECT (self, "BBox: %fx%fx%fx%f", box.x, box.y, box.width, box.height); self->element->SetBoundingBox ((gint) box.x, (gint) box.y, (gint) box.width, From 9e7320b519bdecbc1a86c3e3ebf11236364dd839 Mon Sep 17 00:00:00 2001 From: tvlenin Date: Mon, 4 Nov 2019 08:05:52 -0600 Subject: [PATCH 017/188] Install property to set the output aspect ratio --- gst/inferencecrop/cropelement.cc | 6 ++++-- gst/inferencecrop/cropelement.h | 4 +++- gst/inferencecrop/gstdetectioncrop.cc | 28 ++++++++++++++++++++++++++- 3 files changed, 34 insertions(+), 4 deletions(-) diff --git a/gst/inferencecrop/cropelement.cc b/gst/inferencecrop/cropelement.cc index b6c9eede..bb6aad06 100644 --- a/gst/inferencecrop/cropelement.cc +++ b/gst/inferencecrop/cropelement.cc @@ -30,6 +30,8 @@ CropElement::CropElement () this->y = 0; this->width = 0; this->height = 0; + this->width_ratio = 0; + this->height_ratio = 0; } bool @@ -63,7 +65,7 @@ CropElement::GetElement () void CropElement::Reset () { - this->SetBoundingBox(0, 0, this->image_width, this->image_height); + this->SetBoundingBox(0, 0, this->image_width, this->image_height,1,1); } void @@ -78,7 +80,7 @@ CropElement::SetImageSize (gint width, gint height) } void -CropElement::SetBoundingBox (gint x, gint y, gint width, gint height) +CropElement::SetBoundingBox (gint x, gint y, gint width, gint height, gint width_ratio, gint height_ratio) { this->mutex.lock (); this->x = x; diff --git a/gst/inferencecrop/cropelement.h b/gst/inferencecrop/cropelement.h index e48341d3..0ed5d8f0 100644 --- a/gst/inferencecrop/cropelement.h +++ b/gst/inferencecrop/cropelement.h @@ -33,7 +33,7 @@ class CropElement bool Validate (); GstElement * GetElement (); void SetImageSize (gint width, gint height); - void SetBoundingBox (gint x, gint y, gint width, gint height); + void SetBoundingBox (gint x, gint y, gint width, gint height, gint width_ratio, gint height_ratio); virtual ~CropElement (); virtual const std::string GetFactory () const = 0; virtual GstPad * GetSinkPad () = 0; @@ -58,6 +58,8 @@ class CropElement gint width; gint height; std::mutex mutex; + gint width_ratio; + gint height_ratio; }; #endif //__CROP_ELEMENT_H__ diff --git a/gst/inferencecrop/gstdetectioncrop.cc b/gst/inferencecrop/gstdetectioncrop.cc index 6e6964c2..5ee8d398 100644 --- a/gst/inferencecrop/gstdetectioncrop.cc +++ b/gst/inferencecrop/gstdetectioncrop.cc @@ -84,11 +84,14 @@ static gint gst_detection_crop_find_index (GstDetectionCrop * self, #define PROP_CROP_CLASS_MAX G_MAXINT #define PROP_CROP_CLASS_MIN -1 +#define PROP_CROP_RATIO_DEFAULT "0:0" + enum { PROP_0, PROP_CROP_INDEX, PROP_CROP_CLASS, + PROP_CROP_ASPECT_RATIO, }; struct _GstDetectionCrop @@ -97,6 +100,8 @@ struct _GstDetectionCrop CropElement *element; guint crop_index; gint crop_class; + const gchar *aspect_ratio; + gchar **aspect_ratio_tokens; }; struct _GstDetectionCropClass @@ -147,6 +152,11 @@ gst_detection_crop_class_init (GstDetectionCropClass * klass) "value, the detections will be iterated until a valid class is found and then " "used that one for cropping.", PROP_CROP_CLASS_MIN, PROP_CROP_CLASS_MAX, PROP_CROP_CLASS_DEFAULT, G_PARAM_READWRITE)); + + g_object_class_install_property (object_class, PROP_CROP_ASPECT_RATIO, + g_param_spec_string ("aspect-ratio", "Aspect Ratio", + "Aspect ratio to crop the detections, width and height separated by ':'. " + "If set to 0:0 the detection crop ignore the aspect ratio.", PROP_CROP_RATIO_DEFAULT, G_PARAM_READWRITE)); } static void @@ -158,6 +168,7 @@ gst_detection_crop_init (GstDetectionCrop * self) self->element = new VideoCrop (); self->crop_index = PROP_CROP_INDEX_DEFAULT; self->crop_class = PROP_CROP_CLASS_DEFAULT; + self->aspect_ratio = PROP_CROP_RATIO_DEFAULT; if (FALSE == self->element->Validate ()) { const std::string factory = self->element->GetFactory (); @@ -220,6 +231,12 @@ gst_detection_crop_set_property (GObject * object, guint property_id, self->crop_class = g_value_get_int (value); GST_OBJECT_UNLOCK (self); break; + case PROP_CROP_ASPECT_RATIO: + GST_OBJECT_LOCK (self); + self->aspect_ratio = g_value_dup_string (value); + self->aspect_ratio_tokens = g_strsplit (g_value_get_string (value), ":", 0); + GST_OBJECT_UNLOCK (self); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); break; @@ -245,6 +262,11 @@ gst_detection_crop_get_property (GObject * object, guint property_id, g_value_set_int (value, self->crop_class); GST_OBJECT_UNLOCK (self); break; + case PROP_CROP_ASPECT_RATIO: + GST_OBJECT_LOCK (self); + g_value_set_string (value, self->aspect_ratio); + GST_OBJECT_UNLOCK (self); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); break; @@ -376,6 +398,8 @@ gst_detection_crop_new_buffer (GstPad * pad, GstPadProbeInfo * info, GstDetectionMeta *meta; guint crop_index; gint crop_class; + gint crop_width_ratio; + gint crop_height_ratio; gint requested_index; BBox box; GstPadProbeReturn ret = GST_PAD_PROBE_DROP; @@ -383,6 +407,8 @@ gst_detection_crop_new_buffer (GstPad * pad, GstPadProbeInfo * info, GST_OBJECT_LOCK (self); crop_index = self->crop_index; crop_class = self->crop_class; + crop_width_ratio = atoi(self->aspect_ratio_tokens[0]); + crop_height_ratio = atoi(self->aspect_ratio_tokens[1]); GST_OBJECT_UNLOCK (self); buffer = gst_pad_probe_info_get_buffer (info); @@ -411,7 +437,7 @@ gst_detection_crop_new_buffer (GstPad * pad, GstPadProbeInfo * info, GST_LOG_OBJECT (self, "BBox: %fx%fx%fx%f", box.x, box.y, box.width, box.height); self->element->SetBoundingBox ((gint) box.x, (gint) box.y, (gint) box.width, - (gint) box.height); + (gint) box.height, (gint) crop_width_ratio, (gint) crop_height_ratio); ret = GST_PAD_PROBE_OK; From 2a835f66b1b14ea0f4dcba890e2398aba21531c1 Mon Sep 17 00:00:00 2001 From: tvlenin Date: Mon, 4 Nov 2019 10:00:52 -0600 Subject: [PATCH 018/188] Crop detection with specific aspect ratio --- gst/inferencecrop/cropelement.cc | 10 +++++++--- gst/inferencecrop/cropelement.h | 4 +++- gst/inferencecrop/videocrop.cc | 34 +++++++++++++++++++++++++++++++- gst/inferencecrop/videocrop.h | 4 +++- 4 files changed, 46 insertions(+), 6 deletions(-) diff --git a/gst/inferencecrop/cropelement.cc b/gst/inferencecrop/cropelement.cc index bb6aad06..f67f287a 100644 --- a/gst/inferencecrop/cropelement.cc +++ b/gst/inferencecrop/cropelement.cc @@ -30,8 +30,8 @@ CropElement::CropElement () this->y = 0; this->width = 0; this->height = 0; - this->width_ratio = 0; - this->height_ratio = 0; + this->width_ratio = 1; + this->height_ratio = 1; } bool @@ -87,6 +87,8 @@ CropElement::SetBoundingBox (gint x, gint y, gint width, gint height, gint width this->y = y; this->width = width; this->height = height; + this->width_ratio = width_ratio; + this->height_ratio = height_ratio; this->UpdateElement (this->element, this->image_width, @@ -94,7 +96,9 @@ CropElement::SetBoundingBox (gint x, gint y, gint width, gint height, gint width this->x, this->y, this->width, - this->height); + this->height, + this->width_ratio, + this->height_ratio); this->mutex.unlock (); } diff --git a/gst/inferencecrop/cropelement.h b/gst/inferencecrop/cropelement.h index 0ed5d8f0..ac3cdd87 100644 --- a/gst/inferencecrop/cropelement.h +++ b/gst/inferencecrop/cropelement.h @@ -47,7 +47,9 @@ class CropElement gint x, gint y, gint width, - gint height) = 0; + gint height, + gint width_ratio, + gint height_ratio) = 0; private: GstElement * element; diff --git a/gst/inferencecrop/videocrop.cc b/gst/inferencecrop/videocrop.cc index 92873141..89cdb361 100644 --- a/gst/inferencecrop/videocrop.cc +++ b/gst/inferencecrop/videocrop.cc @@ -21,6 +21,8 @@ #include "videocrop.h" #include +#include + const std::string VideoCrop::GetFactory () const { @@ -34,13 +36,43 @@ VideoCrop::UpdateElement (GstElement * element, gint x, gint y, gint width, - gint height) + gint height, + gint width_ratio, + gint height_ratio) { gint top = y; gint bottom = image_height - y - height; gint left = x; gint right = image_width - x - width; + if(width_ratio <= height_ratio){ + if(width > height){ + top = top - round(((height_ratio * width) / width_ratio - height) / 2); + bottom = bottom - round(((height_ratio * width) / width_ratio - height) / 2); + }else{ + if(height <= (height_ratio * width)){ + top = top - round(((height_ratio * width) / width_ratio - height) / 2); + bottom = bottom - round(((height_ratio * width) / width_ratio - height) / 2); + }else{ + left = left - round(((width_ratio * height) / height_ratio - width) / 2); + right = right - round(((width_ratio * height) / height_ratio - width) / 2); + } + } + }else{ + if(width < height){ + left = left - round(((width_ratio * height) / height_ratio - width) / 2); + right = right - round(((width_ratio * height) / height_ratio - width) / 2); + }else{ + if(width > (width_ratio * height)){ + top = top - round(((height_ratio * width) / width_ratio - height) / 2); + bottom = bottom - round(((height_ratio * width) / width_ratio - height) / 2); + }else{ + left = left - round(((width_ratio * height) / height_ratio - width) / 2); + right = right - round(((width_ratio * height) / height_ratio - width) / 2); + } + } + } + if (top < 0) { top = 0; } diff --git a/gst/inferencecrop/videocrop.h b/gst/inferencecrop/videocrop.h index 3889bdef..c11d0845 100644 --- a/gst/inferencecrop/videocrop.h +++ b/gst/inferencecrop/videocrop.h @@ -40,7 +40,9 @@ class VideoCrop : public CropElement gint x, gint y, gint width, - gint height) override; + gint height, + gint width_ratio, + gint height_ratio) override; private: GstPad * GetPad (const std::string &name); const std::string factory = "videocrop"; From 5626454f1c2b8f4871cdb32210c8ed75f0790805 Mon Sep 17 00:00:00 2001 From: tvlenin Date: Mon, 4 Nov 2019 10:52:38 -0600 Subject: [PATCH 019/188] Add case to ignore the aspect ratio --- gst/inferencecrop/gstdetectioncrop.cc | 4 +-- gst/inferencecrop/videocrop.cc | 38 +++++++++++++++------------ 2 files changed, 23 insertions(+), 19 deletions(-) diff --git a/gst/inferencecrop/gstdetectioncrop.cc b/gst/inferencecrop/gstdetectioncrop.cc index 5ee8d398..4a34b187 100644 --- a/gst/inferencecrop/gstdetectioncrop.cc +++ b/gst/inferencecrop/gstdetectioncrop.cc @@ -156,7 +156,7 @@ gst_detection_crop_class_init (GstDetectionCropClass * klass) g_object_class_install_property (object_class, PROP_CROP_ASPECT_RATIO, g_param_spec_string ("aspect-ratio", "Aspect Ratio", "Aspect ratio to crop the detections, width and height separated by ':'. " - "If set to 0:0 the detection crop ignore the aspect ratio.", PROP_CROP_RATIO_DEFAULT, G_PARAM_READWRITE)); + "If set to 0:0 the detection crop ignore the aspect ratio.", PROP_CROP_RATIO_DEFAULT, G_PARAM_READWRITE)); } static void @@ -234,7 +234,7 @@ gst_detection_crop_set_property (GObject * object, guint property_id, case PROP_CROP_ASPECT_RATIO: GST_OBJECT_LOCK (self); self->aspect_ratio = g_value_dup_string (value); - self->aspect_ratio_tokens = g_strsplit (g_value_get_string (value), ":", 0); + self->aspect_ratio_tokens = g_strsplit (g_value_get_string (value), ":", 0); GST_OBJECT_UNLOCK (self); break; default: diff --git a/gst/inferencecrop/videocrop.cc b/gst/inferencecrop/videocrop.cc index 89cdb361..2ee7d184 100644 --- a/gst/inferencecrop/videocrop.cc +++ b/gst/inferencecrop/videocrop.cc @@ -44,34 +44,38 @@ VideoCrop::UpdateElement (GstElement * element, gint bottom = image_height - y - height; gint left = x; gint right = image_width - x - width; + if(width_ratio != 0 || height_ratio != 0 ){ - if(width_ratio <= height_ratio){ - if(width > height){ - top = top - round(((height_ratio * width) / width_ratio - height) / 2); - bottom = bottom - round(((height_ratio * width) / width_ratio - height) / 2); - }else{ - if(height <= (height_ratio * width)){ + if(width_ratio <= height_ratio){ + if(width > height){ top = top - round(((height_ratio * width) / width_ratio - height) / 2); bottom = bottom - round(((height_ratio * width) / width_ratio - height) / 2); }else{ - left = left - round(((width_ratio * height) / height_ratio - width) / 2); - right = right - round(((width_ratio * height) / height_ratio - width) / 2); + if(height <= (height_ratio * width)){ + top = top - round(((height_ratio * width) / width_ratio - height) / 2); + bottom = bottom - round(((height_ratio * width) / width_ratio - height) / 2); + }else{ + left = left - round(((width_ratio * height) / height_ratio - width) / 2); + right = right - round(((width_ratio * height) / height_ratio - width) / 2); + } } - } - }else{ - if(width < height){ - left = left - round(((width_ratio * height) / height_ratio - width) / 2); - right = right - round(((width_ratio * height) / height_ratio - width) / 2); }else{ - if(width > (width_ratio * height)){ - top = top - round(((height_ratio * width) / width_ratio - height) / 2); - bottom = bottom - round(((height_ratio * width) / width_ratio - height) / 2); - }else{ + if(width < height){ left = left - round(((width_ratio * height) / height_ratio - width) / 2); right = right - round(((width_ratio * height) / height_ratio - width) / 2); + }else{ + if(width > (width_ratio * height)){ + top = top - round(((height_ratio * width) / width_ratio - height) / 2); + bottom = bottom - round(((height_ratio * width) / width_ratio - height) / 2); + }else{ + left = left - round(((width_ratio * height) / height_ratio - width) / 2); + right = right - round(((width_ratio * height) / height_ratio - width) / 2); + } } } + } + if (top < 0) { top = 0; From 89f34deadf9b6bcf90ee7c8828475bcf534ce46b Mon Sep 17 00:00:00 2001 From: tvlenin Date: Mon, 4 Nov 2019 16:03:21 -0600 Subject: [PATCH 020/188] Set the default case to 1:1 --- gst/inferencecrop/gstdetectioncrop.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/gst/inferencecrop/gstdetectioncrop.cc b/gst/inferencecrop/gstdetectioncrop.cc index 4a34b187..e440ae1b 100644 --- a/gst/inferencecrop/gstdetectioncrop.cc +++ b/gst/inferencecrop/gstdetectioncrop.cc @@ -84,7 +84,7 @@ static gint gst_detection_crop_find_index (GstDetectionCrop * self, #define PROP_CROP_CLASS_MAX G_MAXINT #define PROP_CROP_CLASS_MIN -1 -#define PROP_CROP_RATIO_DEFAULT "0:0" +#define PROP_CROP_RATIO_DEFAULT "1:1" enum { @@ -156,7 +156,7 @@ gst_detection_crop_class_init (GstDetectionCropClass * klass) g_object_class_install_property (object_class, PROP_CROP_ASPECT_RATIO, g_param_spec_string ("aspect-ratio", "Aspect Ratio", "Aspect ratio to crop the detections, width and height separated by ':'. " - "If set to 0:0 the detection crop ignore the aspect ratio.", PROP_CROP_RATIO_DEFAULT, G_PARAM_READWRITE)); + "If set to 1:1 the detection crop ignore the aspect ratio.", PROP_CROP_RATIO_DEFAULT, G_PARAM_READWRITE)); } static void @@ -234,7 +234,6 @@ gst_detection_crop_set_property (GObject * object, guint property_id, case PROP_CROP_ASPECT_RATIO: GST_OBJECT_LOCK (self); self->aspect_ratio = g_value_dup_string (value); - self->aspect_ratio_tokens = g_strsplit (g_value_get_string (value), ":", 0); GST_OBJECT_UNLOCK (self); break; default: @@ -405,6 +404,7 @@ gst_detection_crop_new_buffer (GstPad * pad, GstPadProbeInfo * info, GstPadProbeReturn ret = GST_PAD_PROBE_DROP; GST_OBJECT_LOCK (self); + self->aspect_ratio_tokens = g_strsplit (self->aspect_ratio, ":", 0); crop_index = self->crop_index; crop_class = self->crop_class; crop_width_ratio = atoi(self->aspect_ratio_tokens[0]); From 519c23e971230392dccd0a6e4ab79d1bfd979861 Mon Sep 17 00:00:00 2001 From: tvlenin Date: Tue, 5 Nov 2019 16:08:40 -0600 Subject: [PATCH 021/188] Capture the aspect-ratio as a fraction --- gst-libs/gst/r2inference/Makefile.am | 19 +-- gst/inferencecrop/Makefile.am | 21 ++- gst/inferencecrop/cropelement.cc | 66 +++++---- gst/inferencecrop/cropelement.h | 46 +++--- gst/inferencecrop/gstdetectioncrop.cc | 205 ++++++++++++-------------- gst/inferencecrop/gstdetectioncrop.h | 3 +- gst/inferencecrop/videocrop.cc | 98 ++++++------ gst/inferencecrop/videocrop.h | 35 +++-- 8 files changed, 236 insertions(+), 257 deletions(-) diff --git a/gst-libs/gst/r2inference/Makefile.am b/gst-libs/gst/r2inference/Makefile.am index 5e27624c..e9512a66 100644 --- a/gst-libs/gst/r2inference/Makefile.am +++ b/gst-libs/gst/r2inference/Makefile.am @@ -9,7 +9,7 @@ libgstinference_@GST_API_VERSION@_la_SOURCES= \ gstbackend.cc \ gstncsdk.cc \ gsttensorflow.cc \ - gstinferencepreprocess.c \ + gstinferencepreprocess.c \ gstinferencepostprocess.c \ gstinferencedebug.c @@ -28,8 +28,8 @@ libgstinference_@GST_API_VERSION@_la_CFLAGS= \ libgstinference_@GST_API_VERSION@_la_LIBADD= \ $(GST_LIBS) \ - $(GST_BASE_LIBS) \ - -lgstvideo-@GST_API_VERSION@ \ + $(GST_BASE_LIBS) \ + -lgstvideo-@GST_API_VERSION@ \ $(GST_PLUGINS_BASE_LIBS) \ $(R2INFERENCE_LIBS) @@ -42,13 +42,8 @@ gstinferenceinclude_HEADERS= \ gstbackend.h \ gstbackendsubclass.h \ gstncsdk.h \ - gsttensorflow.h \ - gstinferencepreprocess.h \ + gsttensorflow.h \ + gstinferencepreprocess.h \ gstinferencepostprocess.h \ - gstinferencedebug.h \ - gstinferencemeta.h - - - - - + gstinferencedebug.h \ + gstinferencemeta.h diff --git a/gst/inferencecrop/Makefile.am b/gst/inferencecrop/Makefile.am index 98cf6969..0b96c0a0 100644 --- a/gst/inferencecrop/Makefile.am +++ b/gst/inferencecrop/Makefile.am @@ -1,12 +1,11 @@ plugin_LTLIBRARIES = libgstinferencecrop.la libgstinferencecrop_la_SOURCES = \ - gstdetectioncrop.cc \ - cropelement.cc \ - videocrop.cc + gstdetectioncrop.cc \ + cropelement.cc \ + videocrop.cc - -libgstinferencecrop_la_CFLAGS = \ +libgstinferencecrop_la_CFLAGS = \ $(GST_CFLAGS) \ $(GST_BASE_CFLAGS) \ $(GST_PLUGINS_BASE_CFLAGS) \ @@ -16,10 +15,10 @@ libgstinferencecrop_la_CXXFLAGS = \ $(GST_CFLAGS) \ $(GST_BASE_CFLAGS) \ $(GST_PLUGINS_BASE_CFLAGS) \ - -I$(top_srcdir)/gst-libs \ - -std=c++11 + -I$(top_srcdir)/gst-libs \ + -std=c++11 -libgstinferencecrop_la_LIBADD = \ +libgstinferencecrop_la_LIBADD = \ $(GST_LIBS) \ $(GST_BASE_LIBS) \ $(GST_PLUGINS_BASE_LIBS) \ @@ -32,6 +31,6 @@ libgstinferencecrop_la_LIBTOOLFLAGS = \ $(GST_PLUGIN_LIBTOOLFLAGS) noinst_HEADERS = \ - gstdetectioncrop.h \ - cropelement.h \ - videocrop.h + gstdetectioncrop.h \ + cropelement.h \ + videocrop.h diff --git a/gst/inferencecrop/cropelement.cc b/gst/inferencecrop/cropelement.cc index f67f287a..fd3d3640 100644 --- a/gst/inferencecrop/cropelement.cc +++ b/gst/inferencecrop/cropelement.cc @@ -21,8 +21,7 @@ #include "cropelement.h" -CropElement::CropElement () -{ +CropElement::CropElement () { this->element = nullptr; this->image_width = 0; this->image_height = 0; @@ -30,15 +29,14 @@ CropElement::CropElement () this->y = 0; this->width = 0; this->height = 0; - this->width_ratio = 1; - this->height_ratio = 1; + this->width_ratio = PROP_CROP_RATIO_DEFAULT_WIDTH; + this->height_ratio = PROP_CROP_RATIO_DEFAULT_HEIGHT; } bool -CropElement::Validate () -{ - GstElement * element; - +CropElement::Validate () { + GstElement *element; + this->mutex.lock (); if (nullptr == this->element) { const std::string factory = this->GetFactory (); @@ -46,31 +44,36 @@ CropElement::Validate () } element = this->element; this->mutex.unlock (); - + return element != nullptr; } GstElement * -CropElement::GetElement () -{ - GstElement * element; - +CropElement::GetElement () { + GstElement *element; + element = nullptr; + bool validated = Validate(); this->mutex.lock (); - element = this->element; + if (validated) { + element = this->element; + }else{ + const std::string factory = this->GetFactory (); + GST_ERROR_OBJECT (this->element, "Unable to initialize the element %s", factory.c_str ()); + } + this->mutex.unlock (); - + return element; } void -CropElement::Reset () -{ - this->SetBoundingBox(0, 0, this->image_width, this->image_height,1,1); +CropElement::Reset () { + this->SetBoundingBox(0, 0, this->image_width, this->image_height, + this->width_ratio, this->height_ratio); } void -CropElement::SetImageSize (gint width, gint height) -{ +CropElement::SetImageSize (gint width, gint height) { this->mutex.lock (); this->image_width = width; this->image_height = height; @@ -80,8 +83,8 @@ CropElement::SetImageSize (gint width, gint height) } void -CropElement::SetBoundingBox (gint x, gint y, gint width, gint height, gint width_ratio, gint height_ratio) -{ +CropElement::SetBoundingBox (gint x, gint y, gint width, gint height, + gint width_ratio, gint height_ratio) { this->mutex.lock (); this->x = x; this->y = y; @@ -91,19 +94,18 @@ CropElement::SetBoundingBox (gint x, gint y, gint width, gint height, gint width this->height_ratio = height_ratio; this->UpdateElement (this->element, - this->image_width, - this->image_height, - this->x, - this->y, - this->width, - this->height, - this->width_ratio, - this->height_ratio); + this->image_width, + this->image_height, + this->x, + this->y, + this->width, + this->height, + this->width_ratio, + this->height_ratio); this->mutex.unlock (); } -CropElement::~CropElement () -{ +CropElement::~CropElement () { if (nullptr != this->element) { gst_object_unref (this->element); this->element = nullptr; diff --git a/gst/inferencecrop/cropelement.h b/gst/inferencecrop/cropelement.h index ac3cdd87..ee43a50d 100644 --- a/gst/inferencecrop/cropelement.h +++ b/gst/inferencecrop/cropelement.h @@ -26,33 +26,35 @@ #include #include -class CropElement -{ -public: +#define PROP_CROP_RATIO_DEFAULT_WIDTH 1 +#define PROP_CROP_RATIO_DEFAULT_HEIGHT 1 +class CropElement { + public: CropElement (); bool Validate (); - GstElement * GetElement (); + GstElement *GetElement (); void SetImageSize (gint width, gint height); - void SetBoundingBox (gint x, gint y, gint width, gint height, gint width_ratio, gint height_ratio); - virtual ~CropElement (); - virtual const std::string GetFactory () const = 0; - virtual GstPad * GetSinkPad () = 0; - virtual GstPad * GetSrcPad () = 0; + void SetBoundingBox (gint x, gint y, gint width, gint height, gint width_ratio, + gint height_ratio); + virtual ~ CropElement (); + virtual const std::string& GetFactory () const = 0; + virtual GstPad *GetSinkPad () = 0; + virtual GstPad *GetSrcPad () = 0; void Reset (); - -protected: - virtual void UpdateElement (GstElement * element, - gint image_width, - gint image_height, - gint x, - gint y, - gint width, - gint height, - gint width_ratio, - gint height_ratio) = 0; -private: - GstElement * element; + protected: + virtual void UpdateElement (GstElement *element, + gint image_width, + gint image_height, + gint x, + gint y, + gint width, + gint height, + gint width_ratio, + gint height_ratio) = 0; + + private: + GstElement *element; gint image_width; gint image_height; gint x; diff --git a/gst/inferencecrop/gstdetectioncrop.cc b/gst/inferencecrop/gstdetectioncrop.cc index e440ae1b..1c88d8d0 100644 --- a/gst/inferencecrop/gstdetectioncrop.cc +++ b/gst/inferencecrop/gstdetectioncrop.cc @@ -28,7 +28,10 @@ * * Example launch line * |[ - * gst-launch-1.0 -v videotestsrc ! detectioncrop ! xvimagesink + * gst-launch-1.0 v4l2src device=$CAMERA ! videoconvert ! tee name=t t. ! videoscale ! queue ! + net.sink_model t. ! queue ! net.sink_bypass tinyyolov2 name=net model-location=$MODEL_LOCATION ! \ + backend=tensorflow backend::input-layer=$INPUT_LAYER backend::output-layer=OUTPUT_LAYER net.src_bypass \ + detectioncrop aspect-ratio="1:1" ! videoscale ! ximagesink sync=false * ]| * Process video frames from the camera using a detectioncrop model. * @@ -57,24 +60,24 @@ static GstStaticPadTemplate src_template = GST_STATIC_PAD_TEMPLATE ("src", GST_DEBUG_CATEGORY_STATIC (gst_detection_crop_debug_category); #define GST_CAT_DEFAULT gst_detection_crop_debug_category -static void gst_detection_crop_finalize (GObject * object); -static void gst_detection_crop_set_property (GObject * object, - guint property_id, const GValue * value, GParamSpec * pspec); -static void gst_detection_crop_get_property (GObject * object, - guint property_id, GValue * value, GParamSpec * pspec); +static void gst_detection_crop_finalize (GObject *object); +static void gst_detection_crop_set_property (GObject *object, + guint property_id, const GValue *value, GParamSpec *pspec); +static void gst_detection_crop_get_property (GObject *object, + guint property_id, GValue *value, GParamSpec *pspec); static GstStateChangeReturn gst_detection_crop_change_state (GstElement * element, GstStateChange transition); -static gboolean gst_detection_crop_start (GstDetectionCrop * self); -static void gst_detection_crop_set_caps (GstPad * pad, GParamSpec * unused, - GstDetectionCrop * self); -static GstPadProbeReturn gst_detection_crop_new_buffer (GstPad * pad, - GstPadProbeInfo * info, GstDetectionCrop * self); -static gint gst_detection_crop_find_by_index (GstDetectionCrop * self, - guint crop_index, GstDetectionMeta * meta); -static gint gst_detection_crop_find_by_class (GstDetectionCrop * self, - gint crop_class, GstDetectionMeta * meta); -static gint gst_detection_crop_find_index (GstDetectionCrop * self, - guint crop_index, gint crop_class, GstDetectionMeta * meta); +static gboolean gst_detection_crop_start (GstDetectionCrop *self); +static void gst_detection_crop_set_caps (GstPad *pad, GParamSpec *unused, + GstDetectionCrop *self); +static GstPadProbeReturn gst_detection_crop_new_buffer (GstPad *pad, + GstPadProbeInfo *info, GstDetectionCrop *self); +static gint gst_detection_crop_find_by_index (GstDetectionCrop *self, + guint crop_index, GstDetectionMeta *meta); +static gint gst_detection_crop_find_by_class (GstDetectionCrop *self, + gint crop_class, GstDetectionMeta *meta); +static gint gst_detection_crop_find_index (GstDetectionCrop *self, + guint crop_index, gint crop_class, GstDetectionMeta *meta); #define PROP_CROP_INDEX_DEFAULT 0 #define PROP_CROP_INDEX_MAX G_MAXUINT @@ -84,92 +87,88 @@ static gint gst_detection_crop_find_index (GstDetectionCrop * self, #define PROP_CROP_CLASS_MAX G_MAXINT #define PROP_CROP_CLASS_MIN -1 -#define PROP_CROP_RATIO_DEFAULT "1:1" - -enum -{ +enum { PROP_0, PROP_CROP_INDEX, PROP_CROP_CLASS, PROP_CROP_ASPECT_RATIO, }; -struct _GstDetectionCrop -{ +struct _GstDetectionCrop { GstBin parent; CropElement *element; guint crop_index; gint crop_class; - const gchar *aspect_ratio; - gchar **aspect_ratio_tokens; + gint width_ratio; + gint height_ratio; }; -struct _GstDetectionCropClass -{ +struct _GstDetectionCropClass { GstBinClass parent; }; /* class initialization */ G_DEFINE_TYPE_WITH_CODE (GstDetectionCrop, gst_detection_crop, - GST_TYPE_BIN, - GST_DEBUG_CATEGORY_INIT (gst_detection_crop_debug_category, "detectioncrop", - 0, "debug category for detectioncrop element")); + GST_TYPE_BIN, + GST_DEBUG_CATEGORY_INIT (gst_detection_crop_debug_category, "detectioncrop", + 0, "debug category for detectioncrop element")); static void -gst_detection_crop_class_init (GstDetectionCropClass * klass) -{ +gst_detection_crop_class_init (GstDetectionCropClass *klass) { GstElementClass *element_class = GST_ELEMENT_CLASS (klass); GObjectClass *object_class = G_OBJECT_CLASS (klass); gst_element_class_add_pad_template (element_class, - gst_static_pad_template_get (&sink_template)); + gst_static_pad_template_get (&sink_template)); gst_element_class_add_pad_template (element_class, - gst_static_pad_template_get (&src_template)); + gst_static_pad_template_get (&src_template)); gst_element_class_set_static_metadata (GST_ELEMENT_CLASS (klass), - "detectioncrop", "Filter", - "Crops an incoming image based on an inference prediction bounding box", - " Michael Gruner "); + "detectioncrop", "Filter", + "Crops an incoming image based on an inference prediction bounding box", + "Michael Gruner \n\t\t\t" + " Lenin Torres "); element_class->change_state = - GST_DEBUG_FUNCPTR (gst_detection_crop_change_state); + GST_DEBUG_FUNCPTR (gst_detection_crop_change_state); object_class->finalize = gst_detection_crop_finalize; object_class->set_property = gst_detection_crop_set_property; object_class->get_property = gst_detection_crop_get_property; g_object_class_install_property (object_class, PROP_CROP_INDEX, - g_param_spec_uint ("crop-index", "Crop Index", "Index of the detected " - "object to crop in the prediction array. This will be ignored if " - "crop-class is set to a non-negative value", PROP_CROP_INDEX_MIN, - PROP_CROP_INDEX_MAX, PROP_CROP_INDEX_DEFAULT, G_PARAM_READWRITE)); + g_param_spec_uint ("crop-index", "Crop Index", "Index of the detected " + "object to crop in the prediction array. This will be ignored if " + "crop-class is set to a non-negative value", PROP_CROP_INDEX_MIN, + PROP_CROP_INDEX_MAX, PROP_CROP_INDEX_DEFAULT, G_PARAM_READWRITE)); g_object_class_install_property (object_class, PROP_CROP_CLASS, - g_param_spec_int ("crop-class", "Crop Class", - "Object class to crop look for " - "cropping. If set to -1, crop-index will be used. If set to a non-negative " - "value, the detections will be iterated until a valid class is found and then " - "used that one for cropping.", PROP_CROP_CLASS_MIN, - PROP_CROP_CLASS_MAX, PROP_CROP_CLASS_DEFAULT, G_PARAM_READWRITE)); + g_param_spec_int ("crop-class", "Crop Class", + "Object class to crop look for " + "cropping. If set to -1, crop-index will be used. If set to a non-negative " + "value, the detections will be iterated until a valid class is found and then " + "used that one for cropping.", PROP_CROP_CLASS_MIN, + PROP_CROP_CLASS_MAX, PROP_CROP_CLASS_DEFAULT, G_PARAM_READWRITE)); g_object_class_install_property (object_class, PROP_CROP_ASPECT_RATIO, - g_param_spec_string ("aspect-ratio", "Aspect Ratio", - "Aspect ratio to crop the detections, width and height separated by ':'. " - "If set to 1:1 the detection crop ignore the aspect ratio.", PROP_CROP_RATIO_DEFAULT, G_PARAM_READWRITE)); + gst_param_spec_fraction ("aspect-ratio", "Aspect Ratio", + "Aspect ratio to crop the detections, width and height separated by '/'. " + "If set to 0/1 it maintains the aspect ratio of each bounding box.", 0, 1, G_MAXINT, 1, + PROP_CROP_RATIO_DEFAULT_WIDTH, PROP_CROP_RATIO_DEFAULT_HEIGHT, + G_PARAM_READWRITE )); } static void -gst_detection_crop_init (GstDetectionCrop * self) -{ +gst_detection_crop_init (GstDetectionCrop *self) { GstElement *element; GstPad *sinkpad, *sinkgpad, *srcpad, *srcgpad; self->element = new VideoCrop (); self->crop_index = PROP_CROP_INDEX_DEFAULT; self->crop_class = PROP_CROP_CLASS_DEFAULT; - self->aspect_ratio = PROP_CROP_RATIO_DEFAULT; - + self->width_ratio = PROP_CROP_RATIO_DEFAULT_WIDTH; + self->height_ratio = PROP_CROP_RATIO_DEFAULT_HEIGHT; if (FALSE == self->element->Validate ()) { const std::string factory = self->element->GetFactory (); GST_ERROR_OBJECT (self, "Unable to find element %s", factory.c_str ()); @@ -187,9 +186,9 @@ gst_detection_crop_init (GstDetectionCrop * self) gst_element_add_pad (GST_ELEMENT (self), sinkgpad); g_signal_connect (sinkgpad, "notify::caps", - G_CALLBACK (gst_detection_crop_set_caps), self); + G_CALLBACK (gst_detection_crop_set_caps), self); gst_pad_add_probe (sinkgpad, GST_PAD_PROBE_TYPE_BUFFER, - (GstPadProbeCallback) gst_detection_crop_new_buffer, self, NULL); + (GstPadProbeCallback) gst_detection_crop_new_buffer, self, NULL); srcpad = self->element->GetSrcPad (); g_return_if_fail (srcpad); @@ -203,8 +202,7 @@ gst_detection_crop_init (GstDetectionCrop * self) } static void -gst_detection_crop_finalize (GObject * object) -{ +gst_detection_crop_finalize (GObject *object) { GstDetectionCrop *self = GST_DETECTION_CROP (object); delete (self->element); @@ -213,9 +211,8 @@ gst_detection_crop_finalize (GObject * object) } static void -gst_detection_crop_set_property (GObject * object, guint property_id, - const GValue * value, GParamSpec * pspec) -{ +gst_detection_crop_set_property (GObject *object, guint property_id, + const GValue *value, GParamSpec *pspec) { GstDetectionCrop *self = GST_DETECTION_CROP (object); GST_DEBUG_OBJECT (self, "set_property"); @@ -233,7 +230,10 @@ gst_detection_crop_set_property (GObject * object, guint property_id, break; case PROP_CROP_ASPECT_RATIO: GST_OBJECT_LOCK (self); - self->aspect_ratio = g_value_dup_string (value); + if (GST_VALUE_HOLDS_FRACTION (value)) { + self->width_ratio = gst_value_get_fraction_numerator (value); + self->height_ratio = gst_value_get_fraction_denominator (value); + } GST_OBJECT_UNLOCK (self); break; default: @@ -243,9 +243,8 @@ gst_detection_crop_set_property (GObject * object, guint property_id, } static void -gst_detection_crop_get_property (GObject * object, guint property_id, - GValue * value, GParamSpec * pspec) -{ +gst_detection_crop_get_property (GObject *object, guint property_id, + GValue *value, GParamSpec *pspec) { GstDetectionCrop *self = GST_DETECTION_CROP (object); GST_DEBUG_OBJECT (self, "get_property"); @@ -263,7 +262,7 @@ gst_detection_crop_get_property (GObject * object, guint property_id, break; case PROP_CROP_ASPECT_RATIO: GST_OBJECT_LOCK (self); - g_value_set_string (value, self->aspect_ratio); + gst_value_set_fraction (value, self->width_ratio, self->height_ratio); GST_OBJECT_UNLOCK (self); break; default: @@ -273,17 +272,15 @@ gst_detection_crop_get_property (GObject * object, guint property_id, } static gboolean -gst_detection_crop_start (GstDetectionCrop * self) -{ +gst_detection_crop_start (GstDetectionCrop *self) { g_return_val_if_fail (self, FALSE); return self->element->Validate (); } static GstStateChangeReturn -gst_detection_crop_change_state (GstElement * element, - GstStateChange transition) -{ +gst_detection_crop_change_state (GstElement *element, + GstStateChange transition) { GstStateChangeReturn ret; GstDetectionCrop *self = GST_DETECTION_CROP (element); @@ -298,8 +295,8 @@ gst_detection_crop_change_state (GstElement * element, break; } ret = - GST_ELEMENT_CLASS (gst_detection_crop_parent_class)->change_state - (element, transition); + GST_ELEMENT_CLASS (gst_detection_crop_parent_class)->change_state + (element, transition); if (GST_STATE_CHANGE_FAILURE == ret) { GST_ERROR_OBJECT (self, "Parent failed to change state"); goto out; @@ -310,9 +307,8 @@ gst_detection_crop_change_state (GstElement * element, } static void -gst_detection_crop_set_caps (GstPad * pad, GParamSpec * unused, - GstDetectionCrop * self) -{ +gst_detection_crop_set_caps (GstPad *pad, GParamSpec *unused, + GstDetectionCrop *self) { GstCaps *caps; GstStructure *st; gint width; @@ -330,20 +326,20 @@ gst_detection_crop_set_caps (GstPad * pad, GParamSpec * unused, GST_INFO_OBJECT (self, "Set new caps to %" GST_PTR_FORMAT, caps); self->element->SetImageSize (width, height); + gst_caps_unref(caps); } static gint -gst_detection_crop_find_by_index (GstDetectionCrop * self, guint crop_index, - GstDetectionMeta * meta) -{ +gst_detection_crop_find_by_index (GstDetectionCrop *self, guint crop_index, + GstDetectionMeta *meta) { gint ret = crop_index; g_return_val_if_fail (self, -1); g_return_val_if_fail (meta, -1); - if (crop_index >= meta->num_boxes) { + if ((gint)crop_index >= meta->num_boxes) { GST_LOG_OBJECT (self, "Configured crop index is larger than " - "the amount of objects in the prediction"); + "the amount of objects in the prediction"); ret = -1; } @@ -352,9 +348,8 @@ gst_detection_crop_find_by_index (GstDetectionCrop * self, guint crop_index, } static gint -gst_detection_crop_find_by_class (GstDetectionCrop * self, gint crop_class, - GstDetectionMeta * meta) -{ +gst_detection_crop_find_by_class (GstDetectionCrop *self, gint crop_class, + GstDetectionMeta *meta) { gint i; gint ret = -1; @@ -376,9 +371,8 @@ gst_detection_crop_find_by_class (GstDetectionCrop * self, gint crop_class, } static gint -gst_detection_crop_find_index (GstDetectionCrop * self, guint crop_index, - gint crop_class, GstDetectionMeta * meta) -{ +gst_detection_crop_find_index (GstDetectionCrop *self, guint crop_index, + gint crop_class, GstDetectionMeta *meta) { g_return_val_if_fail (self, -1); g_return_val_if_fail (meta, -1); @@ -390,9 +384,8 @@ gst_detection_crop_find_index (GstDetectionCrop * self, guint crop_index, } static GstPadProbeReturn -gst_detection_crop_new_buffer (GstPad * pad, GstPadProbeInfo * info, - GstDetectionCrop * self) -{ +gst_detection_crop_new_buffer (GstPad *pad, GstPadProbeInfo *info, + GstDetectionCrop *self) { GstBuffer *buffer; GstDetectionMeta *meta; guint crop_index; @@ -404,18 +397,17 @@ gst_detection_crop_new_buffer (GstPad * pad, GstPadProbeInfo * info, GstPadProbeReturn ret = GST_PAD_PROBE_DROP; GST_OBJECT_LOCK (self); - self->aspect_ratio_tokens = g_strsplit (self->aspect_ratio, ":", 0); crop_index = self->crop_index; crop_class = self->crop_class; - crop_width_ratio = atoi(self->aspect_ratio_tokens[0]); - crop_height_ratio = atoi(self->aspect_ratio_tokens[1]); + crop_width_ratio = self->width_ratio; + crop_height_ratio = self->height_ratio; GST_OBJECT_UNLOCK (self); buffer = gst_pad_probe_info_get_buffer (info); meta = - (GstDetectionMeta *) gst_buffer_get_meta (buffer, - GST_DETECTION_META_API_TYPE); + (GstDetectionMeta *) gst_buffer_get_meta (buffer, + GST_DETECTION_META_API_TYPE); if (NULL == meta) { GST_LOG_OBJECT (self, "No meta found, dropping buffer"); @@ -428,16 +420,16 @@ gst_detection_crop_new_buffer (GstPad * pad, GstPadProbeInfo * info, } requested_index = - gst_detection_crop_find_index (self, crop_index, crop_class, meta); + gst_detection_crop_find_index (self, crop_index, crop_class, meta); if (-1 == requested_index) { goto out; } box = meta->boxes[requested_index]; GST_LOG_OBJECT (self, "BBox: %fx%fx%fx%f", box.x, box.y, box.width, - box.height); + box.height); self->element->SetBoundingBox ((gint) box.x, (gint) box.y, (gint) box.width, - (gint) box.height, (gint) crop_width_ratio, (gint) crop_height_ratio); + (gint) box.height, (gint) crop_width_ratio, (gint) crop_height_ratio); ret = GST_PAD_PROBE_OK; @@ -446,19 +438,18 @@ gst_detection_crop_new_buffer (GstPad * pad, GstPadProbeInfo * info, } static gboolean -plugin_init (GstPlugin * plugin) -{ +plugin_init (GstPlugin *plugin) { gboolean ret = TRUE; ret = - gst_element_register (plugin, "detectioncrop", GST_RANK_NONE, - GST_TYPE_DETECTION_CROP); + gst_element_register (plugin, "detectioncrop", GST_RANK_NONE, + GST_TYPE_DETECTION_CROP); return ret; } GST_PLUGIN_DEFINE (GST_VERSION_MAJOR, - GST_VERSION_MINOR, - inferencecrop, - "Crops an incoming image based on an inference prediction bounding box", - plugin_init, VERSION, "LGPL", PACKAGE_NAME, GST_PACKAGE_ORIGIN) + GST_VERSION_MINOR, + inferencecrop, + "Crops an incoming image based on an inference prediction bounding box", + plugin_init, VERSION, "LGPL", PACKAGE_NAME, GST_PACKAGE_ORIGIN) diff --git a/gst/inferencecrop/gstdetectioncrop.h b/gst/inferencecrop/gstdetectioncrop.h index 620b8712..5b812224 100644 --- a/gst/inferencecrop/gstdetectioncrop.h +++ b/gst/inferencecrop/gstdetectioncrop.h @@ -27,7 +27,8 @@ G_BEGIN_DECLS #define GST_TYPE_DETECTION_CROP gst_detection_crop_get_type () -G_DECLARE_FINAL_TYPE (GstDetectionCrop, gst_detection_crop, GST, DETECTION_CROP, GstBin) +G_DECLARE_FINAL_TYPE (GstDetectionCrop, gst_detection_crop, GST, DETECTION_CROP, + GstBin) G_END_DECLS diff --git a/gst/inferencecrop/videocrop.cc b/gst/inferencecrop/videocrop.cc index 2ee7d184..f1e82e5c 100644 --- a/gst/inferencecrop/videocrop.cc +++ b/gst/inferencecrop/videocrop.cc @@ -20,62 +20,53 @@ */ #include "videocrop.h" + #include #include -const std::string -VideoCrop::GetFactory () const -{ +const std::string& +VideoCrop::GetFactory () const { return this->factory; } - + void -VideoCrop::UpdateElement (GstElement * element, - gint image_width, - gint image_height, - gint x, - gint y, - gint width, - gint height, - gint width_ratio, - gint height_ratio) -{ +VideoCrop::UpdateElement (GstElement *element, + gint image_width, + gint image_height, + gint x, + gint y, + gint width, + gint height, + gint width_ratio, + gint height_ratio) { gint top = y; gint bottom = image_height - y - height; gint left = x; gint right = image_width - x - width; - if(width_ratio != 0 || height_ratio != 0 ){ - if(width_ratio <= height_ratio){ - if(width > height){ - top = top - round(((height_ratio * width) / width_ratio - height) / 2); - bottom = bottom - round(((height_ratio * width) / width_ratio - height) / 2); - }else{ - if(height <= (height_ratio * width)){ - top = top - round(((height_ratio * width) / width_ratio - height) / 2); - bottom = bottom - round(((height_ratio * width) / width_ratio - height) / 2); - }else{ - left = left - round(((width_ratio * height) / height_ratio - width) / 2); - right = right - round(((width_ratio * height) / height_ratio - width) / 2); - } + if (width_ratio > 0 && height_ratio > 0) { + gint top_bottom_modify = round(((height_ratio * width) / width_ratio - height) / + 2); + gint left_right_modify = round(((width_ratio * height) / height_ratio - width) / + 2); + if (width_ratio <= height_ratio) { + if (width > height) { + top = top - top_bottom_modify; + bottom = bottom - top_bottom_modify; + } else { + left = left - left_right_modify; + right = right - left_right_modify; } - }else{ - if(width < height){ - left = left - round(((width_ratio * height) / height_ratio - width) / 2); - right = right - round(((width_ratio * height) / height_ratio - width) / 2); - }else{ - if(width > (width_ratio * height)){ - top = top - round(((height_ratio * width) / width_ratio - height) / 2); - bottom = bottom - round(((height_ratio * width) / width_ratio - height) / 2); - }else{ - left = left - round(((width_ratio * height) / height_ratio - width) / 2); - right = right - round(((width_ratio * height) / height_ratio - width) / 2); - } + } else { + if (width < height) { + left = left - left_right_modify; + right = right - left_right_modify; + } else { + top = top - top_bottom_modify; + bottom = bottom - top_bottom_modify; } } - } - if (top < 0) { top = 0; @@ -92,31 +83,30 @@ VideoCrop::UpdateElement (GstElement * element, if (right < 0) { right = 0; } - + + g_return_if_fail (element); + g_object_set (element, - "top", top, - "bottom", bottom, - "left", left, - "right", right, NULL); + "top", top, + "bottom", bottom, + "left", left, + "right", right, NULL); } GstPad * -VideoCrop::GetSinkPad () -{ +VideoCrop::GetSinkPad () { return this->GetPad ("sink"); } GstPad * -VideoCrop::GetSrcPad () -{ +VideoCrop::GetSrcPad () { return this->GetPad ("src"); } GstPad * -VideoCrop::GetPad (const std::string &name) -{ - GstElement * element = this->GetElement (); - GstPad * pad = gst_element_get_static_pad (element, name.c_str()); +VideoCrop::GetPad (const std::string &name) { + GstElement *element = this->GetElement (); + GstPad *pad = gst_element_get_static_pad (element, name.c_str()); return pad; } diff --git a/gst/inferencecrop/videocrop.h b/gst/inferencecrop/videocrop.h index c11d0845..76a94f84 100644 --- a/gst/inferencecrop/videocrop.h +++ b/gst/inferencecrop/videocrop.h @@ -26,25 +26,24 @@ #include -class VideoCrop : public CropElement -{ -public: - const std::string GetFactory () const override; - GstPad * GetSrcPad () override; - GstPad * GetSinkPad () override; +class VideoCrop: public CropElement { + public: + const std::string& GetFactory () const override; + GstPad *GetSrcPad () override; + GstPad *GetSinkPad () override; -protected: - void UpdateElement (GstElement * element, - gint image_width, - gint image_height, - gint x, - gint y, - gint width, - gint height, - gint width_ratio, - gint height_ratio) override; -private: - GstPad * GetPad (const std::string &name); + protected: + void UpdateElement (GstElement *element, + gint image_width, + gint image_height, + gint x, + gint y, + gint width, + gint height, + gint width_ratio, + gint height_ratio) override; + private: + GstPad *GetPad (const std::string &name); const std::string factory = "videocrop"; }; From 42c02dc58e25a88b270465c4baffa0a5c9c74a98 Mon Sep 17 00:00:00 2001 From: tvlenin Date: Mon, 9 Dec 2019 13:13:58 -0600 Subject: [PATCH 022/188] Add new element property, to crop the nearest object --- gst/inferencecrop/gstdetectioncrop.cc | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/gst/inferencecrop/gstdetectioncrop.cc b/gst/inferencecrop/gstdetectioncrop.cc index 1c88d8d0..eb4c653a 100644 --- a/gst/inferencecrop/gstdetectioncrop.cc +++ b/gst/inferencecrop/gstdetectioncrop.cc @@ -87,11 +87,14 @@ static gint gst_detection_crop_find_index (GstDetectionCrop *self, #define PROP_CROP_CLASS_MAX G_MAXINT #define PROP_CROP_CLASS_MIN -1 +#define PROP_CROP_NEAREST_DEFAULT FALSE + enum { PROP_0, PROP_CROP_INDEX, PROP_CROP_CLASS, PROP_CROP_ASPECT_RATIO, + PROP_CROP_NEAREST, }; struct _GstDetectionCrop { @@ -101,6 +104,7 @@ struct _GstDetectionCrop { gint crop_class; gint width_ratio; gint height_ratio; + gboolean nearest_object; }; struct _GstDetectionCropClass { @@ -157,6 +161,11 @@ gst_detection_crop_class_init (GstDetectionCropClass *klass) { "If set to 0/1 it maintains the aspect ratio of each bounding box.", 0, 1, G_MAXINT, 1, PROP_CROP_RATIO_DEFAULT_WIDTH, PROP_CROP_RATIO_DEFAULT_HEIGHT, G_PARAM_READWRITE )); + g_object_class_install_property (object_class, PROP_CROP_NEAREST, + g_param_spec_boolean ("nearest-object", "Nearest Object", + "Crop the nearest object detected, if crop-class is set to a non-negative" + "value, it crops the nearest object of that class", PROP_CROP_NEAREST_DEFAULT, + G_PARAM_READWRITE )); } static void @@ -169,6 +178,7 @@ gst_detection_crop_init (GstDetectionCrop *self) { self->crop_class = PROP_CROP_CLASS_DEFAULT; self->width_ratio = PROP_CROP_RATIO_DEFAULT_WIDTH; self->height_ratio = PROP_CROP_RATIO_DEFAULT_HEIGHT; + self->nearest_object = PROP_CROP_NEAREST_DEFAULT; if (FALSE == self->element->Validate ()) { const std::string factory = self->element->GetFactory (); GST_ERROR_OBJECT (self, "Unable to find element %s", factory.c_str ()); @@ -236,6 +246,11 @@ gst_detection_crop_set_property (GObject *object, guint property_id, } GST_OBJECT_UNLOCK (self); break; + case PROP_CROP_NEAREST: + GST_OBJECT_LOCK (self); + self->nearest_object = g_value_get_boolean (value); + GST_OBJECT_UNLOCK (self); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); break; @@ -265,6 +280,11 @@ gst_detection_crop_get_property (GObject *object, guint property_id, gst_value_set_fraction (value, self->width_ratio, self->height_ratio); GST_OBJECT_UNLOCK (self); break; + case PROP_CROP_NEAREST: + GST_OBJECT_LOCK (self); + g_value_set_boolean (value, self->nearest_object); + GST_OBJECT_UNLOCK (self); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); break; From 074e6bcae716d226be7256d7ff82f2c5f6103fd1 Mon Sep 17 00:00:00 2001 From: tvlenin Date: Mon, 9 Dec 2019 14:16:41 -0600 Subject: [PATCH 023/188] Select the nearest object when flag is true --- gst/inferencecrop/gstdetectioncrop.cc | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/gst/inferencecrop/gstdetectioncrop.cc b/gst/inferencecrop/gstdetectioncrop.cc index eb4c653a..748b6adc 100644 --- a/gst/inferencecrop/gstdetectioncrop.cc +++ b/gst/inferencecrop/gstdetectioncrop.cc @@ -75,9 +75,9 @@ static GstPadProbeReturn gst_detection_crop_new_buffer (GstPad *pad, static gint gst_detection_crop_find_by_index (GstDetectionCrop *self, guint crop_index, GstDetectionMeta *meta); static gint gst_detection_crop_find_by_class (GstDetectionCrop *self, - gint crop_class, GstDetectionMeta *meta); + gint crop_class, gboolean nearest_object, GstDetectionMeta *meta); static gint gst_detection_crop_find_index (GstDetectionCrop *self, - guint crop_index, gint crop_class, GstDetectionMeta *meta); + guint crop_index, gint crop_class, gboolean nearest_object, GstDetectionMeta *meta); #define PROP_CROP_INDEX_DEFAULT 0 #define PROP_CROP_INDEX_MAX G_MAXUINT @@ -368,18 +368,23 @@ gst_detection_crop_find_by_index (GstDetectionCrop *self, guint crop_index, } static gint -gst_detection_crop_find_by_class (GstDetectionCrop *self, gint crop_class, +gst_detection_crop_find_by_class (GstDetectionCrop *self, gint crop_class, gboolean nearest_object, GstDetectionMeta *meta) { gint i; gint ret = -1; + gint near = -1; g_return_val_if_fail (self, -1); g_return_val_if_fail (meta, -1); for (i = 0; i < meta->num_boxes; ++i) { if (meta->boxes[i].label == crop_class) { - ret = i; - break; + if(meta->boxes[i].width*meta->boxes[i].height > near){ + ret = i; + if(false == nearest_object){ + break; + } + } } } @@ -392,14 +397,14 @@ gst_detection_crop_find_by_class (GstDetectionCrop *self, gint crop_class, static gint gst_detection_crop_find_index (GstDetectionCrop *self, guint crop_index, - gint crop_class, GstDetectionMeta *meta) { + gint crop_class, gboolean nearest_object, GstDetectionMeta *meta) { g_return_val_if_fail (self, -1); g_return_val_if_fail (meta, -1); if (-1 == crop_class) { return gst_detection_crop_find_by_index (self, crop_index, meta); } else { - return gst_detection_crop_find_by_class (self, crop_class, meta); + return gst_detection_crop_find_by_class (self, crop_class, nearest_object, meta); } } @@ -413,6 +418,7 @@ gst_detection_crop_new_buffer (GstPad *pad, GstPadProbeInfo *info, gint crop_width_ratio; gint crop_height_ratio; gint requested_index; + gboolean nearest_object; BBox box; GstPadProbeReturn ret = GST_PAD_PROBE_DROP; @@ -421,6 +427,7 @@ gst_detection_crop_new_buffer (GstPad *pad, GstPadProbeInfo *info, crop_class = self->crop_class; crop_width_ratio = self->width_ratio; crop_height_ratio = self->height_ratio; + nearest_object = self->nearest_object; GST_OBJECT_UNLOCK (self); buffer = gst_pad_probe_info_get_buffer (info); @@ -440,7 +447,7 @@ gst_detection_crop_new_buffer (GstPad *pad, GstPadProbeInfo *info, } requested_index = - gst_detection_crop_find_index (self, crop_index, crop_class, meta); + gst_detection_crop_find_index (self, crop_index, crop_class, nearest_object, meta); if (-1 == requested_index) { goto out; } From 8621d70d899cce33cbe703d02aac9b306f41afea Mon Sep 17 00:00:00 2001 From: tvlenin Date: Mon, 9 Dec 2019 14:44:57 -0600 Subject: [PATCH 024/188] Fix indentation --- gst/inferencecrop/gstdetectioncrop.cc | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/gst/inferencecrop/gstdetectioncrop.cc b/gst/inferencecrop/gstdetectioncrop.cc index 748b6adc..45c3b1d6 100644 --- a/gst/inferencecrop/gstdetectioncrop.cc +++ b/gst/inferencecrop/gstdetectioncrop.cc @@ -77,7 +77,8 @@ static gint gst_detection_crop_find_by_index (GstDetectionCrop *self, static gint gst_detection_crop_find_by_class (GstDetectionCrop *self, gint crop_class, gboolean nearest_object, GstDetectionMeta *meta); static gint gst_detection_crop_find_index (GstDetectionCrop *self, - guint crop_index, gint crop_class, gboolean nearest_object, GstDetectionMeta *meta); + guint crop_index, gint crop_class, gboolean nearest_object, + GstDetectionMeta *meta); #define PROP_CROP_INDEX_DEFAULT 0 #define PROP_CROP_INDEX_MAX G_MAXUINT @@ -158,14 +159,15 @@ gst_detection_crop_class_init (GstDetectionCropClass *klass) { g_object_class_install_property (object_class, PROP_CROP_ASPECT_RATIO, gst_param_spec_fraction ("aspect-ratio", "Aspect Ratio", "Aspect ratio to crop the detections, width and height separated by '/'. " - "If set to 0/1 it maintains the aspect ratio of each bounding box.", 0, 1, G_MAXINT, 1, + "If set to 0/1 it maintains the aspect ratio of each bounding box.", 0, 1, + G_MAXINT, 1, PROP_CROP_RATIO_DEFAULT_WIDTH, PROP_CROP_RATIO_DEFAULT_HEIGHT, G_PARAM_READWRITE )); g_object_class_install_property (object_class, PROP_CROP_NEAREST, g_param_spec_boolean ("nearest-object", "Nearest Object", "Crop the nearest object detected, if crop-class is set to a non-negative" - "value, it crops the nearest object of that class", PROP_CROP_NEAREST_DEFAULT, - G_PARAM_READWRITE )); + "value, it crops the nearest object of that class", PROP_CROP_NEAREST_DEFAULT, + G_PARAM_READWRITE )); } static void @@ -368,7 +370,8 @@ gst_detection_crop_find_by_index (GstDetectionCrop *self, guint crop_index, } static gint -gst_detection_crop_find_by_class (GstDetectionCrop *self, gint crop_class, gboolean nearest_object, +gst_detection_crop_find_by_class (GstDetectionCrop *self, gint crop_class, + gboolean nearest_object, GstDetectionMeta *meta) { gint i; gint ret = -1; @@ -379,9 +382,9 @@ gst_detection_crop_find_by_class (GstDetectionCrop *self, gint crop_class, gbool for (i = 0; i < meta->num_boxes; ++i) { if (meta->boxes[i].label == crop_class) { - if(meta->boxes[i].width*meta->boxes[i].height > near){ + if (meta->boxes[i].width * meta->boxes[i].height > near) { ret = i; - if(false == nearest_object){ + if (false == nearest_object) { break; } } @@ -404,7 +407,8 @@ gst_detection_crop_find_index (GstDetectionCrop *self, guint crop_index, if (-1 == crop_class) { return gst_detection_crop_find_by_index (self, crop_index, meta); } else { - return gst_detection_crop_find_by_class (self, crop_class, nearest_object, meta); + return gst_detection_crop_find_by_class (self, crop_class, nearest_object, + meta); } } @@ -447,7 +451,8 @@ gst_detection_crop_new_buffer (GstPad *pad, GstPadProbeInfo *info, } requested_index = - gst_detection_crop_find_index (self, crop_index, crop_class, nearest_object, meta); + gst_detection_crop_find_index (self, crop_index, crop_class, nearest_object, + meta); if (-1 == requested_index) { goto out; } From ac5246825a4f727cb915ae198394a69c2aee7020 Mon Sep 17 00:00:00 2001 From: tvlenin Date: Mon, 9 Dec 2019 15:21:07 -0600 Subject: [PATCH 025/188] Refactor, use biggest instead of nearest --- gst/inferencecrop/gstdetectioncrop.cc | 42 +++++++++++++-------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/gst/inferencecrop/gstdetectioncrop.cc b/gst/inferencecrop/gstdetectioncrop.cc index 45c3b1d6..05b2fc87 100644 --- a/gst/inferencecrop/gstdetectioncrop.cc +++ b/gst/inferencecrop/gstdetectioncrop.cc @@ -75,9 +75,9 @@ static GstPadProbeReturn gst_detection_crop_new_buffer (GstPad *pad, static gint gst_detection_crop_find_by_index (GstDetectionCrop *self, guint crop_index, GstDetectionMeta *meta); static gint gst_detection_crop_find_by_class (GstDetectionCrop *self, - gint crop_class, gboolean nearest_object, GstDetectionMeta *meta); + gint crop_class, gboolean biggest_object, GstDetectionMeta *meta); static gint gst_detection_crop_find_index (GstDetectionCrop *self, - guint crop_index, gint crop_class, gboolean nearest_object, + guint crop_index, gint crop_class, gboolean biggest_object, GstDetectionMeta *meta); #define PROP_CROP_INDEX_DEFAULT 0 @@ -88,14 +88,14 @@ static gint gst_detection_crop_find_index (GstDetectionCrop *self, #define PROP_CROP_CLASS_MAX G_MAXINT #define PROP_CROP_CLASS_MIN -1 -#define PROP_CROP_NEAREST_DEFAULT FALSE +#define PROP_CROP_BIGGEST_DEFAULT FALSE enum { PROP_0, PROP_CROP_INDEX, PROP_CROP_CLASS, PROP_CROP_ASPECT_RATIO, - PROP_CROP_NEAREST, + PROP_CROP_BIGGEST, }; struct _GstDetectionCrop { @@ -105,7 +105,7 @@ struct _GstDetectionCrop { gint crop_class; gint width_ratio; gint height_ratio; - gboolean nearest_object; + gboolean biggest_object; }; struct _GstDetectionCropClass { @@ -163,10 +163,10 @@ gst_detection_crop_class_init (GstDetectionCropClass *klass) { G_MAXINT, 1, PROP_CROP_RATIO_DEFAULT_WIDTH, PROP_CROP_RATIO_DEFAULT_HEIGHT, G_PARAM_READWRITE )); - g_object_class_install_property (object_class, PROP_CROP_NEAREST, - g_param_spec_boolean ("nearest-object", "Nearest Object", - "Crop the nearest object detected, if crop-class is set to a non-negative" - "value, it crops the nearest object of that class", PROP_CROP_NEAREST_DEFAULT, + g_object_class_install_property (object_class, PROP_CROP_BIGGEST, + g_param_spec_boolean ("biggest-object", "Biggest Object", + "Crop the biggest object detected, if crop-class is set to a non-negative" + "value, it crops the biggest object of that class", PROP_CROP_BIGGEST_DEFAULT, G_PARAM_READWRITE )); } @@ -180,7 +180,7 @@ gst_detection_crop_init (GstDetectionCrop *self) { self->crop_class = PROP_CROP_CLASS_DEFAULT; self->width_ratio = PROP_CROP_RATIO_DEFAULT_WIDTH; self->height_ratio = PROP_CROP_RATIO_DEFAULT_HEIGHT; - self->nearest_object = PROP_CROP_NEAREST_DEFAULT; + self->biggest_object = PROP_CROP_BIGGEST_DEFAULT; if (FALSE == self->element->Validate ()) { const std::string factory = self->element->GetFactory (); GST_ERROR_OBJECT (self, "Unable to find element %s", factory.c_str ()); @@ -248,9 +248,9 @@ gst_detection_crop_set_property (GObject *object, guint property_id, } GST_OBJECT_UNLOCK (self); break; - case PROP_CROP_NEAREST: + case PROP_CROP_BIGGEST: GST_OBJECT_LOCK (self); - self->nearest_object = g_value_get_boolean (value); + self->biggest_object = g_value_get_boolean (value); GST_OBJECT_UNLOCK (self); break; default: @@ -282,9 +282,9 @@ gst_detection_crop_get_property (GObject *object, guint property_id, gst_value_set_fraction (value, self->width_ratio, self->height_ratio); GST_OBJECT_UNLOCK (self); break; - case PROP_CROP_NEAREST: + case PROP_CROP_BIGGEST: GST_OBJECT_LOCK (self); - g_value_set_boolean (value, self->nearest_object); + g_value_set_boolean (value, self->biggest_object); GST_OBJECT_UNLOCK (self); break; default: @@ -371,7 +371,7 @@ gst_detection_crop_find_by_index (GstDetectionCrop *self, guint crop_index, static gint gst_detection_crop_find_by_class (GstDetectionCrop *self, gint crop_class, - gboolean nearest_object, + gboolean biggest_object, GstDetectionMeta *meta) { gint i; gint ret = -1; @@ -384,7 +384,7 @@ gst_detection_crop_find_by_class (GstDetectionCrop *self, gint crop_class, if (meta->boxes[i].label == crop_class) { if (meta->boxes[i].width * meta->boxes[i].height > near) { ret = i; - if (false == nearest_object) { + if (false == biggest_object) { break; } } @@ -400,14 +400,14 @@ gst_detection_crop_find_by_class (GstDetectionCrop *self, gint crop_class, static gint gst_detection_crop_find_index (GstDetectionCrop *self, guint crop_index, - gint crop_class, gboolean nearest_object, GstDetectionMeta *meta) { + gint crop_class, gboolean biggest_object, GstDetectionMeta *meta) { g_return_val_if_fail (self, -1); g_return_val_if_fail (meta, -1); if (-1 == crop_class) { return gst_detection_crop_find_by_index (self, crop_index, meta); } else { - return gst_detection_crop_find_by_class (self, crop_class, nearest_object, + return gst_detection_crop_find_by_class (self, crop_class, biggest_object, meta); } } @@ -422,7 +422,7 @@ gst_detection_crop_new_buffer (GstPad *pad, GstPadProbeInfo *info, gint crop_width_ratio; gint crop_height_ratio; gint requested_index; - gboolean nearest_object; + gboolean biggest_object; BBox box; GstPadProbeReturn ret = GST_PAD_PROBE_DROP; @@ -431,7 +431,7 @@ gst_detection_crop_new_buffer (GstPad *pad, GstPadProbeInfo *info, crop_class = self->crop_class; crop_width_ratio = self->width_ratio; crop_height_ratio = self->height_ratio; - nearest_object = self->nearest_object; + biggest_object = self->biggest_object; GST_OBJECT_UNLOCK (self); buffer = gst_pad_probe_info_get_buffer (info); @@ -451,7 +451,7 @@ gst_detection_crop_new_buffer (GstPad *pad, GstPadProbeInfo *info, } requested_index = - gst_detection_crop_find_index (self, crop_index, crop_class, nearest_object, + gst_detection_crop_find_index (self, crop_index, crop_class, biggest_object, meta); if (-1 == requested_index) { goto out; From 730b1dc0f70c7044964fde5b33a7277d18cbee58 Mon Sep 17 00:00:00 2001 From: tvlenin Date: Mon, 9 Dec 2019 15:25:21 -0600 Subject: [PATCH 026/188] Fix assignment of biggest bounding box --- gst/inferencecrop/gstdetectioncrop.cc | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/gst/inferencecrop/gstdetectioncrop.cc b/gst/inferencecrop/gstdetectioncrop.cc index 05b2fc87..06e1e827 100644 --- a/gst/inferencecrop/gstdetectioncrop.cc +++ b/gst/inferencecrop/gstdetectioncrop.cc @@ -375,14 +375,17 @@ gst_detection_crop_find_by_class (GstDetectionCrop *self, gint crop_class, GstDetectionMeta *meta) { gint i; gint ret = -1; - gint near = -1; + gint biggest = -1; + gint box_size = -1; g_return_val_if_fail (self, -1); g_return_val_if_fail (meta, -1); for (i = 0; i < meta->num_boxes; ++i) { if (meta->boxes[i].label == crop_class) { - if (meta->boxes[i].width * meta->boxes[i].height > near) { + box_size = meta->boxes[i].width * meta->boxes[i].height; + if (box_size > biggest) { + biggest = box_size; ret = i; if (false == biggest_object) { break; From 1ee83eea7ce6d8dedf88af207a323bd1452ae6ae Mon Sep 17 00:00:00 2001 From: tvlenin Date: Mon, 9 Dec 2019 15:38:04 -0600 Subject: [PATCH 027/188] Modify property description --- gst/inferencecrop/gstdetectioncrop.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/gst/inferencecrop/gstdetectioncrop.cc b/gst/inferencecrop/gstdetectioncrop.cc index 06e1e827..5873a2d6 100644 --- a/gst/inferencecrop/gstdetectioncrop.cc +++ b/gst/inferencecrop/gstdetectioncrop.cc @@ -165,8 +165,10 @@ gst_detection_crop_class_init (GstDetectionCropClass *klass) { G_PARAM_READWRITE )); g_object_class_install_property (object_class, PROP_CROP_BIGGEST, g_param_spec_boolean ("biggest-object", "Biggest Object", - "Crop the biggest object detected, if crop-class is set to a non-negative" - "value, it crops the biggest object of that class", PROP_CROP_BIGGEST_DEFAULT, + "Crop the biggest object detected of the class given in crop-class." + "The biggest object usually implies the nearest object to the camera." + "This only applies if crop-class is non-negative.", + PROP_CROP_BIGGEST_DEFAULT, G_PARAM_READWRITE )); } From 99f9f497052e709a25d12cdfb6d07eb7a5a9206a Mon Sep 17 00:00:00 2001 From: Carlos Rodriguez Date: Mon, 16 Dec 2019 12:47:37 -0600 Subject: [PATCH 028/188] Add initial InferenceMeta support This commit contains testing purposes code in order to test the meta with fixed values --- gst-libs/gst/r2inference/gstinferencemeta.c | 96 +++++++++++++++++++++ gst-libs/gst/r2inference/gstinferencemeta.h | 28 ++++++ 2 files changed, 124 insertions(+) diff --git a/gst-libs/gst/r2inference/gstinferencemeta.c b/gst-libs/gst/r2inference/gstinferencemeta.c index 8b3a50a8..a5e6af14 100644 --- a/gst-libs/gst/r2inference/gstinferencemeta.c +++ b/gst-libs/gst/r2inference/gstinferencemeta.c @@ -14,6 +14,12 @@ #include #include +#define NUM_TEST_PREDICTIONS 3 + +static gboolean gst_inference_meta_init (GstMeta * meta, + gpointer params, GstBuffer * buffer); +static void gst_inference_meta_free (GstMeta * meta, GstBuffer * buffer); + static gboolean gst_classification_meta_init (GstMeta * meta, gpointer params, GstBuffer * buffer); static void gst_classification_meta_free (GstMeta * meta, GstBuffer * buffer); @@ -32,6 +38,38 @@ static gboolean gst_detection_meta_scale (GstBuffer * transbuf, static gboolean gst_classification_meta_copy (GstBuffer * transbuf, GstMeta * meta, GstBuffer * buffer); +GType +gst_inference_meta_api_get_type (void) +{ + static volatile GType type = 0; + static const gchar *tags[] = + { GST_META_TAG_VIDEO_STR, GST_META_TAG_VIDEO_ORIENTATION_STR, + GST_META_TAG_VIDEO_SIZE_STR, NULL + }; + + if (g_once_init_enter (&type)) { + GType _type = gst_meta_api_type_register ("GstInferenceMetaAPI", tags); + g_once_init_leave (&type, _type); + } + return type; +} + +/* inference metadata */ +const GstMetaInfo * +gst_inference_meta_get_info (void) +{ + static const GstMetaInfo *inference_meta_info = NULL; + + if (g_once_init_enter (&inference_meta_info)) { + const GstMetaInfo *meta = gst_meta_register (GST_INFERENCE_META_API_TYPE, + "GstInferenceMeta", sizeof (GstInferenceMeta), + gst_inference_meta_init, gst_inference_meta_free, + NULL); + g_once_init_leave (&inference_meta_info, meta); + } + return inference_meta_info; +} + GType gst_embedding_meta_api_get_type (void) { @@ -319,3 +357,61 @@ gst_classification_meta_transform (GstBuffer * dest, GstMeta * meta, /* No transform supported */ return FALSE; } + +/* inference metadata functions */ +static gboolean +gst_inference_meta_init (GstMeta * meta, gpointer params, GstBuffer * buffer) +{ + GstInferenceMeta *imeta = (GstInferenceMeta *) meta; + Prediction *root; + gint i; + + /* Create root Prediction */ + imeta->prediction = g_malloc (sizeof (Prediction)); + root = imeta->prediction; + root->id = rand (); + root->enabled = TRUE; + + /* This code is for testing purposes only */ + root->num_predictions = NUM_TEST_PREDICTIONS; + root->predictions = g_malloc (root->num_predictions * sizeof (Prediction)); + for (i = 0; i < root->num_predictions; i++) { + Prediction *predict = &root->predictions[i]; + BBox *box; + predict->enabled = TRUE; + predict->id = root->id + i; + predict->predictions = NULL; + predict->box = g_malloc (sizeof (BBox)); + box = predict->box; + /* Generate predictions with different information for testing purposes */ + box->label = 14 + i; + box->prob = 1; + box->x = 10 * i; + box->y = 10 * i; + box->width = 100; + box->height = 100; + } + + return TRUE; +} + +static void +gst_inference_meta_free (GstMeta * meta, GstBuffer * buffer) +{ + GstInferenceMeta *imeta = (GstInferenceMeta *) meta; + + g_return_if_fail (meta != NULL); + g_return_if_fail (buffer != NULL); + + /* Free all the predictions tree recursively */ + if (imeta->prediction) { + Prediction *root = imeta->prediction; + gint i; + for (i = 0; i < root->num_predictions; i++) { + Prediction *predict = &root->predictions[i]; + g_free (predict->box); + } + g_free (root->predictions); + g_free (root); + } +} diff --git a/gst-libs/gst/r2inference/gstinferencemeta.h b/gst-libs/gst/r2inference/gstinferencemeta.h index 76bdc92d..0197459f 100644 --- a/gst-libs/gst/r2inference/gstinferencemeta.h +++ b/gst-libs/gst/r2inference/gstinferencemeta.h @@ -21,6 +21,8 @@ G_BEGIN_DECLS #define GST_CLASSIFICATION_META_INFO (gst_classification_meta_get_info()) #define GST_DETECTION_META_API_TYPE (gst_detection_meta_api_get_type()) #define GST_DETECTION_META_INFO (gst_detection_meta_get_info()) +#define GST_INFERENCE_META_API_TYPE (gst_inference_meta_api_get_type()) +#define GST_INFERENCE_META_INFO (gst_inference_meta_get_info()) /** * Basic bounding box structure for detection */ @@ -35,6 +37,29 @@ struct _BBox gdouble height; }; +/** + * Prediction structure for inference meta + */ +typedef struct _Prediction Prediction; +struct _Prediction +{ + gint id; + gboolean enabled; + gint num_predictions; + BBox *box; + Prediction *predictions; +}; + +/** + * Implements the placeholder for inference information. + */ +typedef struct _GstInferenceMeta GstInferenceMeta; +struct _GstInferenceMeta +{ + GstMeta meta; + Prediction *prediction; +}; + /** * Implements the placeholder for embedding information. */ @@ -68,6 +93,9 @@ struct _GstDetectionMeta BBox *boxes; }; +GType gst_inference_meta_api_get_type (void); +const GstMetaInfo *gst_inference_meta_get_info (void); + GType gst_embedding_meta_api_get_type (void); const GstMetaInfo *gst_embedding_meta_get_info (void); From 429a74787892ac6ca30041ad2362809757a08c31 Mon Sep 17 00:00:00 2001 From: Carlos Rodriguez Date: Mon, 16 Dec 2019 13:25:48 -0600 Subject: [PATCH 029/188] Add InferenceMeta to buffer This commit is for testing purposes only --- gst-libs/gst/r2inference/gstvideoinference.c | 25 +++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/gst-libs/gst/r2inference/gstvideoinference.c b/gst-libs/gst/r2inference/gstvideoinference.c index 01ea6863..2b0268ca 100644 --- a/gst-libs/gst/r2inference/gstvideoinference.c +++ b/gst-libs/gst/r2inference/gstvideoinference.c @@ -21,6 +21,7 @@ #include "gstvideoinference.h" #include "gstinferencebackends.h" +#include "gstinferencemeta.h" #include "gstbackend.h" #include @@ -48,6 +49,7 @@ GST_DEBUG_CATEGORY_STATIC (gst_video_inference_debug_category); enum { NEW_PREDICTION_SIGNAL, + NEW_INFERENCE_SIGNAL, LAST_SIGNAL }; @@ -73,6 +75,7 @@ struct _GstVideoInferencePrivate GstCollectPads *cpads; GstVideoInferencePad *sink_bypass_data; GstVideoInferencePad *sink_model_data; + const GstMetaInfo *inference_meta_info; GstPad *sink_bypass; GstPad *src_bypass; @@ -222,6 +225,11 @@ gst_video_inference_class_init (GstVideoInferenceClass * klass) G_SIGNAL_RUN_FIRST, 0, NULL, NULL, NULL, G_TYPE_NONE, 4, G_TYPE_POINTER, G_TYPE_POINTER, G_TYPE_POINTER, G_TYPE_POINTER); + gst_video_inference_signals[NEW_INFERENCE_SIGNAL] = + g_signal_new ("new-inference", G_TYPE_FROM_CLASS (klass), + G_SIGNAL_RUN_FIRST, 0, NULL, NULL, NULL, G_TYPE_NONE, 4, G_TYPE_POINTER, + G_TYPE_POINTER, G_TYPE_POINTER, G_TYPE_POINTER); + klass->start = NULL; klass->stop = NULL; klass->preprocess = NULL; @@ -240,6 +248,7 @@ gst_video_inference_init (GstVideoInference * self) priv->src_bypass = NULL; priv->sink_model = NULL; priv->src_model = NULL; + priv->inference_meta_info = gst_inference_meta_get_info (); priv->cpads = gst_collect_pads_new (); gst_collect_pads_set_function (priv->cpads, gst_video_inference_collected, @@ -834,8 +843,11 @@ gst_video_inference_postprocess (GstVideoInference * self, GstVideoInferencePad * pad_model, GstBuffer * buffer_bypass, GstVideoInferencePad * pad_bypass) { + GstVideoInferencePrivate *priv = GST_VIDEO_INFERENCE_PRIVATE (self); GstMeta *meta_model = NULL; GstMeta *meta_bypass = NULL; + GstMeta *inference_meta_model = NULL; + GstMeta *inference_meta_bypass = NULL; GstVideoFrame frame_model; GstVideoFrame frame_bypass; GstVideoInfo *info_model = NULL; @@ -882,7 +894,18 @@ gst_video_inference_postprocess (GstVideoInference * self, video_inference_transform_meta (buffer_model, info_model, meta_model, buffer_bypass, info_bypass); g_signal_emit (self, gst_video_inference_signals[NEW_PREDICTION_SIGNAL], 0, - meta_model, &frame_bypass, meta_bypass, pbpass); + meta_model, &frame_model, meta_bypass, pbpass); + + /* This code is for testing purposes only */ + inference_meta_model = + gst_buffer_add_meta (buffer_model, priv->inference_meta_info, NULL); + if (pbpass) { + inference_meta_bypass = + gst_buffer_add_meta (buffer_bypass, priv->inference_meta_info, NULL); + } + g_signal_emit (self, gst_video_inference_signals[NEW_INFERENCE_SIGNAL], 0, + inference_meta_model, &frame_model, inference_meta_bypass, + buffer_bypass ? &frame_bypass : NULL); } else { video_inference_remove_meta (buffer_model, meta_model); video_inference_remove_meta (buffer_bypass, meta_bypass); From bffa9b5fd2510b25b2f6d50d10fe64dc4fd3da23 Mon Sep 17 00:00:00 2001 From: Carlos Rodriguez Date: Wed, 18 Dec 2019 13:34:56 -0600 Subject: [PATCH 030/188] Add GNode implementation to Inference Meta Also, testing code was removed. --- gst-libs/gst/r2inference/gstinferencemeta.c | 64 ++++++++++----------- gst-libs/gst/r2inference/gstinferencemeta.h | 3 +- 2 files changed, 30 insertions(+), 37 deletions(-) diff --git a/gst-libs/gst/r2inference/gstinferencemeta.c b/gst-libs/gst/r2inference/gstinferencemeta.c index a5e6af14..3d191124 100644 --- a/gst-libs/gst/r2inference/gstinferencemeta.c +++ b/gst-libs/gst/r2inference/gstinferencemeta.c @@ -19,7 +19,7 @@ static gboolean gst_inference_meta_init (GstMeta * meta, gpointer params, GstBuffer * buffer); static void gst_inference_meta_free (GstMeta * meta, GstBuffer * buffer); - +static gboolean gst_inference_clean_nodes (GNode * node, gpointer data); static gboolean gst_classification_meta_init (GstMeta * meta, gpointer params, GstBuffer * buffer); static void gst_classification_meta_free (GstMeta * meta, GstBuffer * buffer); @@ -364,54 +364,48 @@ gst_inference_meta_init (GstMeta * meta, gpointer params, GstBuffer * buffer) { GstInferenceMeta *imeta = (GstInferenceMeta *) meta; Prediction *root; - gint i; /* Create root Prediction */ - imeta->prediction = g_malloc (sizeof (Prediction)); - root = imeta->prediction; + root = g_malloc (sizeof (Prediction)); root->id = rand (); root->enabled = TRUE; + root->box = NULL; - /* This code is for testing purposes only */ - root->num_predictions = NUM_TEST_PREDICTIONS; - root->predictions = g_malloc (root->num_predictions * sizeof (Prediction)); - for (i = 0; i < root->num_predictions; i++) { - Prediction *predict = &root->predictions[i]; - BBox *box; - predict->enabled = TRUE; - predict->id = root->id + i; - predict->predictions = NULL; - predict->box = g_malloc (sizeof (BBox)); - box = predict->box; - /* Generate predictions with different information for testing purposes */ - box->label = 14 + i; - box->prob = 1; - box->x = 10 * i; - box->y = 10 * i; - box->width = 100; - box->height = 100; - } + imeta->prediction = root; + imeta->node = g_node_new (imeta->prediction); return TRUE; } +static gboolean +gst_inference_clean_nodes (GNode * node, gpointer data) +{ + Prediction *predict = (Prediction *) node->data; + + g_return_val_if_fail (predict != NULL, TRUE); + + /* Delete the Box in the Prediction */ + if (predict->box) { + g_free (predict->box); + predict->box = NULL; + } + /* Delete the prediction */ + g_free (predict); + + return FALSE; +} + static void gst_inference_meta_free (GstMeta * meta, GstBuffer * buffer) { - GstInferenceMeta *imeta = (GstInferenceMeta *) meta; + GstInferenceMeta *imeta = NULL; g_return_if_fail (meta != NULL); g_return_if_fail (buffer != NULL); - /* Free all the predictions tree recursively */ - if (imeta->prediction) { - Prediction *root = imeta->prediction; - gint i; - for (i = 0; i < root->num_predictions; i++) { - Prediction *predict = &root->predictions[i]; - g_free (predict->box); - } - g_free (root->predictions); - g_free (root); - } + imeta = (GstInferenceMeta *) meta; + + g_node_traverse (imeta->node, G_LEVEL_ORDER, G_TRAVERSE_ALL, -1, + gst_inference_clean_nodes, NULL); + g_node_destroy (imeta->node); } diff --git a/gst-libs/gst/r2inference/gstinferencemeta.h b/gst-libs/gst/r2inference/gstinferencemeta.h index 0197459f..36b06bb7 100644 --- a/gst-libs/gst/r2inference/gstinferencemeta.h +++ b/gst-libs/gst/r2inference/gstinferencemeta.h @@ -45,9 +45,7 @@ struct _Prediction { gint id; gboolean enabled; - gint num_predictions; BBox *box; - Prediction *predictions; }; /** @@ -58,6 +56,7 @@ struct _GstInferenceMeta { GstMeta meta; Prediction *prediction; + GNode *node; }; /** From 3ffdd0f2ebdd003b8e5adf5775ce2ee6ec9d9851 Mon Sep 17 00:00:00 2001 From: Carlos Rodriguez Date: Wed, 18 Dec 2019 13:48:46 -0600 Subject: [PATCH 031/188] Make create boxes function meta independent Create boxes function doesn't need the detection meta nor the VideoInfo to create boxes. --- ext/r2inference/gsttinyyolov2.c | 2 +- ext/r2inference/gsttinyyolov3.c | 2 +- .../gst/r2inference/gstinferencepostprocess.c | 6 ------ .../gst/r2inference/gstinferencepostprocess.h | 19 ++++++------------- 4 files changed, 8 insertions(+), 21 deletions(-) diff --git a/ext/r2inference/gsttinyyolov2.c b/ext/r2inference/gsttinyyolov2.c index c33f3799..d9fd4973 100644 --- a/ext/r2inference/gsttinyyolov2.c +++ b/ext/r2inference/gsttinyyolov2.c @@ -289,7 +289,7 @@ gst_tinyyolov2_postprocess (GstVideoInference * vi, const gpointer prediction, detect_meta->num_boxes = 0; tinyyolov2 = GST_TINYYOLOV2 (vi); - gst_create_boxes (vi, prediction, detect_meta, info_model, valid_prediction, + gst_create_boxes (vi, prediction, valid_prediction, &detect_meta->boxes, &detect_meta->num_boxes, tinyyolov2->obj_thresh, tinyyolov2->prob_thresh, tinyyolov2->iou_thresh); diff --git a/ext/r2inference/gsttinyyolov3.c b/ext/r2inference/gsttinyyolov3.c index f7a53ebb..ff6378dc 100644 --- a/ext/r2inference/gsttinyyolov3.c +++ b/ext/r2inference/gsttinyyolov3.c @@ -255,7 +255,7 @@ gst_tinyyolov3_postprocess (GstVideoInference * vi, const gpointer prediction, detect_meta->num_boxes = 0; tinyyolov3 = GST_TINYYOLOV3 (vi); - gst_create_boxes_float (vi, prediction, detect_meta, info_model, + gst_create_boxes_float (vi, prediction, valid_prediction, &detect_meta->boxes, &detect_meta->num_boxes, tinyyolov3->obj_thresh, tinyyolov3->prob_thresh, tinyyolov3->iou_thresh); diff --git a/gst-libs/gst/r2inference/gstinferencepostprocess.c b/gst-libs/gst/r2inference/gstinferencepostprocess.c index 576f5edf..f844978a 100644 --- a/gst-libs/gst/r2inference/gstinferencepostprocess.c +++ b/gst-libs/gst/r2inference/gstinferencepostprocess.c @@ -224,7 +224,6 @@ gst_get_boxes_from_prediction (gfloat obj_thresh, gfloat prob_thresh, gboolean gst_create_boxes (GstVideoInference * vi, const gpointer prediction, - GstDetectionMeta * detect_meta, GstVideoInfo * info_model, gboolean * valid_prediction, BBox ** resulting_boxes, gint * elements, gfloat obj_thresh, gfloat prob_thresh, gfloat iou_thresh) { @@ -236,8 +235,6 @@ gst_create_boxes (GstVideoInference * vi, const gpointer prediction, g_return_val_if_fail (vi != NULL, FALSE); g_return_val_if_fail (prediction != NULL, FALSE); - g_return_val_if_fail (detect_meta != NULL, FALSE); - g_return_val_if_fail (info_model != NULL, FALSE); g_return_val_if_fail (valid_prediction != NULL, FALSE); g_return_val_if_fail (resulting_boxes != NULL, FALSE); g_return_val_if_fail (elements != NULL, FALSE); @@ -307,7 +304,6 @@ gst_get_boxes_from_prediction_float (gfloat obj_thresh, gfloat prob_thresh, gboolean gst_create_boxes_float (GstVideoInference * vi, const gpointer prediction, - GstDetectionMeta * detect_meta, GstVideoInfo * info_model, gboolean * valid_prediction, BBox ** resulting_boxes, gint * elements, gdouble obj_thresh, gdouble prob_thresh, gdouble iou_thresh) @@ -318,8 +314,6 @@ gst_create_boxes_float (GstVideoInference * vi, const gpointer prediction, g_return_val_if_fail (vi != NULL, FALSE); g_return_val_if_fail (prediction != NULL, FALSE); - g_return_val_if_fail (detect_meta != NULL, FALSE); - g_return_val_if_fail (info_model != NULL, FALSE); g_return_val_if_fail (valid_prediction != NULL, FALSE); g_return_val_if_fail (resulting_boxes != NULL, FALSE); g_return_val_if_fail (elements != NULL, FALSE); diff --git a/gst-libs/gst/r2inference/gstinferencepostprocess.h b/gst-libs/gst/r2inference/gstinferencepostprocess.h index a691cfed..fd2059ee 100644 --- a/gst-libs/gst/r2inference/gstinferencepostprocess.h +++ b/gst-libs/gst/r2inference/gstinferencepostprocess.h @@ -26,7 +26,6 @@ #define __GST_INFERENCE_POSTPROCESS_H__ G_BEGIN_DECLS - /** * \brief Fill all the classification meta with predictions * @@ -34,17 +33,14 @@ G_BEGIN_DECLS * \param prediction Value of the prediction * \param predsize Size of the prediction */ - -gboolean gst_fill_classification_meta(GstClassificationMeta *class_meta, const gpointer prediction, - gsize predsize); + gboolean gst_fill_classification_meta (GstClassificationMeta * class_meta, + const gpointer prediction, gsize predsize); /** * \brief Fill all the detection meta with the boxes * * \param vi Father object of every architecture * \param prediction Value of the prediction - * \param detect_meta Meta to fill - * \param info_model Info about the model to use * \param valid_prediction Check if the prediction is valid * \param resulting_boxes The output boxes of the prediction * \param elements The number of objects @@ -53,7 +49,6 @@ gboolean gst_fill_classification_meta(GstClassificationMeta *class_meta, const g * \param iou_thresh Intersection over union threshold */ gboolean gst_create_boxes (GstVideoInference * vi, const gpointer prediction, - GstDetectionMeta *detect_meta, GstVideoInfo * info_model, gboolean * valid_prediction, BBox ** resulting_boxes, gint * elements, gfloat obj_thresh, gfloat prob_thresh, gfloat iou_thresh); @@ -62,8 +57,6 @@ gboolean gst_create_boxes (GstVideoInference * vi, const gpointer prediction, * * \param vi Father object of every architecture * \param prediction Value of the prediction - * \param detect_meta Meta to fill - * \param info_model Info about the model to use * \param valid_prediction Check if the prediction is valid * \param resulting_boxes The output boxes of the prediction * \param elements The number of objects @@ -71,10 +64,10 @@ gboolean gst_create_boxes (GstVideoInference * vi, const gpointer prediction, * \param prob_thresh Class probability threshold * \param iou_thresh Intersection over union threshold */ -gboolean gst_create_boxes_float (GstVideoInference * vi, const gpointer prediction, - GstDetectionMeta *detect_meta, GstVideoInfo * info_model, - gboolean * valid_prediction, BBox ** resulting_boxes, - gint * elements, gdouble obj_thresh, gdouble prob_thresh, gdouble iou_thresh); +gboolean gst_create_boxes_float (GstVideoInference * vi, + const gpointer prediction, gboolean * valid_prediction, + BBox ** resulting_boxes, gint * elements, gdouble obj_thresh, + gdouble prob_thresh, gdouble iou_thresh); G_END_DECLS From e48202d4c82c5a914bd681d1a5985b4000e89494 Mon Sep 17 00:00:00 2001 From: Carlos Rodriguez Date: Wed, 18 Dec 2019 13:53:55 -0600 Subject: [PATCH 032/188] Add new function to create prediction from box --- .../gst/r2inference/gstinferencepostprocess.c | 19 +++++++++++++++++++ .../gst/r2inference/gstinferencepostprocess.h | 9 ++++++++- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/gst-libs/gst/r2inference/gstinferencepostprocess.c b/gst-libs/gst/r2inference/gstinferencepostprocess.c index f844978a..d80ba98b 100644 --- a/gst-libs/gst/r2inference/gstinferencepostprocess.c +++ b/gst-libs/gst/r2inference/gstinferencepostprocess.c @@ -248,6 +248,25 @@ gst_create_boxes (GstVideoInference * vi, const gpointer prediction, return TRUE; } +Prediction * +gst_create_prediction_from_box (GstVideoInference * vi, BBox * box) +{ + Prediction *predict = NULL; + BBox *new_box = NULL; + + g_return_val_if_fail (vi != NULL, NULL); + g_return_val_if_fail (box != NULL, NULL); + + predict = g_malloc (sizeof (Prediction)); + new_box = g_malloc (sizeof (BBox)); + memcpy (new_box, box, sizeof (BBox)); + predict->box = new_box; + predict->id = rand (); + predict->enabled = TRUE; + + return predict; +} + static void gst_get_boxes_from_prediction_float (gfloat obj_thresh, gfloat prob_thresh, gpointer prediction, BBox * boxes, gint * elements, gint total_boxes) diff --git a/gst-libs/gst/r2inference/gstinferencepostprocess.h b/gst-libs/gst/r2inference/gstinferencepostprocess.h index fd2059ee..103abaac 100644 --- a/gst-libs/gst/r2inference/gstinferencepostprocess.h +++ b/gst-libs/gst/r2inference/gstinferencepostprocess.h @@ -69,6 +69,13 @@ gboolean gst_create_boxes_float (GstVideoInference * vi, BBox ** resulting_boxes, gint * elements, gdouble obj_thresh, gdouble prob_thresh, gdouble iou_thresh); -G_END_DECLS +/** + * \brief Create Prediction from box + * + * \param vi Father object of every architecture + * \param box Box used to fill Prediction + */ +Prediction *gst_create_prediction_from_box (GstVideoInference * vi, BBox * box); +G_END_DECLS #endif From d6ec749aeed21d75de28bf74280d8676b285225b Mon Sep 17 00:00:00 2001 From: Carlos Rodriguez Date: Wed, 18 Dec 2019 14:11:28 -0600 Subject: [PATCH 033/188] Add debug function to print predictions in inference meta --- gst-libs/gst/r2inference/gstinferencedebug.c | 38 ++++++++++++++++++++ gst-libs/gst/r2inference/gstinferencedebug.h | 24 ++++++++++--- 2 files changed, 57 insertions(+), 5 deletions(-) diff --git a/gst-libs/gst/r2inference/gstinferencedebug.c b/gst-libs/gst/r2inference/gstinferencedebug.c index ea84737b..2492932e 100644 --- a/gst-libs/gst/r2inference/gstinferencedebug.c +++ b/gst-libs/gst/r2inference/gstinferencedebug.c @@ -20,6 +20,8 @@ */ #include "gstinferencedebug.h" +static gboolean gst_inference_print_prediction (GNode * node, gpointer data); + void gst_inference_print_embedding (GstVideoInference * vi, GstDebugCategory * category, GstClassificationMeta * class_meta, @@ -92,3 +94,39 @@ gst_inference_print_boxes (GstVideoInference * vi, GstDebugCategory * category, detect_meta->boxes[index].height, detect_meta->boxes[index].prob); } } + +static gboolean +gst_inference_print_prediction (GNode * node, gpointer data) +{ + Prediction *predict = (Prediction *) node->data; + GstDebugCategory *category = (GstDebugCategory *) data; + BBox *box = NULL; + + g_return_val_if_fail (category != NULL, FALSE); + g_return_val_if_fail (predict != NULL, FALSE); + + box = predict->box; + + if (box != NULL) { + GST_CAT_LOG (category, + "Prediction ID:%d - Level:%d - Box: [class:%d, x:%f, y:%f, width:%f, height:%f, prob:%f]", + predict->id, g_node_depth (node), box->label, box->x, box->y, + box->width, box->height, box->prob); + } else { + GST_CAT_LOG (category, "Prediction ID:%d - Level:%d - No Box", + predict->id, g_node_depth (node)); + } + + return FALSE; +} + +void +gst_inference_print_predictions (GstVideoInference * vi, + GstDebugCategory * category, GstInferenceMeta * inference_meta) +{ + g_return_if_fail (vi != NULL); + g_return_if_fail (inference_meta != NULL); + + g_node_traverse (inference_meta->node, G_LEVEL_ORDER, G_TRAVERSE_ALL, -1, + gst_inference_print_prediction, (gpointer) category); +} diff --git a/gst-libs/gst/r2inference/gstinferencedebug.h b/gst-libs/gst/r2inference/gstinferencedebug.h index d4e4acde..1f8d61a9 100644 --- a/gst-libs/gst/r2inference/gstinferencedebug.h +++ b/gst-libs/gst/r2inference/gstinferencedebug.h @@ -25,7 +25,6 @@ #include G_BEGIN_DECLS - /** * \brief Display the vector with the predictions * @@ -35,8 +34,9 @@ G_BEGIN_DECLS * \param prediction Value of the prediction * \param gstlevel Level of debuging */ - -void gst_inference_print_embedding(GstVideoInference * vi, GstDebugCategory *category, GstClassificationMeta *class_meta, const gpointer prediction, GstDebugLevel gstlevel); +void gst_inference_print_embedding (GstVideoInference * vi, + GstDebugCategory * category, GstClassificationMeta * class_meta, + const gpointer prediction, GstDebugLevel gstlevel); /** * \brief Display the vector with the predictions @@ -48,7 +48,9 @@ void gst_inference_print_embedding(GstVideoInference * vi, GstDebugCategory *cat * \param gstlevel Level of debuging */ -void gst_inference_print_highest_probability(GstVideoInference * vi, GstDebugCategory *category, GstClassificationMeta *class_meta, const gpointer prediction, GstDebugLevel gstlevel); +void gst_inference_print_highest_probability (GstVideoInference * vi, + GstDebugCategory * category, GstClassificationMeta * class_meta, + const gpointer prediction, GstDebugLevel gstlevel); /** * \brief Display the vector with the predictions @@ -58,7 +60,19 @@ void gst_inference_print_highest_probability(GstVideoInference * vi, GstDebugCat * \param detect_meta Meta detected */ -void gst_inference_print_boxes(GstVideoInference * vi, GstDebugCategory *category, GstDetectionMeta *detect_meta); +void gst_inference_print_boxes (GstVideoInference * vi, + GstDebugCategory * category, GstDetectionMeta * detect_meta); + +/** + * \brief Display the predictions in the inference meta + * + * \param vi Father object of every architecture + * \param category The debug category + * \param inference_meta Inference Meta + */ + +void gst_inference_print_predictions (GstVideoInference * vi, + GstDebugCategory * category, GstInferenceMeta * inference_meta); G_END_DECLS #endif // GST_INFERENCE_DEBUG_H From fd450d880db3509b8a2cba4c47a0aa54c1af8e56 Mon Sep 17 00:00:00 2001 From: Carlos Rodriguez Date: Wed, 18 Dec 2019 14:15:55 -0600 Subject: [PATCH 034/188] Add new-meta property to select between old metas or new meta --- gst-libs/gst/r2inference/gstvideoinference.c | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/gst-libs/gst/r2inference/gstvideoinference.c b/gst-libs/gst/r2inference/gstvideoinference.c index 2b0268ca..34ba03e8 100644 --- a/gst-libs/gst/r2inference/gstvideoinference.c +++ b/gst-libs/gst/r2inference/gstvideoinference.c @@ -45,6 +45,7 @@ GST_DEBUG_CATEGORY_STATIC (gst_video_inference_debug_category); #define GST_CAT_DEFAULT gst_video_inference_debug_category #define DEFAULT_MODEL_LOCATION NULL +#define DEFAULT_NEW_META FALSE enum { @@ -57,7 +58,8 @@ enum { PROP_0, PROP_BACKEND, - PROP_MODEL_LOCATION + PROP_MODEL_LOCATION, + PROP_NEW_META }; @@ -85,6 +87,7 @@ struct _GstVideoInferencePrivate GstBackend *backend; gchar *model_location; + gboolean use_new_meta; }; /* GObject methods */ @@ -220,6 +223,11 @@ gst_video_inference_class_init (GstVideoInferenceClass * klass) "Path to the model to use", DEFAULT_MODEL_LOCATION, G_PARAM_READWRITE)); + g_object_class_install_property (oclass, PROP_NEW_META, + g_param_spec_boolean ("new-meta", "Use new Meta", + "Use new inference meta instead of old metas implementation", + DEFAULT_NEW_META, G_PARAM_READWRITE)); + gst_video_inference_signals[NEW_PREDICTION_SIGNAL] = g_signal_new ("new-prediction", G_TYPE_FROM_CLASS (klass), G_SIGNAL_RUN_FIRST, 0, NULL, NULL, NULL, G_TYPE_NONE, 4, G_TYPE_POINTER, @@ -249,6 +257,7 @@ gst_video_inference_init (GstVideoInference * self) priv->sink_model = NULL; priv->src_model = NULL; priv->inference_meta_info = gst_inference_meta_get_info (); + priv->use_new_meta = DEFAULT_NEW_META; priv->cpads = gst_collect_pads_new (); gst_collect_pads_set_function (priv->cpads, gst_video_inference_collected, @@ -291,6 +300,11 @@ gst_video_inference_set_property (GObject * object, } GST_OBJECT_UNLOCK (self); break; + case PROP_NEW_META: + GST_OBJECT_LOCK (self); + priv->use_new_meta = g_value_get_boolean (value); + GST_OBJECT_UNLOCK (self); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); break; @@ -313,6 +327,9 @@ gst_video_inference_get_property (GObject * object, case PROP_MODEL_LOCATION: g_value_set_string (value, priv->model_location); break; + case PROP_NEW_META: + g_value_set_boolean (value, priv->use_new_meta); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); break; From 4ba60869d84004b0fbc729f50a68e56559a15221 Mon Sep 17 00:00:00 2001 From: Carlos Rodriguez Date: Wed, 18 Dec 2019 14:26:49 -0600 Subject: [PATCH 035/188] Separate New Meta implementation from old implementation Using the new-meta property you can select which implementation to use. Additionally, if the new implementation is being used, we first check if the root prediction is enabled or not, in order to avoid processing that frame --- gst-libs/gst/r2inference/gstvideoinference.c | 135 +++++++++++++------ gst-libs/gst/r2inference/gstvideoinference.h | 23 ++-- 2 files changed, 106 insertions(+), 52 deletions(-) diff --git a/gst-libs/gst/r2inference/gstvideoinference.c b/gst-libs/gst/r2inference/gstvideoinference.c index 34ba03e8..cebd22a4 100644 --- a/gst-libs/gst/r2inference/gstvideoinference.c +++ b/gst-libs/gst/r2inference/gstvideoinference.c @@ -863,8 +863,6 @@ gst_video_inference_postprocess (GstVideoInference * self, GstVideoInferencePrivate *priv = GST_VIDEO_INFERENCE_PRIVATE (self); GstMeta *meta_model = NULL; GstMeta *meta_bypass = NULL; - GstMeta *inference_meta_model = NULL; - GstMeta *inference_meta_bypass = NULL; GstVideoFrame frame_model; GstVideoFrame frame_bypass; GstVideoInfo *info_model = NULL; @@ -878,54 +876,92 @@ gst_video_inference_postprocess (GstVideoInference * self, g_return_val_if_fail (buffer_model, FALSE); g_return_val_if_fail (pad_model, FALSE); - /* Subclass didn't implement a post-process, dont fail, just ignore */ - if (NULL == klass->postprocess) { - return TRUE; - } - info_model = &(pad_model->info); info_bypass = pad_bypass ? &(pad_bypass->info) : NULL; - if (!video_inference_prepare_postprocess (klass->inference_meta_info, - buffer_model, info_model, &frame_model, &meta_model)) { - return FALSE; - } + /* This conditional contains some duplicated code + that will be removed once the old metas implementation + is removed. For now let's keep it in that way to + avoid breaking the old method. + */ + if (priv->use_new_meta) { + /* Subclass didn't implement a post-process, dont fail, just ignore */ + if (NULL == klass->postprocess_meta) { + return TRUE; + } - if (!video_inference_prepare_postprocess (klass->inference_meta_info, - buffer_bypass, info_bypass, &frame_bypass, NULL)) { - return FALSE; - } + meta_model = + gst_buffer_get_meta (buffer_model, gst_inference_meta_api_get_type ()); + if (!meta_model) { + /* Create New Inference Meta */ + meta_model = + gst_buffer_add_meta (buffer_model, priv->inference_meta_info, NULL); + } - GST_LOG_OBJECT (self, "Calling frame postprocess"); - if (!klass->postprocess (self, prediction_data, prediction_size, meta_model, - info_model, &pred_valid)) { - GST_ELEMENT_ERROR (self, STREAM, FAILED, ("Subclass failed at preprocess"), - (NULL)); - return FALSE; - } + /* Prepare postprocess */ + if (!video_inference_prepare_postprocess (priv->inference_meta_info, + buffer_model, info_model, &frame_model, NULL)) { + return FALSE; + } + + if (!video_inference_prepare_postprocess (priv->inference_meta_info, + buffer_bypass, info_bypass, &frame_bypass, NULL)) { + return FALSE; + } - if (pred_valid) { - GstVideoFrame *pbpass = buffer_bypass ? &frame_bypass : NULL; - - meta_bypass = - video_inference_transform_meta (buffer_model, info_model, meta_model, - buffer_bypass, info_bypass); - g_signal_emit (self, gst_video_inference_signals[NEW_PREDICTION_SIGNAL], 0, - meta_model, &frame_model, meta_bypass, pbpass); - - /* This code is for testing purposes only */ - inference_meta_model = - gst_buffer_add_meta (buffer_model, priv->inference_meta_info, NULL); - if (pbpass) { - inference_meta_bypass = - gst_buffer_add_meta (buffer_bypass, priv->inference_meta_info, NULL); + /* Subclass Processing */ + if (!klass->postprocess_meta (self, prediction_data, prediction_size, + meta_model, info_model, &pred_valid)) { + GST_ELEMENT_ERROR (self, STREAM, FAILED, + ("Subclass failed at preprocess"), (NULL)); + return FALSE; + } + if (pred_valid) { + GstVideoFrame *pbpass = buffer_bypass ? &frame_bypass : NULL; + + /* TODO: Transfer model meta to bypass */ + + g_signal_emit (self, gst_video_inference_signals[NEW_INFERENCE_SIGNAL], 0, + meta_model, &frame_model, meta_bypass, pbpass); } - g_signal_emit (self, gst_video_inference_signals[NEW_INFERENCE_SIGNAL], 0, - inference_meta_model, &frame_model, inference_meta_bypass, - buffer_bypass ? &frame_bypass : NULL); } else { - video_inference_remove_meta (buffer_model, meta_model); - video_inference_remove_meta (buffer_bypass, meta_bypass); + /* Subclass didn't implement a post-process, dont fail, just ignore */ + if (NULL == klass->postprocess) { + return TRUE; + } + + if (!video_inference_prepare_postprocess (klass->inference_meta_info, + buffer_model, info_model, &frame_model, &meta_model)) { + return FALSE; + } + + if (!video_inference_prepare_postprocess (klass->inference_meta_info, + buffer_bypass, info_bypass, &frame_bypass, NULL)) { + return FALSE; + } + + GST_LOG_OBJECT (self, "Calling frame postprocess"); + if (!klass->postprocess (self, prediction_data, prediction_size, meta_model, + info_model, &pred_valid)) { + GST_ELEMENT_ERROR (self, STREAM, FAILED, + ("Subclass failed at preprocess"), (NULL)); + return FALSE; + } + + if (pred_valid) { + GstVideoFrame *pbpass = buffer_bypass ? &frame_bypass : NULL; + + meta_bypass = + video_inference_transform_meta (buffer_model, info_model, meta_model, + buffer_bypass, info_bypass); + g_signal_emit (self, gst_video_inference_signals[NEW_PREDICTION_SIGNAL], + 0, meta_model, &frame_model, meta_bypass, pbpass); + + } else { + video_inference_remove_meta (buffer_model, meta_model); + video_inference_remove_meta (buffer_bypass, meta_bypass); + } + } video_inference_frame_unmap (buffer_model, &frame_model); @@ -995,6 +1031,20 @@ gst_video_inference_collected (GstCollectPads * pads, gpointer user_data) } if (buffer_model) { + if (priv->use_new_meta) { + GstMeta *meta_model = gst_buffer_get_meta (buffer_model, + gst_inference_meta_api_get_type ()); + if (meta_model) { + /* Check if root is enabled to be processed, if not, just forward buffer */ + GstInferenceMeta *inference_meta = (GstInferenceMeta *) meta_model; + Prediction *root = inference_meta->prediction; + if (!root->enabled) { + GST_INFO_OBJECT (self, + "Current Prediction is not enabled, bypassing processing..."); + goto buffers_forward; + } + } + } /* Run preprocess and inference on the model and generate prediction */ if (!gst_video_inference_model_buffer_process (self, klass, priv, buffer_model, &prediction_data, &prediction_size)) { @@ -1011,6 +1061,7 @@ gst_video_inference_collected (GstCollectPads * pads, gpointer user_data) } } +buffers_forward: /* Forward buffer to model src pad */ ret = gst_video_inference_forward_buffer (self, buffer_model, priv->src_model); diff --git a/gst-libs/gst/r2inference/gstvideoinference.h b/gst-libs/gst/r2inference/gstvideoinference.h index 444ec824..006a439d 100644 --- a/gst-libs/gst/r2inference/gstvideoinference.h +++ b/gst-libs/gst/r2inference/gstvideoinference.h @@ -26,25 +26,28 @@ #include G_BEGIN_DECLS - #define GST_TYPE_VIDEO_INFERENCE gst_video_inference_get_type () -G_DECLARE_DERIVABLE_TYPE(GstVideoInference, gst_video_inference, GST, - VIDEO_INFERENCE, GstElement); +G_DECLARE_DERIVABLE_TYPE (GstVideoInference, gst_video_inference, GST, + VIDEO_INFERENCE, GstElement); struct _GstVideoInferenceClass { GstElementClass parent_class; - gboolean (* start) (GstVideoInference *self); - gboolean (* stop) (GstVideoInference *self); - gboolean (* preprocess) (GstVideoInference *self, GstVideoFrame *inframe, - GstVideoFrame *outframe); - gboolean (* postprocess) (GstVideoInference *self, const gpointer prediction, - gsize size, GstMeta *meta_model, GstVideoInfo *info_model, gboolean *valid_prediction); + gboolean (*start) (GstVideoInference * self); + gboolean (*stop) (GstVideoInference * self); + gboolean (*preprocess) (GstVideoInference * self, GstVideoFrame * inframe, + GstVideoFrame * outframe); + gboolean (*postprocess) (GstVideoInference * self, + const gpointer prediction, gsize size, GstMeta * meta_model, + GstVideoInfo * info_model, gboolean * valid_prediction); + gboolean (*postprocess_meta) (GstVideoInference * self, + const gpointer prediction, gsize size, GstMeta * meta_model, + GstVideoInfo * info_model, gboolean * valid_prediction); const GstMetaInfo *inference_meta_info; + }; G_END_DECLS - #endif //__GST_VIDEO_INFERENCE_H__ From 99398b431337fbc438e8ac37336604802b00fc8a Mon Sep 17 00:00:00 2001 From: Carlos Rodriguez Date: Wed, 18 Dec 2019 15:05:37 -0600 Subject: [PATCH 036/188] Implement process meta in tinyyolov2 --- ext/r2inference/gsttinyyolov2.c | 51 +++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/ext/r2inference/gsttinyyolov2.c b/ext/r2inference/gsttinyyolov2.c index d9fd4973..4b795292 100644 --- a/ext/r2inference/gsttinyyolov2.c +++ b/ext/r2inference/gsttinyyolov2.c @@ -83,6 +83,10 @@ static gboolean gst_tinyyolov2_postprocess (GstVideoInference * vi, const gpointer prediction, gsize predsize, GstMeta * meta_model, GstVideoInfo * info_model, gboolean * valid_prediction); +static gboolean +gst_tinyyolov2_postprocess_meta (GstVideoInference * vi, + const gpointer prediction, gsize predsize, GstMeta * meta_model, + GstVideoInfo * info_model, gboolean * valid_prediction); static gboolean gst_tinyyolov2_start (GstVideoInference * vi); static gboolean gst_tinyyolov2_stop (GstVideoInference * vi); @@ -180,6 +184,8 @@ gst_tinyyolov2_class_init (GstTinyyolov2Class * klass) vi_class->stop = GST_DEBUG_FUNCPTR (gst_tinyyolov2_stop); vi_class->preprocess = GST_DEBUG_FUNCPTR (gst_tinyyolov2_preprocess); vi_class->postprocess = GST_DEBUG_FUNCPTR (gst_tinyyolov2_postprocess); + vi_class->postprocess_meta = + GST_DEBUG_FUNCPTR (gst_tinyyolov2_postprocess_meta); vi_class->inference_meta_info = gst_detection_meta_get_info (); } @@ -300,6 +306,51 @@ gst_tinyyolov2_postprocess (GstVideoInference * vi, const gpointer prediction, return TRUE; } +static gboolean +gst_tinyyolov2_postprocess_meta (GstVideoInference * vi, + const gpointer prediction, gsize predsize, GstMeta * meta_model, + GstVideoInfo * info_model, gboolean * valid_prediction) +{ + GstTinyyolov2 *tinyyolov2 = NULL; + GstInferenceMeta *imeta = NULL; + BBox *boxes = NULL; + gint num_boxes, i; + + g_return_val_if_fail (vi != NULL, FALSE); + g_return_val_if_fail (meta_model != NULL, FALSE); + g_return_val_if_fail (info_model != NULL, FALSE); + + imeta = (GstInferenceMeta *) meta_model; + tinyyolov2 = GST_TINYYOLOV2 (vi); + + GST_LOG_OBJECT (tinyyolov2, "Postprocess Meta"); + + /* Create boxes from prediction data */ + gst_create_boxes (vi, prediction, valid_prediction, + &boxes, &num_boxes, tinyyolov2->obj_thresh, + tinyyolov2->prob_thresh, tinyyolov2->iou_thresh); + + GST_LOG_OBJECT (tinyyolov2, "Number of predictions: %d", num_boxes); + + for (i = 0; i < num_boxes; i++) { + Prediction *predict = gst_create_prediction_from_box (vi, &boxes[i]); + /* Create new node */ + GNode *new_node = g_node_new (predict); + /* Add new node as child of parent node */ + g_node_append (imeta->node, new_node); + } + + /* Free boxes after creation */ + g_free (boxes); + + /* Log predictions */ + gst_inference_print_predictions (vi, gst_tinyyolov2_debug_category, imeta); + + *valid_prediction = (num_boxes > 0) ? TRUE : FALSE; + + return TRUE; +} + static gboolean gst_tinyyolov2_start (GstVideoInference * vi) { From a101492ed426abc62c72e5c7f7b5ac9642f46072 Mon Sep 17 00:00:00 2001 From: Carlos Rodriguez Date: Thu, 19 Dec 2019 16:40:16 -0600 Subject: [PATCH 037/188] Handle node inside each Prediction only --- ext/r2inference/gsttinyyolov2.c | 3 ++- gst-libs/gst/r2inference/gstinferencedebug.c | 6 +++++- gst-libs/gst/r2inference/gstinferencemeta.c | 10 +++++----- gst-libs/gst/r2inference/gstinferencemeta.h | 2 +- 4 files changed, 13 insertions(+), 8 deletions(-) diff --git a/ext/r2inference/gsttinyyolov2.c b/ext/r2inference/gsttinyyolov2.c index 4b795292..6fd2a9bc 100644 --- a/ext/r2inference/gsttinyyolov2.c +++ b/ext/r2inference/gsttinyyolov2.c @@ -333,11 +333,12 @@ gst_tinyyolov2_postprocess_meta (GstVideoInference * vi, GST_LOG_OBJECT (tinyyolov2, "Number of predictions: %d", num_boxes); for (i = 0; i < num_boxes; i++) { + Prediction *root = imeta->prediction; Prediction *predict = gst_create_prediction_from_box (vi, &boxes[i]); /* Create new node */ GNode *new_node = g_node_new (predict); /* Add new node as child of parent node */ - g_node_append (imeta->node, new_node); + g_node_append (root->node, new_node); } /* Free boxes after creation */ diff --git a/gst-libs/gst/r2inference/gstinferencedebug.c b/gst-libs/gst/r2inference/gstinferencedebug.c index 2492932e..2f5344ca 100644 --- a/gst-libs/gst/r2inference/gstinferencedebug.c +++ b/gst-libs/gst/r2inference/gstinferencedebug.c @@ -124,9 +124,13 @@ void gst_inference_print_predictions (GstVideoInference * vi, GstDebugCategory * category, GstInferenceMeta * inference_meta) { + Prediction *root = NULL; + g_return_if_fail (vi != NULL); g_return_if_fail (inference_meta != NULL); - g_node_traverse (inference_meta->node, G_LEVEL_ORDER, G_TRAVERSE_ALL, -1, + root = inference_meta->prediction; + + g_node_traverse (root->node, G_LEVEL_ORDER, G_TRAVERSE_ALL, -1, gst_inference_print_prediction, (gpointer) category); } diff --git a/gst-libs/gst/r2inference/gstinferencemeta.c b/gst-libs/gst/r2inference/gstinferencemeta.c index 3d191124..ed4c7cc8 100644 --- a/gst-libs/gst/r2inference/gstinferencemeta.c +++ b/gst-libs/gst/r2inference/gstinferencemeta.c @@ -14,8 +14,6 @@ #include #include -#define NUM_TEST_PREDICTIONS 3 - static gboolean gst_inference_meta_init (GstMeta * meta, gpointer params, GstBuffer * buffer); static void gst_inference_meta_free (GstMeta * meta, GstBuffer * buffer); @@ -370,9 +368,9 @@ gst_inference_meta_init (GstMeta * meta, gpointer params, GstBuffer * buffer) root->id = rand (); root->enabled = TRUE; root->box = NULL; + root->node = g_node_new (root); imeta->prediction = root; - imeta->node = g_node_new (imeta->prediction); return TRUE; } @@ -399,13 +397,15 @@ static void gst_inference_meta_free (GstMeta * meta, GstBuffer * buffer) { GstInferenceMeta *imeta = NULL; + Prediction *root = NULL; g_return_if_fail (meta != NULL); g_return_if_fail (buffer != NULL); imeta = (GstInferenceMeta *) meta; + root = imeta->prediction; - g_node_traverse (imeta->node, G_LEVEL_ORDER, G_TRAVERSE_ALL, -1, + g_node_traverse (root->node, G_LEVEL_ORDER, G_TRAVERSE_ALL, -1, gst_inference_clean_nodes, NULL); - g_node_destroy (imeta->node); + g_node_destroy (root->node); } diff --git a/gst-libs/gst/r2inference/gstinferencemeta.h b/gst-libs/gst/r2inference/gstinferencemeta.h index 36b06bb7..af0e6b03 100644 --- a/gst-libs/gst/r2inference/gstinferencemeta.h +++ b/gst-libs/gst/r2inference/gstinferencemeta.h @@ -46,6 +46,7 @@ struct _Prediction gint id; gboolean enabled; BBox *box; + GNode *node; }; /** @@ -56,7 +57,6 @@ struct _GstInferenceMeta { GstMeta meta; Prediction *prediction; - GNode *node; }; /** From 668e64366ff8271428ae6c391563665bfc6ce809 Mon Sep 17 00:00:00 2001 From: Carlos Rodriguez Date: Thu, 19 Dec 2019 16:43:10 -0600 Subject: [PATCH 038/188] Add Classification structure to the Prediction --- gst-libs/gst/r2inference/gstinferencemeta.c | 25 ++++++++++++++++++- gst-libs/gst/r2inference/gstinferencemeta.h | 14 +++++++++++ .../gst/r2inference/gstinferencepostprocess.c | 1 + 3 files changed, 39 insertions(+), 1 deletion(-) diff --git a/gst-libs/gst/r2inference/gstinferencemeta.c b/gst-libs/gst/r2inference/gstinferencemeta.c index ed4c7cc8..f2d58134 100644 --- a/gst-libs/gst/r2inference/gstinferencemeta.c +++ b/gst-libs/gst/r2inference/gstinferencemeta.c @@ -368,6 +368,7 @@ gst_inference_meta_init (GstMeta * meta, gpointer params, GstBuffer * buffer) root->id = rand (); root->enabled = TRUE; root->box = NULL; + root->classifications = NULL; root->node = g_node_new (root); imeta->prediction = root; @@ -375,6 +376,23 @@ gst_inference_meta_init (GstMeta * meta, gpointer params, GstBuffer * buffer) return TRUE; } +static void +gst_inference_clean_classifications (gpointer data) +{ + Classification *class = (Classification *) data; + + g_return_if_fail (class != NULL); + + /* Delete class */ + if (class->classes_probs) + g_free (class->classes_probs); + + if (class->class_label) + g_free (class->class_label); + + g_free (class); +} + static gboolean gst_inference_clean_nodes (GNode * node, gpointer data) { @@ -383,10 +401,15 @@ gst_inference_clean_nodes (GNode * node, gpointer data) g_return_val_if_fail (predict != NULL, TRUE); /* Delete the Box in the Prediction */ - if (predict->box) { + if (predict->box != NULL) { g_free (predict->box); predict->box = NULL; } + /* Delete classifications */ + if (predict->classifications != NULL) { + g_list_free_full (predict->classifications, + gst_inference_clean_classifications); + } /* Delete the prediction */ g_free (predict); diff --git a/gst-libs/gst/r2inference/gstinferencemeta.h b/gst-libs/gst/r2inference/gstinferencemeta.h index af0e6b03..8a6706e0 100644 --- a/gst-libs/gst/r2inference/gstinferencemeta.h +++ b/gst-libs/gst/r2inference/gstinferencemeta.h @@ -37,6 +37,19 @@ struct _BBox gdouble height; }; +/** + * Basic class structure for inference + */ +typedef struct _Classification Classification; +struct _Classification +{ + gint class_id; + gdouble class_prob; + gchar *class_label; + gint num_classes; + gdouble *classes_probs; +}; + /** * Prediction structure for inference meta */ @@ -46,6 +59,7 @@ struct _Prediction gint id; gboolean enabled; BBox *box; + GList *classifications; GNode *node; }; diff --git a/gst-libs/gst/r2inference/gstinferencepostprocess.c b/gst-libs/gst/r2inference/gstinferencepostprocess.c index d80ba98b..09171d7b 100644 --- a/gst-libs/gst/r2inference/gstinferencepostprocess.c +++ b/gst-libs/gst/r2inference/gstinferencepostprocess.c @@ -263,6 +263,7 @@ gst_create_prediction_from_box (GstVideoInference * vi, BBox * box) predict->box = new_box; predict->id = rand (); predict->enabled = TRUE; + predict->classifications = NULL; return predict; } From 4ea4b5fc15a1f3d4cc33bb5f58964df8cd070571 Mon Sep 17 00:00:00 2001 From: Carlos Rodriguez Date: Fri, 20 Dec 2019 11:11:46 -0600 Subject: [PATCH 039/188] Add function to create class from prediction --- .../gst/r2inference/gstinferencepostprocess.c | 37 +++++++++++++++++++ .../gst/r2inference/gstinferencepostprocess.h | 10 +++++ 2 files changed, 47 insertions(+) diff --git a/gst-libs/gst/r2inference/gstinferencepostprocess.c b/gst-libs/gst/r2inference/gstinferencepostprocess.c index 09171d7b..5aac5203 100644 --- a/gst-libs/gst/r2inference/gstinferencepostprocess.c +++ b/gst-libs/gst/r2inference/gstinferencepostprocess.c @@ -268,6 +268,43 @@ gst_create_prediction_from_box (GstVideoInference * vi, BBox * box) return predict; } +Classification * +gst_create_class_from_prediction (GstVideoInference * vi, + const gpointer prediction, gsize predsize) +{ + Classification *new_class = NULL; + gdouble max = -1; + gint index = 0; + + g_return_val_if_fail (vi != NULL, NULL); + + /* Create new Classification */ + new_class = g_malloc (sizeof (Classification)); + /* Fill Classification with inference output */ + new_class->num_classes = predsize / sizeof (gfloat); + new_class->classes_probs = + g_malloc (new_class->num_classes * sizeof (gdouble)); + for (gint i = 0; i < new_class->num_classes; ++i) { + new_class->classes_probs[i] = (gdouble) ((gfloat *) prediction)[i]; + } + + /* Obtain the highest probability and set it as class */ + for (gint i = 0; i < new_class->num_classes; ++i) { + gfloat current = ((gfloat *) prediction)[i]; + if (current > max) { + max = current; + index = i; + } + } + + new_class->class_id = index; + new_class->class_prob = max; + /* TODO: Fill class label based on labels file */ + new_class->class_label = NULL; + + return new_class; +} + static void gst_get_boxes_from_prediction_float (gfloat obj_thresh, gfloat prob_thresh, gpointer prediction, BBox * boxes, gint * elements, gint total_boxes) diff --git a/gst-libs/gst/r2inference/gstinferencepostprocess.h b/gst-libs/gst/r2inference/gstinferencepostprocess.h index 103abaac..3b810c5d 100644 --- a/gst-libs/gst/r2inference/gstinferencepostprocess.h +++ b/gst-libs/gst/r2inference/gstinferencepostprocess.h @@ -77,5 +77,15 @@ gboolean gst_create_boxes_float (GstVideoInference * vi, */ Prediction *gst_create_prediction_from_box (GstVideoInference * vi, BBox * box); +/** + * \brief Create Classification from prediction data + * + * \param vi Father object of every architecture + * \param prediction Value of the prediction + * \param predsize Size of the prediction + */ +Classification *gst_create_class_from_prediction (GstVideoInference * vi, + const gpointer prediction, gsize predsize); + G_END_DECLS #endif From 022b0ca25b5fdb9f824ce599b7364ced2d4927c7 Mon Sep 17 00:00:00 2001 From: Carlos Rodriguez Date: Fri, 20 Dec 2019 11:17:53 -0600 Subject: [PATCH 040/188] Add helper to print prediction classes --- gst-libs/gst/r2inference/gstinferencedebug.c | 29 ++++++++++++++++++++ gst-libs/gst/r2inference/gstinferencedebug.h | 11 ++++++++ 2 files changed, 40 insertions(+) diff --git a/gst-libs/gst/r2inference/gstinferencedebug.c b/gst-libs/gst/r2inference/gstinferencedebug.c index 2f5344ca..3864bdb3 100644 --- a/gst-libs/gst/r2inference/gstinferencedebug.c +++ b/gst-libs/gst/r2inference/gstinferencedebug.c @@ -21,6 +21,7 @@ #include "gstinferencedebug.h" static gboolean gst_inference_print_prediction (GNode * node, gpointer data); +static void gst_inference_print_class (gpointer data, gpointer user_data); void gst_inference_print_embedding (GstVideoInference * vi, @@ -134,3 +135,31 @@ gst_inference_print_predictions (GstVideoInference * vi, g_node_traverse (root->node, G_LEVEL_ORDER, G_TRAVERSE_ALL, -1, gst_inference_print_prediction, (gpointer) category); } + +static void +gst_inference_print_class (gpointer data, gpointer user_data) +{ + Classification *class = (Classification *) data; + GstDebugCategory *category = (GstDebugCategory *) user_data; + + g_return_if_fail (category != NULL); + g_return_if_fail (class != NULL); + + GST_CAT_LOG (category, "Class info: ID: %d - Probability: %.2f - Label: %s", + class->class_id, class->class_prob, + (class->class_label != NULL) ? class->class_label : "No Label set"); +} + +void +gst_inference_print_classes (GstVideoInference * vi, + GstDebugCategory * category, GstInferenceMeta * inference_meta) +{ + Prediction *root = NULL; + + g_return_if_fail (vi != NULL); + g_return_if_fail (inference_meta != NULL); + + root = inference_meta->prediction; + g_list_foreach (root->classifications, gst_inference_print_class, + (gpointer) category); +} diff --git a/gst-libs/gst/r2inference/gstinferencedebug.h b/gst-libs/gst/r2inference/gstinferencedebug.h index 1f8d61a9..b34ace39 100644 --- a/gst-libs/gst/r2inference/gstinferencedebug.h +++ b/gst-libs/gst/r2inference/gstinferencedebug.h @@ -74,5 +74,16 @@ void gst_inference_print_boxes (GstVideoInference * vi, void gst_inference_print_predictions (GstVideoInference * vi, GstDebugCategory * category, GstInferenceMeta * inference_meta); +/** + * \brief Display the classes in the inference meta + * + * \param vi Father object of every architecture + * \param category The debug category + * \param inference_meta Inference Meta + */ + +void gst_inference_print_classes (GstVideoInference * vi, + GstDebugCategory * category, GstInferenceMeta * inference_meta); + G_END_DECLS #endif // GST_INFERENCE_DEBUG_H From 75569508253b95bbe7567244e083496d7b8b78cb Mon Sep 17 00:00:00 2001 From: Carlos Rodriguez Date: Fri, 20 Dec 2019 11:21:23 -0600 Subject: [PATCH 041/188] Support new meta approach in InceptionV2 --- ext/r2inference/gstinceptionv2.c | 39 ++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/ext/r2inference/gstinceptionv2.c b/ext/r2inference/gstinceptionv2.c index c9d99e2c..f6f50f5c 100644 --- a/ext/r2inference/gstinceptionv2.c +++ b/ext/r2inference/gstinceptionv2.c @@ -67,6 +67,10 @@ static gboolean gst_inceptionv2_preprocess (GstVideoInference * vi, static gboolean gst_inceptionv2_postprocess (GstVideoInference * vi, const gpointer prediction, gsize predsize, GstMeta * meta_model, GstVideoInfo * info_model, gboolean * valid_prediction); +static gboolean +gst_inceptionv2_postprocess_meta (GstVideoInference * vi, + const gpointer prediction, gsize predsize, GstMeta * meta_model, + GstVideoInfo * info_model, gboolean * valid_prediction); static gboolean gst_inceptionv2_start (GstVideoInference * vi); static gboolean gst_inceptionv2_stop (GstVideoInference * vi); @@ -140,6 +144,8 @@ gst_inceptionv2_class_init (GstInceptionv2Class * klass) vi_class->stop = GST_DEBUG_FUNCPTR (gst_inceptionv2_stop); vi_class->preprocess = GST_DEBUG_FUNCPTR (gst_inceptionv2_preprocess); vi_class->postprocess = GST_DEBUG_FUNCPTR (gst_inceptionv2_postprocess); + vi_class->postprocess_meta = + GST_DEBUG_FUNCPTR (gst_inceptionv2_postprocess_meta); vi_class->inference_meta_info = gst_classification_meta_get_info (); } @@ -228,6 +234,39 @@ gst_inceptionv2_postprocess (GstVideoInference * vi, const gpointer prediction, return TRUE; } +static gboolean +gst_inceptionv2_postprocess_meta (GstVideoInference * vi, + const gpointer prediction, gsize predsize, GstMeta * meta_model, + GstVideoInfo * info_model, gboolean * valid_prediction) +{ + GstInferenceMeta *imeta = NULL; + Classification *class = NULL; + Prediction *root = NULL; + + g_return_val_if_fail (vi != NULL, FALSE); + g_return_val_if_fail (meta_model != NULL, FALSE); + g_return_val_if_fail (info_model != NULL, FALSE); + + GST_LOG_OBJECT (vi, "Postprocess Meta"); + + imeta = (GstInferenceMeta *) meta_model; + + root = imeta->prediction; + if (!root) { + GST_ERROR_OBJECT (vi, "Prediction is not part of the Inference Meta"); + return FALSE; + } + + class = gst_create_class_from_prediction (vi, prediction, predsize); + root->classifications = + g_list_append (root->classifications, (gpointer) class); + + gst_inference_print_classes (vi, gst_inceptionv2_debug_category, imeta); + + *valid_prediction = TRUE; + return TRUE; +} + static gboolean gst_inceptionv2_start (GstVideoInference * vi) { From 81306e8c6820b177775aef7c35630b52532da974 Mon Sep 17 00:00:00 2001 From: Michael Gruner Date: Thu, 19 Dec 2019 16:55:08 -0600 Subject: [PATCH 042/188] Move prediction to a miniobject based standalone module --- ext/r2inference/gstinceptionv2.c | 2 +- ext/r2inference/gsttinyyolov2.c | 6 +- gst-libs/gst/r2inference/Makefile.am | 6 +- gst-libs/gst/r2inference/gstinferencedebug.c | 25 ++-- gst-libs/gst/r2inference/gstinferencemeta.c | 31 ++--- gst-libs/gst/r2inference/gstinferencemeta.h | 33 +----- .../gst/r2inference/gstinferencepostprocess.c | 17 ++- .../gst/r2inference/gstinferencepostprocess.h | 2 +- .../gst/r2inference/gstinferenceprediction.c | 112 ++++++++++++++++++ .../gst/r2inference/gstinferenceprediction.h | 68 +++++++++++ gst-libs/gst/r2inference/gstvideoinference.c | 2 +- tests/examples/detection/customlogic.cc | 3 +- tests/examples/detection/customlogic.h | 6 +- tests/examples/detection/gstdetection.c | 5 +- 14 files changed, 232 insertions(+), 86 deletions(-) create mode 100644 gst-libs/gst/r2inference/gstinferenceprediction.c create mode 100644 gst-libs/gst/r2inference/gstinferenceprediction.h diff --git a/ext/r2inference/gstinceptionv2.c b/ext/r2inference/gstinceptionv2.c index f6f50f5c..e7387fd0 100644 --- a/ext/r2inference/gstinceptionv2.c +++ b/ext/r2inference/gstinceptionv2.c @@ -241,7 +241,7 @@ gst_inceptionv2_postprocess_meta (GstVideoInference * vi, { GstInferenceMeta *imeta = NULL; Classification *class = NULL; - Prediction *root = NULL; + GstInferencePrediction *root = NULL; g_return_val_if_fail (vi != NULL, FALSE); g_return_val_if_fail (meta_model != NULL, FALSE); diff --git a/ext/r2inference/gsttinyyolov2.c b/ext/r2inference/gsttinyyolov2.c index 6fd2a9bc..b6a18d5f 100644 --- a/ext/r2inference/gsttinyyolov2.c +++ b/ext/r2inference/gsttinyyolov2.c @@ -333,12 +333,12 @@ gst_tinyyolov2_postprocess_meta (GstVideoInference * vi, GST_LOG_OBJECT (tinyyolov2, "Number of predictions: %d", num_boxes); for (i = 0; i < num_boxes; i++) { - Prediction *root = imeta->prediction; - Prediction *predict = gst_create_prediction_from_box (vi, &boxes[i]); + GstInferencePrediction *predict = + gst_create_prediction_from_box (vi, &boxes[i]); /* Create new node */ GNode *new_node = g_node_new (predict); /* Add new node as child of parent node */ - g_node_append (root->node, new_node); + g_node_append (imeta->node, new_node); } /* Free boxes after creation */ diff --git a/gst-libs/gst/r2inference/Makefile.am b/gst-libs/gst/r2inference/Makefile.am index e9512a66..6c6a8301 100644 --- a/gst-libs/gst/r2inference/Makefile.am +++ b/gst-libs/gst/r2inference/Makefile.am @@ -11,7 +11,8 @@ libgstinference_@GST_API_VERSION@_la_SOURCES= \ gsttensorflow.cc \ gstinferencepreprocess.c \ gstinferencepostprocess.c \ - gstinferencedebug.c + gstinferencedebug.c \ + gstinferenceprediction.c libgstinference_@GST_API_VERSION@_la_CXXFLAGS= \ $(GST_CXXFLAGS) \ @@ -46,4 +47,5 @@ gstinferenceinclude_HEADERS= \ gstinferencepreprocess.h \ gstinferencepostprocess.h \ gstinferencedebug.h \ - gstinferencemeta.h + gstinferencemeta.h \ + gstinferenceprediction.h diff --git a/gst-libs/gst/r2inference/gstinferencedebug.c b/gst-libs/gst/r2inference/gstinferencedebug.c index 3864bdb3..7d58154c 100644 --- a/gst-libs/gst/r2inference/gstinferencedebug.c +++ b/gst-libs/gst/r2inference/gstinferencedebug.c @@ -99,24 +99,19 @@ gst_inference_print_boxes (GstVideoInference * vi, GstDebugCategory * category, static gboolean gst_inference_print_prediction (GNode * node, gpointer data) { - Prediction *predict = (Prediction *) node->data; + GstInferencePrediction *predict = (GstInferencePrediction *) node->data; GstDebugCategory *category = (GstDebugCategory *) data; - BBox *box = NULL; + BoundingBox *box = NULL; g_return_val_if_fail (category != NULL, FALSE); g_return_val_if_fail (predict != NULL, FALSE); - box = predict->box; + box = &predict->bbox; - if (box != NULL) { - GST_CAT_LOG (category, - "Prediction ID:%d - Level:%d - Box: [class:%d, x:%f, y:%f, width:%f, height:%f, prob:%f]", - predict->id, g_node_depth (node), box->label, box->x, box->y, - box->width, box->height, box->prob); - } else { - GST_CAT_LOG (category, "Prediction ID:%d - Level:%d - No Box", - predict->id, g_node_depth (node)); - } + GST_CAT_LOG (category, + "GstInferencePrediction ID:%llu - Level: %d - Enabled:%s - Box: [x:%u, y:%u, width:%u, height:%u]", + predict->id, g_node_depth (node), predict->enabled ? "true" : "false", + box->x, box->y, box->width, box->height); return FALSE; } @@ -125,14 +120,14 @@ void gst_inference_print_predictions (GstVideoInference * vi, GstDebugCategory * category, GstInferenceMeta * inference_meta) { - Prediction *root = NULL; + GstInferencePrediction *root = NULL; g_return_if_fail (vi != NULL); g_return_if_fail (inference_meta != NULL); root = inference_meta->prediction; - g_node_traverse (root->node, G_LEVEL_ORDER, G_TRAVERSE_ALL, -1, + g_node_traverse (inference_meta->node, G_LEVEL_ORDER, G_TRAVERSE_ALL, -1, gst_inference_print_prediction, (gpointer) category); } @@ -154,7 +149,7 @@ void gst_inference_print_classes (GstVideoInference * vi, GstDebugCategory * category, GstInferenceMeta * inference_meta) { - Prediction *root = NULL; + GstInferencePrediction *root = NULL; g_return_if_fail (vi != NULL); g_return_if_fail (inference_meta != NULL); diff --git a/gst-libs/gst/r2inference/gstinferencemeta.c b/gst-libs/gst/r2inference/gstinferencemeta.c index f2d58134..012296ef 100644 --- a/gst-libs/gst/r2inference/gstinferencemeta.c +++ b/gst-libs/gst/r2inference/gstinferencemeta.c @@ -361,15 +361,10 @@ static gboolean gst_inference_meta_init (GstMeta * meta, gpointer params, GstBuffer * buffer) { GstInferenceMeta *imeta = (GstInferenceMeta *) meta; - Prediction *root; + GstInferencePrediction *root; /* Create root Prediction */ - root = g_malloc (sizeof (Prediction)); - root->id = rand (); - root->enabled = TRUE; - root->box = NULL; - root->classifications = NULL; - root->node = g_node_new (root); + root = gst_inference_prediction_new (); imeta->prediction = root; @@ -396,22 +391,18 @@ gst_inference_clean_classifications (gpointer data) static gboolean gst_inference_clean_nodes (GNode * node, gpointer data) { - Prediction *predict = (Prediction *) node->data; + GstInferencePrediction *prediction = (GstInferencePrediction *) node->data; - g_return_val_if_fail (predict != NULL, TRUE); + g_return_val_if_fail (prediction != NULL, TRUE); - /* Delete the Box in the Prediction */ - if (predict->box != NULL) { - g_free (predict->box); - predict->box = NULL; - } /* Delete classifications */ - if (predict->classifications != NULL) { - g_list_free_full (predict->classifications, + if (prediction->classifications != NULL) { + g_list_free_full (prediction->classifications, gst_inference_clean_classifications); } + /* Delete the prediction */ - g_free (predict); + gst_inference_prediction_unref (prediction); return FALSE; } @@ -420,7 +411,7 @@ static void gst_inference_meta_free (GstMeta * meta, GstBuffer * buffer) { GstInferenceMeta *imeta = NULL; - Prediction *root = NULL; + GstInferencePrediction *root = NULL; g_return_if_fail (meta != NULL); g_return_if_fail (buffer != NULL); @@ -428,7 +419,7 @@ gst_inference_meta_free (GstMeta * meta, GstBuffer * buffer) imeta = (GstInferenceMeta *) meta; root = imeta->prediction; - g_node_traverse (root->node, G_LEVEL_ORDER, G_TRAVERSE_ALL, -1, + g_node_traverse (imeta->node, G_LEVEL_ORDER, G_TRAVERSE_ALL, -1, gst_inference_clean_nodes, NULL); - g_node_destroy (root->node); + g_node_destroy (imeta->node); } diff --git a/gst-libs/gst/r2inference/gstinferencemeta.h b/gst-libs/gst/r2inference/gstinferencemeta.h index 8a6706e0..26a69d95 100644 --- a/gst-libs/gst/r2inference/gstinferencemeta.h +++ b/gst-libs/gst/r2inference/gstinferencemeta.h @@ -14,6 +14,8 @@ #include +#include + G_BEGIN_DECLS #define GST_EMBEDDING_META_API_TYPE (gst_embedding_meta_api_get_type()) #define GST_EMBEDDING_META_INFO (gst_embedding_meta_get_info()) @@ -23,6 +25,7 @@ G_BEGIN_DECLS #define GST_DETECTION_META_INFO (gst_detection_meta_get_info()) #define GST_INFERENCE_META_API_TYPE (gst_inference_meta_api_get_type()) #define GST_INFERENCE_META_INFO (gst_inference_meta_get_info()) + /** * Basic bounding box structure for detection */ @@ -37,32 +40,6 @@ struct _BBox gdouble height; }; -/** - * Basic class structure for inference - */ -typedef struct _Classification Classification; -struct _Classification -{ - gint class_id; - gdouble class_prob; - gchar *class_label; - gint num_classes; - gdouble *classes_probs; -}; - -/** - * Prediction structure for inference meta - */ -typedef struct _Prediction Prediction; -struct _Prediction -{ - gint id; - gboolean enabled; - BBox *box; - GList *classifications; - GNode *node; -}; - /** * Implements the placeholder for inference information. */ @@ -70,7 +47,9 @@ typedef struct _GstInferenceMeta GstInferenceMeta; struct _GstInferenceMeta { GstMeta meta; - Prediction *prediction; + + GstInferencePrediction *prediction; + GNode *node; }; /** diff --git a/gst-libs/gst/r2inference/gstinferencepostprocess.c b/gst-libs/gst/r2inference/gstinferencepostprocess.c index 5aac5203..66856ac5 100644 --- a/gst-libs/gst/r2inference/gstinferencepostprocess.c +++ b/gst-libs/gst/r2inference/gstinferencepostprocess.c @@ -248,22 +248,19 @@ gst_create_boxes (GstVideoInference * vi, const gpointer prediction, return TRUE; } -Prediction * +GstInferencePrediction * gst_create_prediction_from_box (GstVideoInference * vi, BBox * box) { - Prediction *predict = NULL; - BBox *new_box = NULL; + GstInferencePrediction *predict = NULL; g_return_val_if_fail (vi != NULL, NULL); g_return_val_if_fail (box != NULL, NULL); - predict = g_malloc (sizeof (Prediction)); - new_box = g_malloc (sizeof (BBox)); - memcpy (new_box, box, sizeof (BBox)); - predict->box = new_box; - predict->id = rand (); - predict->enabled = TRUE; - predict->classifications = NULL; + predict = gst_inference_prediction_new (); + predict->bbox.x = box->x; + predict->bbox.y = box->x; + predict->bbox.width = box->width; + predict->bbox.height = box->height; return predict; } diff --git a/gst-libs/gst/r2inference/gstinferencepostprocess.h b/gst-libs/gst/r2inference/gstinferencepostprocess.h index 3b810c5d..26b9920c 100644 --- a/gst-libs/gst/r2inference/gstinferencepostprocess.h +++ b/gst-libs/gst/r2inference/gstinferencepostprocess.h @@ -75,7 +75,7 @@ gboolean gst_create_boxes_float (GstVideoInference * vi, * \param vi Father object of every architecture * \param box Box used to fill Prediction */ -Prediction *gst_create_prediction_from_box (GstVideoInference * vi, BBox * box); +GstInferencePrediction *gst_create_prediction_from_box (GstVideoInference * vi, BBox * box); /** * \brief Create Classification from prediction data diff --git a/gst-libs/gst/r2inference/gstinferenceprediction.c b/gst-libs/gst/r2inference/gstinferenceprediction.c new file mode 100644 index 00000000..10f59ac2 --- /dev/null +++ b/gst-libs/gst/r2inference/gstinferenceprediction.c @@ -0,0 +1,112 @@ +/* Copyright (C) 2019 RidgeRun, LLC (http://www.ridgerun.com) + * All Rights Reserved. + * + * The contents of this software are proprietary and confidential to RidgeRun, + * LLC. No part of this program may be photocopied, reproduced or translated + * into another programming language without prior written consent of + * RidgeRun, LLC. The user is free to modify the source code after obtaining + * a software license from RidgeRun. All source code changes must be provided + * back to RidgeRun without any encumbrance. +*/ + +#include "gstinferenceprediction.h" + +static GType gst_inference_prediction_get_type (void); +GST_DEFINE_MINI_OBJECT_TYPE (GstInferencePrediction, gst_inference_prediction); + +static void bounding_box_reset (BoundingBox * bbox); +static GstMiniObject *_gst_inference_prediction_copy (const GstMiniObject * + obj); + +static void classification_reset (GList * classification); + +GstInferencePrediction * +gst_inference_prediction_new (void) +{ + GstInferencePrediction *self = g_slice_new (GstInferencePrediction); + + gst_mini_object_init (GST_MINI_OBJECT_CAST (self), 0, + gst_inference_prediction_get_type (), _gst_inference_prediction_copy, + NULL, NULL); + + gst_inference_prediction_reset (self); + + return self; +} + +void +gst_inference_prediction_reset (GstInferencePrediction * self) +{ + g_return_if_fail (self); + + self->id = 0; + self->enabled = FALSE; + + bounding_box_reset (&self->bbox); + classification_reset (self->classifications); +} + +static void +bounding_box_reset (BoundingBox * bbox) +{ + bbox->x = 0; + bbox->y = 0; + bbox->width = 0; + bbox->height = 0; +} + +static void +classification_reset (GList * classifications) +{ + GList * iter = NULL; + + for (iter = classifications; iter != NULL; iter = g_list_next (iter)) { + Classification *c = (Classification *)iter->data; + c->class_id = 0; + c->class_prob = 0.0f; + c->class_label = NULL; + c->num_classes = 0; + c->classes_probs = NULL; + } +} + +GstInferencePrediction * +gst_inference_prediction_copy (const GstInferencePrediction * self) +{ + GstInferencePrediction *other = NULL; + + g_return_val_if_fail (self, NULL); + + other = gst_inference_prediction_new (); + + other->id = self->id; + other->enabled = self->enabled; + other->bbox = self->bbox; + + return other; +} + +static GstMiniObject * +_gst_inference_prediction_copy (const GstMiniObject * obj) +{ + return + GST_MINI_OBJECT_CAST (gst_inference_prediction_copy ( + (GstInferencePrediction *) obj)); +} + +GstInferencePrediction * +gst_inference_prediction_ref (GstInferencePrediction * self) +{ + g_return_val_if_fail (self, NULL); + + return (GstInferencePrediction *) + gst_mini_object_ref (GST_MINI_OBJECT_CAST (self)); +} + +void +gst_inference_prediction_unref (GstInferencePrediction * self) +{ + g_return_if_fail (self); + + gst_mini_object_unref (GST_MINI_OBJECT_CAST (self)); +} diff --git a/gst-libs/gst/r2inference/gstinferenceprediction.h b/gst-libs/gst/r2inference/gstinferenceprediction.h new file mode 100644 index 00000000..ba3bc14e --- /dev/null +++ b/gst-libs/gst/r2inference/gstinferenceprediction.h @@ -0,0 +1,68 @@ +/* Copyright (C) 2019 RidgeRun, LLC (http://www.ridgerun.com) + * All Rights Reserved. + * + * The contents of this software are proprietary and confidential to RidgeRun, + * LLC. No part of this program may be photocopied, reproduced or translated + * into another programming language without prior written consent of + * RidgeRun, LLC. The user is free to modify the source code after obtaining + * a software license from RidgeRun. All source code changes must be provided + * back to RidgeRun without any encumbrance. +*/ + +#ifndef __GST_INFERENCE_PREDICTION__ +#define __GST_INFERENCE_PREDICTION__ + +#include + +G_BEGIN_DECLS + +typedef struct _BoundingBox BoundingBox; +typedef struct _GstInferencePrediction GstInferencePrediction; + +/** + * Classification to be assigned to an object + */ +typedef struct _Classification Classification; +struct _Classification +{ + gint class_id; + gdouble class_prob; + gchar *class_label; + gint num_classes; + gdouble *classes_probs; +}; + +/** + * Size and coordinates of an image region + */ +struct _BoundingBox +{ + guint x; + guint y; + guint width; + guint height; +}; + +/** + * Abstraction that represents a prediction + */ +struct _GstInferencePrediction +{ + GstMiniObject base; + + guint64 id; + gboolean enabled; + BoundingBox bbox; + GList * classifications; +}; + +GstInferencePrediction * gst_inference_prediction_new (void); +void gst_inference_prediction_reset (GstInferencePrediction * self); +GstInferencePrediction * gst_inference_prediction_copy (const GstInferencePrediction * self); +void gst_inference_prediction_free (GstInferencePrediction * self); +GstInferencePrediction * gst_inference_prediction_ref (GstInferencePrediction * self); +void gst_inference_prediction_unref (GstInferencePrediction * self); + +G_END_DECLS + +#endif // __GST_INFERENCE_PREDICTION__ diff --git a/gst-libs/gst/r2inference/gstvideoinference.c b/gst-libs/gst/r2inference/gstvideoinference.c index cebd22a4..d5c28d73 100644 --- a/gst-libs/gst/r2inference/gstvideoinference.c +++ b/gst-libs/gst/r2inference/gstvideoinference.c @@ -1037,7 +1037,7 @@ gst_video_inference_collected (GstCollectPads * pads, gpointer user_data) if (meta_model) { /* Check if root is enabled to be processed, if not, just forward buffer */ GstInferenceMeta *inference_meta = (GstInferenceMeta *) meta_model; - Prediction *root = inference_meta->prediction; + GstInferencePrediction *root = inference_meta->prediction; if (!root->enabled) { GST_INFO_OBJECT (self, "Current Prediction is not enabled, bypassing processing..."); diff --git a/tests/examples/detection/customlogic.cc b/tests/examples/detection/customlogic.cc index ce457e63..bf1b41d6 100644 --- a/tests/examples/detection/customlogic.cc +++ b/tests/examples/detection/customlogic.cc @@ -16,7 +16,8 @@ void handle_prediction (unsigned char *image, - int width, int height, unsigned int size, BoundingBox * boxes, int num_boxes) + int width, int height, unsigned int size, PredictionBox * boxes, + int num_boxes) { /* FILLME: * Put here your custom logic, you may use C++ here. diff --git a/tests/examples/detection/customlogic.h b/tests/examples/detection/customlogic.h index a28bb42a..d7af047c 100644 --- a/tests/examples/detection/customlogic.h +++ b/tests/examples/detection/customlogic.h @@ -17,8 +17,8 @@ extern "C" { #endif -typedef struct _BoundingBox BoundingBox; -struct _BoundingBox { +typedef struct _PredictionBox PredictionBox; +struct _PredictionBox { int category; double probability; int x; @@ -32,7 +32,7 @@ handle_prediction (unsigned char *image, int width, int height, unsigned int size, - BoundingBox *boxes, + PredictionBox *boxes, int num_boxes); #ifdef __cplusplus diff --git a/tests/examples/detection/gstdetection.c b/tests/examples/detection/gstdetection.c index d0c3b5b4..cf24155b 100644 --- a/tests/examples/detection/gstdetection.c +++ b/tests/examples/detection/gstdetection.c @@ -172,7 +172,7 @@ gst_detection_process_inference (GstElement * element, gpointer user_data) { gint index; - BoundingBox *boxes; + PredictionBox *boxes; g_return_if_fail (element); g_return_if_fail (model_meta); @@ -182,7 +182,8 @@ gst_detection_process_inference (GstElement * element, g_return_if_fail (user_data); boxes = - (BoundingBox *) g_malloc (bypass_meta->num_boxes * sizeof (BoundingBox)); + (PredictionBox *) g_malloc (bypass_meta->num_boxes * + sizeof (PredictionBox)); for (index = 0; index < bypass_meta->num_boxes; index++) { boxes[index].category = bypass_meta->boxes[index].label; From aceef832a0eb7f3a136baaa571c56cf07676acdd Mon Sep 17 00:00:00 2001 From: Michael Gruner Date: Thu, 19 Dec 2019 17:33:28 -0600 Subject: [PATCH 043/188] Have the prediction serialize itself to ease printing of a prediction --- gst-libs/gst/r2inference/gstinferencedebug.c | 11 +++--- .../gst/r2inference/gstinferenceprediction.c | 39 +++++++++++++++++++ .../gst/r2inference/gstinferenceprediction.h | 1 + 3 files changed, 45 insertions(+), 6 deletions(-) diff --git a/gst-libs/gst/r2inference/gstinferencedebug.c b/gst-libs/gst/r2inference/gstinferencedebug.c index 7d58154c..5a0f20d1 100644 --- a/gst-libs/gst/r2inference/gstinferencedebug.c +++ b/gst-libs/gst/r2inference/gstinferencedebug.c @@ -101,17 +101,16 @@ gst_inference_print_prediction (GNode * node, gpointer data) { GstInferencePrediction *predict = (GstInferencePrediction *) node->data; GstDebugCategory *category = (GstDebugCategory *) data; - BoundingBox *box = NULL; + gchar *spred = NULL; g_return_val_if_fail (category != NULL, FALSE); g_return_val_if_fail (predict != NULL, FALSE); - box = &predict->bbox; + spred = gst_inference_prediction_to_string (predict); - GST_CAT_LOG (category, - "GstInferencePrediction ID:%llu - Level: %d - Enabled:%s - Box: [x:%u, y:%u, width:%u, height:%u]", - predict->id, g_node_depth (node), predict->enabled ? "true" : "false", - box->x, box->y, box->width, box->height); + GST_CAT_LOG (category, "\n%s", spred); + + g_free (spred); return FALSE; } diff --git a/gst-libs/gst/r2inference/gstinferenceprediction.c b/gst-libs/gst/r2inference/gstinferenceprediction.c index 10f59ac2..669403eb 100644 --- a/gst-libs/gst/r2inference/gstinferenceprediction.c +++ b/gst-libs/gst/r2inference/gstinferenceprediction.c @@ -15,6 +15,8 @@ static GType gst_inference_prediction_get_type (void); GST_DEFINE_MINI_OBJECT_TYPE (GstInferencePrediction, gst_inference_prediction); static void bounding_box_reset (BoundingBox * bbox); +static gchar *bounding_box_to_string_with_level (BoundingBox * bbox, + gint level); static GstMiniObject *_gst_inference_prediction_copy (const GstMiniObject * obj); @@ -110,3 +112,40 @@ gst_inference_prediction_unref (GstInferencePrediction * self) gst_mini_object_unref (GST_MINI_OBJECT_CAST (self)); } + +gchar * +gst_inference_prediction_to_string (GstInferencePrediction * self) +{ + gchar *bbox = NULL; + gchar *prediction = NULL; + + g_return_val_if_fail (self, NULL); + + bbox = bounding_box_to_string_with_level (&self->bbox, 1); + prediction = g_strdup_printf ("{\n" + " id : %llu\n" + " enabled : %s\n" + " bbox : %s\n" "}", self->id, self->enabled ? "True" : "False", bbox); + + g_free (bbox); + + return prediction; +} + +static gchar * +bounding_box_to_string_with_level (BoundingBox * bbox, gint level) +{ + gint indent = level * 2; + + g_return_val_if_fail (bbox, NULL); + + return g_strdup_printf ("{\n" + "%*s x : %u\n" + "%*s y : %u\n" + "%*s width : %u\n" + "%*s height : %u\n" + "%*s}", + indent, "", bbox->x, + indent, "", bbox->y, + indent, "", bbox->width, indent, "", bbox->height, indent, ""); +} diff --git a/gst-libs/gst/r2inference/gstinferenceprediction.h b/gst-libs/gst/r2inference/gstinferenceprediction.h index ba3bc14e..f5eb8ee9 100644 --- a/gst-libs/gst/r2inference/gstinferenceprediction.h +++ b/gst-libs/gst/r2inference/gstinferenceprediction.h @@ -62,6 +62,7 @@ GstInferencePrediction * gst_inference_prediction_copy (const GstInferencePredic void gst_inference_prediction_free (GstInferencePrediction * self); GstInferencePrediction * gst_inference_prediction_ref (GstInferencePrediction * self); void gst_inference_prediction_unref (GstInferencePrediction * self); +gchar * gst_inference_prediction_to_string (GstInferencePrediction * self); G_END_DECLS From d1208fc7ecee77b80335a8aba05061ac3ba732e7 Mon Sep 17 00:00:00 2001 From: Michael Gruner Date: Fri, 20 Dec 2019 17:05:38 -0600 Subject: [PATCH 044/188] Extend prediction to have subpredictions --- .../gst/r2inference/gstinferenceprediction.c | 203 +++++++++++++----- .../gst/r2inference/gstinferenceprediction.h | 1 + 2 files changed, 145 insertions(+), 59 deletions(-) diff --git a/gst-libs/gst/r2inference/gstinferenceprediction.c b/gst-libs/gst/r2inference/gstinferenceprediction.c index 669403eb..964ca951 100644 --- a/gst-libs/gst/r2inference/gstinferenceprediction.c +++ b/gst-libs/gst/r2inference/gstinferenceprediction.c @@ -14,47 +14,49 @@ static GType gst_inference_prediction_get_type (void); GST_DEFINE_MINI_OBJECT_TYPE (GstInferencePrediction, gst_inference_prediction); +static void prediction_free (GstInferencePrediction * obj); +static GSList *prediction_get_children (GstInferencePrediction * self); +static void prediction_reset (GstInferencePrediction * self); +static gchar *prediction_to_string (GstInferencePrediction * self, gint level); + static void bounding_box_reset (BoundingBox * bbox); -static gchar *bounding_box_to_string_with_level (BoundingBox * bbox, - gint level); -static GstMiniObject *_gst_inference_prediction_copy (const GstMiniObject * - obj); +static gchar *bounding_box_to_string (BoundingBox * bbox, gint level); static void classification_reset (GList * classification); +static void node_get_children (GNode * node, gpointer data); + GstInferencePrediction * gst_inference_prediction_new (void) { GstInferencePrediction *self = g_slice_new (GstInferencePrediction); gst_mini_object_init (GST_MINI_OBJECT_CAST (self), 0, - gst_inference_prediction_get_type (), _gst_inference_prediction_copy, - NULL, NULL); - - gst_inference_prediction_reset (self); + gst_inference_prediction_get_type (), + (GstMiniObjectCopyFunction) gst_inference_prediction_copy, NULL, + (GstMiniObjectFreeFunction) prediction_free); + + self->predictions = NULL; + prediction_reset (self); return self; } -void -gst_inference_prediction_reset (GstInferencePrediction * self) +GstInferencePrediction * +gst_inference_prediction_ref (GstInferencePrediction * self) { - g_return_if_fail (self); - - self->id = 0; - self->enabled = FALSE; + g_return_val_if_fail (self, NULL); - bounding_box_reset (&self->bbox); - classification_reset (self->classifications); + return (GstInferencePrediction *) + gst_mini_object_ref (GST_MINI_OBJECT_CAST (self)); } -static void -bounding_box_reset (BoundingBox * bbox) +void +gst_inference_prediction_unref (GstInferencePrediction * self) { - bbox->x = 0; - bbox->y = 0; - bbox->width = 0; - bbox->height = 0; + g_return_if_fail (self); + + gst_mini_object_unref (GST_MINI_OBJECT_CAST (self)); } static void @@ -88,64 +90,147 @@ gst_inference_prediction_copy (const GstInferencePrediction * self) return other; } -static GstMiniObject * -_gst_inference_prediction_copy (const GstMiniObject * obj) +static gchar * +bounding_box_to_string (BoundingBox * bbox, gint level) { - return - GST_MINI_OBJECT_CAST (gst_inference_prediction_copy ( - (GstInferencePrediction *) obj)); + gint indent = level * 2; + + g_return_val_if_fail (bbox, NULL); + + return g_strdup_printf ("{\n" + "%*s x : %u\n" + "%*s y : %u\n" + "%*s width : %u\n" + "%*s height : %u\n" + "%*s}", + indent, "", bbox->x, + indent, "", bbox->y, + indent, "", bbox->width, indent, "", bbox->height, indent, ""); } -GstInferencePrediction * -gst_inference_prediction_ref (GstInferencePrediction * self) +static gchar * +prediction_to_string (GstInferencePrediction * self, gint level) { + GSList *subpreds = NULL; + GSList *iter = NULL; + GString *string = NULL; + gint indent = level * 2; + gchar *bbox = NULL; + gchar *children = NULL; + gchar *prediction = NULL; + g_return_val_if_fail (self, NULL); - return (GstInferencePrediction *) - gst_mini_object_ref (GST_MINI_OBJECT_CAST (self)); + string = g_string_new (NULL); + + subpreds = prediction_get_children (self); + for (iter = subpreds; iter != NULL; iter = g_slist_next (iter)) { + GstInferencePrediction *pred = (GstInferencePrediction *) iter->data; + gchar *child = prediction_to_string (pred, level + 2); + + g_string_append_printf (string, "%s, ", child); + g_free (child); + } + + children = g_string_free (string, FALSE); + bbox = bounding_box_to_string (&self->bbox, level + 1); + + prediction = g_strdup_printf ("{\n" + "%*s id : %llu,\n" + "%*s enabled : %s,\n" + "%*s bbox : %s,\n" + "%*s predictions : [\n" + "%*s %s\n" + "%*s ]\n" + "%*s}", + indent, "", self->id, + indent, "", self->enabled ? "True" : "False", + indent, "", bbox, + indent, "", indent, "", children, indent, "", indent, ""); + + g_free (bbox); + g_free (children); + + return prediction; } -void -gst_inference_prediction_unref (GstInferencePrediction * self) +gchar * +gst_inference_prediction_to_string (GstInferencePrediction * self) +{ + return prediction_to_string (self, 0); +} + +static void +bounding_box_reset (BoundingBox * bbox) +{ + g_return_if_fail (bbox); + + bbox->x = 0; + bbox->y = 0; + bbox->width = 0; + bbox->height = 0; +} + +static void +prediction_reset (GstInferencePrediction * self) { g_return_if_fail (self); - gst_mini_object_unref (GST_MINI_OBJECT_CAST (self)); + self->id = 0; + self->enabled = FALSE; + + bounding_box_reset (&self->bbox); + classification_reset (self->classifications); + + /* Free al children */ + prediction_free (self); + + self->predictions = g_node_new (self); } -gchar * -gst_inference_prediction_to_string (GstInferencePrediction * self) +static void +node_get_children (GNode * node, gpointer data) { - gchar *bbox = NULL; - gchar *prediction = NULL; + GSList **children = (GSList **) data; + GstInferencePrediction *prediction; - g_return_val_if_fail (self, NULL); + g_return_if_fail (node); + g_return_if_fail (children); - bbox = bounding_box_to_string_with_level (&self->bbox, 1); - prediction = g_strdup_printf ("{\n" - " id : %llu\n" - " enabled : %s\n" - " bbox : %s\n" "}", self->id, self->enabled ? "True" : "False", bbox); + prediction = (GstInferencePrediction *) node->data; - g_free (bbox); + *children = g_slist_append (*children, prediction); +} - return prediction; +static GSList * +prediction_get_children (GstInferencePrediction * self) +{ + GSList *children = NULL; + + g_return_val_if_fail (self, NULL); + + if (self->predictions) { + g_node_children_foreach (self->predictions, G_TRAVERSE_ALL, + node_get_children, &children); + } + + return children; } -static gchar * -bounding_box_to_string_with_level (BoundingBox * bbox, gint level) +static void +prediction_free (GstInferencePrediction * self) { - gint indent = level * 2; + GSList *children = prediction_get_children (self); + GSList *iter = NULL; - g_return_val_if_fail (bbox, NULL); + for (iter = children; iter != NULL; iter = g_slist_next (iter)) { + GstInferencePrediction *child = (GstInferencePrediction *) iter->data; - return g_strdup_printf ("{\n" - "%*s x : %u\n" - "%*s y : %u\n" - "%*s width : %u\n" - "%*s height : %u\n" - "%*s}", - indent, "", bbox->x, - indent, "", bbox->y, - indent, "", bbox->width, indent, "", bbox->height, indent, ""); + gst_inference_prediction_unref (child); + } + + if (self->predictions) { + g_node_destroy (self->predictions); + self->predictions = NULL; + } } diff --git a/gst-libs/gst/r2inference/gstinferenceprediction.h b/gst-libs/gst/r2inference/gstinferenceprediction.h index f5eb8ee9..a6641c80 100644 --- a/gst-libs/gst/r2inference/gstinferenceprediction.h +++ b/gst-libs/gst/r2inference/gstinferenceprediction.h @@ -54,6 +54,7 @@ struct _GstInferencePrediction gboolean enabled; BoundingBox bbox; GList * classifications; + GNode * predictions; }; GstInferencePrediction * gst_inference_prediction_new (void); From 54f8553acaf085544ac1425515aec623a06b7762 Mon Sep 17 00:00:00 2001 From: Michael Gruner Date: Fri, 20 Dec 2019 18:04:17 -0600 Subject: [PATCH 045/188] Implement prediction deep copy --- .../gst/r2inference/gstinferenceprediction.c | 48 +++++++++++++++++-- 1 file changed, 43 insertions(+), 5 deletions(-) diff --git a/gst-libs/gst/r2inference/gstinferenceprediction.c b/gst-libs/gst/r2inference/gstinferenceprediction.c index 964ca951..27f8f209 100644 --- a/gst-libs/gst/r2inference/gstinferenceprediction.c +++ b/gst-libs/gst/r2inference/gstinferenceprediction.c @@ -14,6 +14,8 @@ static GType gst_inference_prediction_get_type (void); GST_DEFINE_MINI_OBJECT_TYPE (GstInferencePrediction, gst_inference_prediction); +static GstInferencePrediction *prediction_copy (const GstInferencePrediction * + self); static void prediction_free (GstInferencePrediction * obj); static GSList *prediction_get_children (GstInferencePrediction * self); static void prediction_reset (GstInferencePrediction * self); @@ -25,6 +27,8 @@ static gchar *bounding_box_to_string (BoundingBox * bbox, gint level); static void classification_reset (GList * classification); static void node_get_children (GNode * node, gpointer data); +static gpointer node_copy (gconstpointer node, gpointer data); +static gboolean node_assign (GNode * node, gpointer data); GstInferencePrediction * gst_inference_prediction_new (void) @@ -35,7 +39,7 @@ gst_inference_prediction_new (void) gst_inference_prediction_get_type (), (GstMiniObjectCopyFunction) gst_inference_prediction_copy, NULL, (GstMiniObjectFreeFunction) prediction_free); - + self->predictions = NULL; prediction_reset (self); @@ -62,10 +66,10 @@ gst_inference_prediction_unref (GstInferencePrediction * self) static void classification_reset (GList * classifications) { - GList * iter = NULL; + GList *iter = NULL; for (iter = classifications; iter != NULL; iter = g_list_next (iter)) { - Classification *c = (Classification *)iter->data; + Classification *c = (Classification *) iter->data; c->class_id = 0; c->class_prob = 0.0f; c->class_label = NULL; @@ -74,8 +78,8 @@ classification_reset (GList * classifications) } } -GstInferencePrediction * -gst_inference_prediction_copy (const GstInferencePrediction * self) +static GstInferencePrediction * +prediction_copy (const GstInferencePrediction * self) { GstInferencePrediction *other = NULL; @@ -90,6 +94,40 @@ gst_inference_prediction_copy (const GstInferencePrediction * self) return other; } +static gpointer +node_copy (gconstpointer node, gpointer data) +{ + GstInferencePrediction *self = (GstInferencePrediction *) node; + + return prediction_copy (self); +} + +static gboolean +node_assign (GNode * node, gpointer data) +{ + GstInferencePrediction *pred = (GstInferencePrediction *) node->data; + + pred->predictions = node; + + return FALSE; +} + +GstInferencePrediction * +gst_inference_prediction_copy (const GstInferencePrediction * self) +{ + GNode *other = NULL; + + g_return_val_if_fail (self, NULL); + + /* Copy the binary tree */ + other = g_node_copy_deep (self->predictions, node_copy, NULL); + + /* Now finish assigning the nodes to the predictions */ + g_node_traverse (other, G_IN_ORDER, G_TRAVERSE_ALL, -1, node_assign, NULL); + + return (GstInferencePrediction *) other->data; +} + static gchar * bounding_box_to_string (BoundingBox * bbox, gint level) { From 8873934d8489f18adbbfcf996802ca3b021b5d92 Mon Sep 17 00:00:00 2001 From: Michael Gruner Date: Fri, 20 Dec 2019 18:18:52 -0600 Subject: [PATCH 046/188] Remove external tree leftovers --- ext/r2inference/gsttinyyolov2.c | 13 +++--- gst-libs/gst/r2inference/gstinferencedebug.c | 33 ++++----------- gst-libs/gst/r2inference/gstinferencemeta.c | 44 +------------------- gst-libs/gst/r2inference/gstinferencemeta.h | 1 - 4 files changed, 16 insertions(+), 75 deletions(-) diff --git a/ext/r2inference/gsttinyyolov2.c b/ext/r2inference/gsttinyyolov2.c index b6a18d5f..63b89665 100644 --- a/ext/r2inference/gsttinyyolov2.c +++ b/ext/r2inference/gsttinyyolov2.c @@ -332,13 +332,16 @@ gst_tinyyolov2_postprocess_meta (GstVideoInference * vi, GST_LOG_OBJECT (tinyyolov2, "Number of predictions: %d", num_boxes); + if (NULL == imeta->prediction) { + imeta->prediction = gst_inference_prediction_new (); + imeta->prediction->bbox.width = info_model->width; + imeta->prediction->bbox.height = info_model->height; + } + for (i = 0; i < num_boxes; i++) { - GstInferencePrediction *predict = + GstInferencePrediction *pred = gst_create_prediction_from_box (vi, &boxes[i]); - /* Create new node */ - GNode *new_node = g_node_new (predict); - /* Add new node as child of parent node */ - g_node_append (imeta->node, new_node); + g_node_append (imeta->prediction->predictions, pred->predictions); } /* Free boxes after creation */ diff --git a/gst-libs/gst/r2inference/gstinferencedebug.c b/gst-libs/gst/r2inference/gstinferencedebug.c index 5a0f20d1..cd25ea65 100644 --- a/gst-libs/gst/r2inference/gstinferencedebug.c +++ b/gst-libs/gst/r2inference/gstinferencedebug.c @@ -20,9 +20,6 @@ */ #include "gstinferencedebug.h" -static gboolean gst_inference_print_prediction (GNode * node, gpointer data); -static void gst_inference_print_class (gpointer data, gpointer user_data); - void gst_inference_print_embedding (GstVideoInference * vi, GstDebugCategory * category, GstClassificationMeta * class_meta, @@ -96,38 +93,22 @@ gst_inference_print_boxes (GstVideoInference * vi, GstDebugCategory * category, } } -static gboolean -gst_inference_print_prediction (GNode * node, gpointer data) -{ - GstInferencePrediction *predict = (GstInferencePrediction *) node->data; - GstDebugCategory *category = (GstDebugCategory *) data; - gchar *spred = NULL; - - g_return_val_if_fail (category != NULL, FALSE); - g_return_val_if_fail (predict != NULL, FALSE); - - spred = gst_inference_prediction_to_string (predict); - - GST_CAT_LOG (category, "\n%s", spred); - - g_free (spred); - - return FALSE; -} - void gst_inference_print_predictions (GstVideoInference * vi, GstDebugCategory * category, GstInferenceMeta * inference_meta) { - GstInferencePrediction *root = NULL; + GstInferencePrediction *pred = NULL; + gchar *spred = NULL; g_return_if_fail (vi != NULL); + g_return_if_fail (category != NULL); g_return_if_fail (inference_meta != NULL); - root = inference_meta->prediction; + pred = inference_meta->prediction; - g_node_traverse (inference_meta->node, G_LEVEL_ORDER, G_TRAVERSE_ALL, -1, - gst_inference_print_prediction, (gpointer) category); + spred = gst_inference_prediction_to_string (pred); + GST_CAT_LOG (category, "\n%s", spred); + g_free (spred); } static void diff --git a/gst-libs/gst/r2inference/gstinferencemeta.c b/gst-libs/gst/r2inference/gstinferencemeta.c index 012296ef..d5a8b79a 100644 --- a/gst-libs/gst/r2inference/gstinferencemeta.c +++ b/gst-libs/gst/r2inference/gstinferencemeta.c @@ -17,7 +17,6 @@ static gboolean gst_inference_meta_init (GstMeta * meta, gpointer params, GstBuffer * buffer); static void gst_inference_meta_free (GstMeta * meta, GstBuffer * buffer); -static gboolean gst_inference_clean_nodes (GNode * node, gpointer data); static gboolean gst_classification_meta_init (GstMeta * meta, gpointer params, GstBuffer * buffer); static void gst_classification_meta_free (GstMeta * meta, GstBuffer * buffer); @@ -371,55 +370,14 @@ gst_inference_meta_init (GstMeta * meta, gpointer params, GstBuffer * buffer) return TRUE; } -static void -gst_inference_clean_classifications (gpointer data) -{ - Classification *class = (Classification *) data; - - g_return_if_fail (class != NULL); - - /* Delete class */ - if (class->classes_probs) - g_free (class->classes_probs); - - if (class->class_label) - g_free (class->class_label); - - g_free (class); -} - -static gboolean -gst_inference_clean_nodes (GNode * node, gpointer data) -{ - GstInferencePrediction *prediction = (GstInferencePrediction *) node->data; - - g_return_val_if_fail (prediction != NULL, TRUE); - - /* Delete classifications */ - if (prediction->classifications != NULL) { - g_list_free_full (prediction->classifications, - gst_inference_clean_classifications); - } - - /* Delete the prediction */ - gst_inference_prediction_unref (prediction); - - return FALSE; -} - static void gst_inference_meta_free (GstMeta * meta, GstBuffer * buffer) { GstInferenceMeta *imeta = NULL; - GstInferencePrediction *root = NULL; g_return_if_fail (meta != NULL); g_return_if_fail (buffer != NULL); imeta = (GstInferenceMeta *) meta; - root = imeta->prediction; - - g_node_traverse (imeta->node, G_LEVEL_ORDER, G_TRAVERSE_ALL, -1, - gst_inference_clean_nodes, NULL); - g_node_destroy (imeta->node); + gst_inference_prediction_unref (imeta->prediction); } diff --git a/gst-libs/gst/r2inference/gstinferencemeta.h b/gst-libs/gst/r2inference/gstinferencemeta.h index 26a69d95..415ed180 100644 --- a/gst-libs/gst/r2inference/gstinferencemeta.h +++ b/gst-libs/gst/r2inference/gstinferencemeta.h @@ -49,7 +49,6 @@ struct _GstInferenceMeta GstMeta meta; GstInferencePrediction *prediction; - GNode *node; }; /** From 2142008013a7f883ad1d565daba8ec446d5fa209 Mon Sep 17 00:00:00 2001 From: Michael Gruner Date: Fri, 20 Dec 2019 18:29:03 -0600 Subject: [PATCH 047/188] Expose prediction methods to list and append new children --- ext/r2inference/gsttinyyolov2.c | 2 +- .../gst/r2inference/gstinferenceprediction.c | 95 ++++++++++--------- .../gst/r2inference/gstinferenceprediction.h | 2 + 3 files changed, 55 insertions(+), 44 deletions(-) diff --git a/ext/r2inference/gsttinyyolov2.c b/ext/r2inference/gsttinyyolov2.c index 63b89665..13db3b5d 100644 --- a/ext/r2inference/gsttinyyolov2.c +++ b/ext/r2inference/gsttinyyolov2.c @@ -341,7 +341,7 @@ gst_tinyyolov2_postprocess_meta (GstVideoInference * vi, for (i = 0; i < num_boxes; i++) { GstInferencePrediction *pred = gst_create_prediction_from_box (vi, &boxes[i]); - g_node_append (imeta->prediction->predictions, pred->predictions); + gst_inference_prediction_append (imeta->prediction, pred); } /* Free boxes after creation */ diff --git a/gst-libs/gst/r2inference/gstinferenceprediction.c b/gst-libs/gst/r2inference/gstinferenceprediction.c index 27f8f209..73290038 100644 --- a/gst-libs/gst/r2inference/gstinferenceprediction.c +++ b/gst-libs/gst/r2inference/gstinferenceprediction.c @@ -17,7 +17,6 @@ GST_DEFINE_MINI_OBJECT_TYPE (GstInferencePrediction, gst_inference_prediction); static GstInferencePrediction *prediction_copy (const GstInferencePrediction * self); static void prediction_free (GstInferencePrediction * obj); -static GSList *prediction_get_children (GstInferencePrediction * self); static void prediction_reset (GstInferencePrediction * self); static gchar *prediction_to_string (GstInferencePrediction * self, gint level); @@ -63,19 +62,14 @@ gst_inference_prediction_unref (GstInferencePrediction * self) gst_mini_object_unref (GST_MINI_OBJECT_CAST (self)); } -static void -classification_reset (GList * classifications) +void +gst_inference_prediction_append (GstInferencePrediction * self, + GstInferencePrediction * child) { - GList *iter = NULL; + g_return_if_fail (self); + g_return_if_fail (child); - for (iter = classifications; iter != NULL; iter = g_list_next (iter)) { - Classification *c = (Classification *) iter->data; - c->class_id = 0; - c->class_prob = 0.0f; - c->class_label = NULL; - c->num_classes = 0; - c->classes_probs = NULL; - } + g_node_append (self->predictions, child->predictions); } static GstInferencePrediction * @@ -161,7 +155,7 @@ prediction_to_string (GstInferencePrediction * self, gint level) string = g_string_new (NULL); - subpreds = prediction_get_children (self); + subpreds = gst_inference_prediction_get_children (self); for (iter = subpreds; iter != NULL; iter = g_slist_next (iter)) { GstInferencePrediction *pred = (GstInferencePrediction *) iter->data; gchar *child = prediction_to_string (pred, level + 2); @@ -198,6 +192,50 @@ gst_inference_prediction_to_string (GstInferencePrediction * self) return prediction_to_string (self, 0); } +static void +node_get_children (GNode * node, gpointer data) +{ + GSList **children = (GSList **) data; + GstInferencePrediction *prediction; + + g_return_if_fail (node); + g_return_if_fail (children); + + prediction = (GstInferencePrediction *) node->data; + + *children = g_slist_append (*children, prediction); +} + +GSList * +gst_inference_prediction_get_children (GstInferencePrediction * self) +{ + GSList *children = NULL; + + g_return_val_if_fail (self, NULL); + + if (self->predictions) { + g_node_children_foreach (self->predictions, G_TRAVERSE_ALL, + node_get_children, &children); + } + + return children; +} + +static void +classification_reset (GList * classifications) +{ + GList * iter = NULL; + + for (iter = classifications; iter != NULL; iter = g_list_next (iter)) { + Classification *c = (Classification *)iter->data; + c->class_id = 0; + c->class_prob = 0.0f; + c->class_label = NULL; + c->num_classes = 0; + c->classes_probs = NULL; + } +} + static void bounding_box_reset (BoundingBox * bbox) { @@ -226,39 +264,10 @@ prediction_reset (GstInferencePrediction * self) self->predictions = g_node_new (self); } -static void -node_get_children (GNode * node, gpointer data) -{ - GSList **children = (GSList **) data; - GstInferencePrediction *prediction; - - g_return_if_fail (node); - g_return_if_fail (children); - - prediction = (GstInferencePrediction *) node->data; - - *children = g_slist_append (*children, prediction); -} - -static GSList * -prediction_get_children (GstInferencePrediction * self) -{ - GSList *children = NULL; - - g_return_val_if_fail (self, NULL); - - if (self->predictions) { - g_node_children_foreach (self->predictions, G_TRAVERSE_ALL, - node_get_children, &children); - } - - return children; -} - static void prediction_free (GstInferencePrediction * self) { - GSList *children = prediction_get_children (self); + GSList *children = gst_inference_prediction_get_children (self); GSList *iter = NULL; for (iter = children; iter != NULL; iter = g_slist_next (iter)) { diff --git a/gst-libs/gst/r2inference/gstinferenceprediction.h b/gst-libs/gst/r2inference/gstinferenceprediction.h index a6641c80..eb905b20 100644 --- a/gst-libs/gst/r2inference/gstinferenceprediction.h +++ b/gst-libs/gst/r2inference/gstinferenceprediction.h @@ -64,6 +64,8 @@ void gst_inference_prediction_free (GstInferencePrediction * self); GstInferencePrediction * gst_inference_prediction_ref (GstInferencePrediction * self); void gst_inference_prediction_unref (GstInferencePrediction * self); gchar * gst_inference_prediction_to_string (GstInferencePrediction * self); +void gst_inference_prediction_append (GstInferencePrediction * self, GstInferencePrediction * child); +GSList * gst_inference_prediction_get_children (GstInferencePrediction * self); G_END_DECLS From 3b1c846b0990b66ca7a917c4725c06c56b4f380e Mon Sep 17 00:00:00 2001 From: Michael Gruner Date: Tue, 24 Dec 2019 11:40:38 -0600 Subject: [PATCH 048/188] Free existing classes on destruction --- .../gst/r2inference/gstinferenceprediction.c | 29 ++++++++----------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/gst-libs/gst/r2inference/gstinferenceprediction.c b/gst-libs/gst/r2inference/gstinferenceprediction.c index 73290038..93b1706d 100644 --- a/gst-libs/gst/r2inference/gstinferenceprediction.c +++ b/gst-libs/gst/r2inference/gstinferenceprediction.c @@ -23,7 +23,7 @@ static gchar *prediction_to_string (GstInferencePrediction * self, gint level); static void bounding_box_reset (BoundingBox * bbox); static gchar *bounding_box_to_string (BoundingBox * bbox, gint level); -static void classification_reset (GList * classification); +static void classification_free (GList ** classification); static void node_get_children (GNode * node, gpointer data); static gpointer node_copy (gconstpointer node, gpointer data); @@ -40,6 +40,8 @@ gst_inference_prediction_new (void) (GstMiniObjectFreeFunction) prediction_free); self->predictions = NULL; + self->classifications = NULL; + prediction_reset (self); return self; @@ -221,21 +223,6 @@ gst_inference_prediction_get_children (GstInferencePrediction * self) return children; } -static void -classification_reset (GList * classifications) -{ - GList * iter = NULL; - - for (iter = classifications; iter != NULL; iter = g_list_next (iter)) { - Classification *c = (Classification *)iter->data; - c->class_id = 0; - c->class_prob = 0.0f; - c->class_label = NULL; - c->num_classes = 0; - c->classes_probs = NULL; - } -} - static void bounding_box_reset (BoundingBox * bbox) { @@ -256,7 +243,6 @@ prediction_reset (GstInferencePrediction * self) self->enabled = FALSE; bounding_box_reset (&self->bbox); - classification_reset (self->classifications); /* Free al children */ prediction_free (self); @@ -264,6 +250,13 @@ prediction_reset (GstInferencePrediction * self) self->predictions = g_node_new (self); } +static void +classification_free (GList ** classifications) +{ + g_list_free_full (*classifications, g_free); + *classifications = NULL; +} + static void prediction_free (GstInferencePrediction * self) { @@ -276,6 +269,8 @@ prediction_free (GstInferencePrediction * self) gst_inference_prediction_unref (child); } + classification_free (&self->classifications); + if (self->predictions) { g_node_destroy (self->predictions); self->predictions = NULL; From 4a51b627bac0eed5ba6fbb8ba874324b21e2fe6c Mon Sep 17 00:00:00 2001 From: Michael Gruner Date: Tue, 24 Dec 2019 12:34:43 -0600 Subject: [PATCH 049/188] Install yolo2 classes into predictions --- gst-libs/gst/r2inference/gstinferencepostprocess.c | 10 ++++++++++ gst-libs/gst/r2inference/gstinferenceprediction.c | 10 ++++++++++ gst-libs/gst/r2inference/gstinferenceprediction.h | 1 + 3 files changed, 21 insertions(+) diff --git a/gst-libs/gst/r2inference/gstinferencepostprocess.c b/gst-libs/gst/r2inference/gstinferencepostprocess.c index 66856ac5..d1eba609 100644 --- a/gst-libs/gst/r2inference/gstinferencepostprocess.c +++ b/gst-libs/gst/r2inference/gstinferencepostprocess.c @@ -252,6 +252,7 @@ GstInferencePrediction * gst_create_prediction_from_box (GstVideoInference * vi, BBox * box) { GstInferencePrediction *predict = NULL; + Classification *c = NULL; g_return_val_if_fail (vi != NULL, NULL); g_return_val_if_fail (box != NULL, NULL); @@ -262,6 +263,15 @@ gst_create_prediction_from_box (GstVideoInference * vi, BBox * box) predict->bbox.width = box->width; predict->bbox.height = box->height; + c = g_malloc0 (sizeof (Classification)); + c->class_id = box->label; + c->class_prob = box->prob; + c->class_label = NULL; + c->num_classes = 1; + c->classes_probs = &c->class_prob; + + gst_inference_prediction_append_classification (predict, c); + return predict; } diff --git a/gst-libs/gst/r2inference/gstinferenceprediction.c b/gst-libs/gst/r2inference/gstinferenceprediction.c index 93b1706d..ba83b60f 100644 --- a/gst-libs/gst/r2inference/gstinferenceprediction.c +++ b/gst-libs/gst/r2inference/gstinferenceprediction.c @@ -276,3 +276,13 @@ prediction_free (GstInferencePrediction * self) self->predictions = NULL; } } + +void +gst_inference_prediction_append_classification (GstInferencePrediction * self, + Classification * c) +{ + g_return_if_fail (self); + g_return_if_fail (c); + + self->classifications = g_list_append (self->classifications, c); +} diff --git a/gst-libs/gst/r2inference/gstinferenceprediction.h b/gst-libs/gst/r2inference/gstinferenceprediction.h index eb905b20..d18fb282 100644 --- a/gst-libs/gst/r2inference/gstinferenceprediction.h +++ b/gst-libs/gst/r2inference/gstinferenceprediction.h @@ -66,6 +66,7 @@ void gst_inference_prediction_unref (GstInferencePrediction * self); gchar * gst_inference_prediction_to_string (GstInferencePrediction * self); void gst_inference_prediction_append (GstInferencePrediction * self, GstInferencePrediction * child); GSList * gst_inference_prediction_get_children (GstInferencePrediction * self); +void gst_inference_prediction_append_classification (GstInferencePrediction * self, Classification * c); G_END_DECLS From 7c09a510de20948d10100284907217b56bc5f1b7 Mon Sep 17 00:00:00 2001 From: Michael Gruner Date: Tue, 24 Dec 2019 12:49:35 -0600 Subject: [PATCH 050/188] Add classes to prediction serialization --- .../gst/r2inference/gstinferenceprediction.c | 75 +++++++++++++++++-- 1 file changed, 68 insertions(+), 7 deletions(-) diff --git a/gst-libs/gst/r2inference/gstinferenceprediction.c b/gst-libs/gst/r2inference/gstinferenceprediction.c index ba83b60f..8be6512f 100644 --- a/gst-libs/gst/r2inference/gstinferenceprediction.c +++ b/gst-libs/gst/r2inference/gstinferenceprediction.c @@ -19,11 +19,16 @@ static GstInferencePrediction *prediction_copy (const GstInferencePrediction * static void prediction_free (GstInferencePrediction * obj); static void prediction_reset (GstInferencePrediction * self); static gchar *prediction_to_string (GstInferencePrediction * self, gint level); +static gchar *prediction_children_to_string (GstInferencePrediction * self, + gint level); +static gchar *prediction_classes_to_string (GstInferencePrediction * self, + gint level); static void bounding_box_reset (BoundingBox * bbox); static gchar *bounding_box_to_string (BoundingBox * bbox, gint level); static void classification_free (GList ** classification); +static gchar *classification_to_string (Classification * c, gint level); static void node_get_children (GNode * node, gpointer data); static gpointer node_copy (gconstpointer node, gpointer data); @@ -124,6 +129,22 @@ gst_inference_prediction_copy (const GstInferencePrediction * self) return (GstInferencePrediction *) other->data; } +static gchar * +classification_to_string (Classification * c, gint level) +{ + gint indent = level * 2; + + g_return_val_if_fail (c, NULL); + + return g_strdup_printf ("{\n" + "%*s ID : %u\n" + "%*s Label : %s\n" + "%*s Probability : %f\n" + "%*s}", + indent, "", c->class_id, + indent, "", c->class_label, indent, "", c->class_prob, indent, ""); +} + static gchar * bounding_box_to_string (BoundingBox * bbox, gint level) { @@ -143,36 +164,74 @@ bounding_box_to_string (BoundingBox * bbox, gint level) } static gchar * -prediction_to_string (GstInferencePrediction * self, gint level) +prediction_children_to_string (GstInferencePrediction * self, gint level) { GSList *subpreds = NULL; GSList *iter = NULL; GString *string = NULL; - gint indent = level * 2; - gchar *bbox = NULL; - gchar *children = NULL; - gchar *prediction = NULL; g_return_val_if_fail (self, NULL); + /* Build the child predictions using a GString */ string = g_string_new (NULL); subpreds = gst_inference_prediction_get_children (self); + for (iter = subpreds; iter != NULL; iter = g_slist_next (iter)) { GstInferencePrediction *pred = (GstInferencePrediction *) iter->data; - gchar *child = prediction_to_string (pred, level + 2); + gchar *child = prediction_to_string (pred, level + 1); g_string_append_printf (string, "%s, ", child); g_free (child); } - children = g_string_free (string, FALSE); + return g_string_free (string, FALSE); +} + +static gchar * +prediction_classes_to_string (GstInferencePrediction * self, gint level) +{ + GList *iter = NULL; + GString *string = NULL; + + g_return_val_if_fail (self, NULL); + + /* Build the classes for this predictions using a GString */ + string = g_string_new (NULL); + + for (iter = self->classifications; iter != NULL; iter = g_list_next (iter)) { + Classification *c = (Classification *) iter->data; + gchar *sclass = classification_to_string (c, level + 1); + + g_string_append_printf (string, "%s, ", sclass); + g_free (sclass); + } + + return g_string_free (string, FALSE); +} + +static gchar * +prediction_to_string (GstInferencePrediction * self, gint level) +{ + gint indent = level * 2; + gchar *bbox = NULL; + gchar *children = NULL; + gchar *classes = NULL; + gchar *prediction = NULL; + + g_return_val_if_fail (self, NULL); + bbox = bounding_box_to_string (&self->bbox, level + 1); + classes = prediction_classes_to_string (self, level + 1); + children = prediction_children_to_string (self, level + 1); prediction = g_strdup_printf ("{\n" "%*s id : %llu,\n" "%*s enabled : %s,\n" "%*s bbox : %s,\n" + "%*s classes : [\n" + "%*s %s\n" + "%*s ],\n" "%*s predictions : [\n" "%*s %s\n" "%*s ]\n" @@ -180,10 +239,12 @@ prediction_to_string (GstInferencePrediction * self, gint level) indent, "", self->id, indent, "", self->enabled ? "True" : "False", indent, "", bbox, + indent, "", indent, "", classes, indent, "", indent, "", indent, "", children, indent, "", indent, ""); g_free (bbox); g_free (children); + g_free (classes); return prediction; } From 1a2387cce8a10ec2f8d7b794a49c1738e0b24c42 Mon Sep 17 00:00:00 2001 From: Michael Gruner Date: Thu, 26 Dec 2019 19:23:42 -0600 Subject: [PATCH 051/188] Move classification to a separate module --- ext/r2inference/gstinceptionv2.c | 10 +- gst-libs/gst/r2inference/Makefile.am | 6 +- .../r2inference/gstinferenceclassification.c | 177 ++++++++++++++++++ .../r2inference/gstinferenceclassification.h | 48 +++++ gst-libs/gst/r2inference/gstinferencedebug.c | 28 --- .../gst/r2inference/gstinferencepostprocess.c | 51 +++-- .../gst/r2inference/gstinferencepostprocess.h | 2 +- .../gst/r2inference/gstinferenceprediction.c | 49 ++--- .../gst/r2inference/gstinferenceprediction.h | 18 +- 9 files changed, 280 insertions(+), 109 deletions(-) create mode 100644 gst-libs/gst/r2inference/gstinferenceclassification.c create mode 100644 gst-libs/gst/r2inference/gstinferenceclassification.h diff --git a/ext/r2inference/gstinceptionv2.c b/ext/r2inference/gstinceptionv2.c index e7387fd0..be38511e 100644 --- a/ext/r2inference/gstinceptionv2.c +++ b/ext/r2inference/gstinceptionv2.c @@ -240,7 +240,7 @@ gst_inceptionv2_postprocess_meta (GstVideoInference * vi, GstVideoInfo * info_model, gboolean * valid_prediction) { GstInferenceMeta *imeta = NULL; - Classification *class = NULL; + GstInferenceClassification *c = NULL; GstInferencePrediction *root = NULL; g_return_val_if_fail (vi != NULL, FALSE); @@ -257,11 +257,9 @@ gst_inceptionv2_postprocess_meta (GstVideoInference * vi, return FALSE; } - class = gst_create_class_from_prediction (vi, prediction, predsize); - root->classifications = - g_list_append (root->classifications, (gpointer) class); - - gst_inference_print_classes (vi, gst_inceptionv2_debug_category, imeta); + c = gst_create_class_from_prediction (vi, prediction, predsize); + gst_inference_prediction_append_classification (root, c); + gst_inference_print_predictions (vi, gst_inceptionv2_debug_category, imeta); *valid_prediction = TRUE; return TRUE; diff --git a/gst-libs/gst/r2inference/Makefile.am b/gst-libs/gst/r2inference/Makefile.am index 6c6a8301..d59b1972 100644 --- a/gst-libs/gst/r2inference/Makefile.am +++ b/gst-libs/gst/r2inference/Makefile.am @@ -12,7 +12,8 @@ libgstinference_@GST_API_VERSION@_la_SOURCES= \ gstinferencepreprocess.c \ gstinferencepostprocess.c \ gstinferencedebug.c \ - gstinferenceprediction.c + gstinferenceprediction.c \ + gstinferenceclassification.c libgstinference_@GST_API_VERSION@_la_CXXFLAGS= \ $(GST_CXXFLAGS) \ @@ -48,4 +49,5 @@ gstinferenceinclude_HEADERS= \ gstinferencepostprocess.h \ gstinferencedebug.h \ gstinferencemeta.h \ - gstinferenceprediction.h + gstinferenceprediction.h \ + gstinferenceclassification.h diff --git a/gst-libs/gst/r2inference/gstinferenceclassification.c b/gst-libs/gst/r2inference/gstinferenceclassification.c new file mode 100644 index 00000000..1c4bb9f9 --- /dev/null +++ b/gst-libs/gst/r2inference/gstinferenceclassification.c @@ -0,0 +1,177 @@ +/* Copyright (C) 2019 RidgeRun, LLC (http://www.ridgerun.com) + * All Rights Reserved. + * + * The contents of this software are proprietary and confidential to RidgeRun, + * LLC. No part of this program may be photocopied, reproduced or translated + * into another programming language without prior written consent of + * RidgeRun, LLC. The user is free to modify the source code after obtaining + * a software license from RidgeRun. All source code changes must be provided + * back to RidgeRun without any encumbrance. +*/ + +#include "gstinferenceclassification.h" + +static GType gst_inference_classification_get_type (void); +GST_DEFINE_MINI_OBJECT_TYPE (GstInferenceClassification, + gst_inference_classification); + +static void classification_free (GstInferenceClassification * self); +static void classification_reset (GstInferenceClassification * self); + +static gdouble *probabilities_copy (const gdouble * from, gint num_classes); + +static void +classification_reset (GstInferenceClassification * self) +{ + g_return_if_fail (self); + + self->class_id = -1; + self->class_prob = 0.0f; + self->num_classes = 0; + + if (self->class_label) { + g_free (self->class_label); + } + self->class_label = NULL; + + if (self->probabilities) { + g_free (self->probabilities); + } + self->probabilities = NULL; + + if (self->labels) { + g_strfreev (self->labels); + } + self->labels = NULL; +} + +GstInferenceClassification * +gst_inference_classification_new (void) +{ + GstInferenceClassification *self = g_slice_new (GstInferenceClassification); + + gst_mini_object_init (GST_MINI_OBJECT_CAST (self), 0, + gst_inference_classification_get_type (), + (GstMiniObjectCopyFunction) gst_inference_classification_copy, NULL, + (GstMiniObjectFreeFunction) classification_free); + + self->class_label = NULL; + self->probabilities = NULL; + self->labels = NULL; + + classification_reset (self); + + return self; +} + +static gdouble * +probabilities_copy (const gdouble * from, gint num_classes) +{ + gsize size = 0; + gdouble *to = NULL; + + g_return_val_if_fail (from, NULL); + g_return_val_if_fail (num_classes > 0, NULL); + + size = num_classes * sizeof (double); + to = g_malloc0 (size); + + memcpy (to, from, size); + + return to; +} + +GstInferenceClassification * +gst_inference_classification_new_full (gint class_id, gdouble class_prob, + const gchar * class_label, gint num_classes, const gdouble * probabilities, + gchar ** labels) +{ + GstInferenceClassification *self = gst_inference_classification_new (); + + self->class_id = class_id; + self->class_prob = class_prob; + self->num_classes = num_classes; + + if (class_label) { + self->class_label = g_strdup (class_label); + } + + if (probabilities && num_classes > 0) { + self->probabilities = probabilities_copy (probabilities, num_classes); + } + + if (labels) { + self->labels = g_strdupv (labels); + } + + return self; +} + +GstInferenceClassification * +gst_inference_classification_ref (GstInferenceClassification * self) +{ + g_return_val_if_fail (self, NULL); + + return (GstInferenceClassification *) + gst_mini_object_ref (GST_MINI_OBJECT_CAST (self)); +} + +void +gst_inference_classification_unref (GstInferenceClassification * self) +{ + g_return_if_fail (self); + + gst_mini_object_unref (GST_MINI_OBJECT_CAST (self)); +} + +GstInferenceClassification * +gst_inference_classification_copy (const GstInferenceClassification * self) +{ + GstInferenceClassification *other = NULL; + + g_return_val_if_fail (self, NULL); + + other = gst_inference_classification_new (); + + other->class_id = self->class_id; + other->class_prob = self->class_prob; + other->num_classes = self->num_classes; + + if (self->class_label) { + other->class_label = g_strdup (self->class_label); + } + + if (self->probabilities) { + other->probabilities = + probabilities_copy (self->probabilities, self->num_classes); + } + + if (self->labels) { + other->labels = g_strdupv (self->labels); + } + + return other; +} + +gchar * +gst_inference_classification_to_string (GstInferenceClassification * self, + gint level) +{ + gint indent = level * 2; + + g_return_val_if_fail (self, NULL); + + return g_strdup_printf ("{\n" + "%*s Class : %d/%d\n" + "%*s Label : %s\n" + "%*s Probability : %f\n" + "%*s}", + indent, "", self->class_id, self->num_classes, + indent, "", self->class_label, indent, "", self->class_prob, indent, ""); +} + +static void +classification_free (GstInferenceClassification * self) +{ + classification_reset (self); +} diff --git a/gst-libs/gst/r2inference/gstinferenceclassification.h b/gst-libs/gst/r2inference/gstinferenceclassification.h new file mode 100644 index 00000000..388f990c --- /dev/null +++ b/gst-libs/gst/r2inference/gstinferenceclassification.h @@ -0,0 +1,48 @@ +/* Copyright (C) 2019 RidgeRun, LLC (http://www.ridgerun.com) + * All Rights Reserved. + * + * The contents of this software are proprietary and confidential to RidgeRun, + * LLC. No part of this program may be photocopied, reproduced or translated + * into another programming language without prior written consent of + * RidgeRun, LLC. The user is free to modify the source code after obtaining + * a software license from RidgeRun. All source code changes must be provided + * back to RidgeRun without any encumbrance. +*/ + +#ifndef __GST_INFERENCE_CLASSIFICATION__ +#define __GST_INFERENCE_CLASSIFICATION__ + +#include + +G_BEGIN_DECLS + +/** + * Classification to be assigned to an object + */ +typedef struct _GstInferenceClassification GstInferenceClassification; +struct _GstInferenceClassification +{ + GstMiniObject base; + + gint class_id; + gdouble class_prob; + gchar *class_label; + gint num_classes; + gdouble *probabilities; + gchar **labels; +}; + +GstInferenceClassification * gst_inference_classification_new (void); +GstInferenceClassification * gst_inference_classification_new_full (gint class_id, gdouble class_prob, + const gchar * class_label, gint num_classes, const gdouble * probabilities, + gchar ** labels); +void gst_inference_classification_reset (GstInferenceClassification * self); +GstInferenceClassification * gst_inference_classification_copy (const GstInferenceClassification * self); +void gst_inference_classification_free (GstInferenceClassification * self); +GstInferenceClassification * gst_inference_classification_ref (GstInferenceClassification * self); +void gst_inference_classification_unref (GstInferenceClassification * self); +gchar * gst_inference_classification_to_string (GstInferenceClassification * self, gint level); + +G_END_DECLS + +#endif // __GST_INFERENCE_CLASSIFICATION__ diff --git a/gst-libs/gst/r2inference/gstinferencedebug.c b/gst-libs/gst/r2inference/gstinferencedebug.c index cd25ea65..2a0cfdeb 100644 --- a/gst-libs/gst/r2inference/gstinferencedebug.c +++ b/gst-libs/gst/r2inference/gstinferencedebug.c @@ -110,31 +110,3 @@ gst_inference_print_predictions (GstVideoInference * vi, GST_CAT_LOG (category, "\n%s", spred); g_free (spred); } - -static void -gst_inference_print_class (gpointer data, gpointer user_data) -{ - Classification *class = (Classification *) data; - GstDebugCategory *category = (GstDebugCategory *) user_data; - - g_return_if_fail (category != NULL); - g_return_if_fail (class != NULL); - - GST_CAT_LOG (category, "Class info: ID: %d - Probability: %.2f - Label: %s", - class->class_id, class->class_prob, - (class->class_label != NULL) ? class->class_label : "No Label set"); -} - -void -gst_inference_print_classes (GstVideoInference * vi, - GstDebugCategory * category, GstInferenceMeta * inference_meta) -{ - GstInferencePrediction *root = NULL; - - g_return_if_fail (vi != NULL); - g_return_if_fail (inference_meta != NULL); - - root = inference_meta->prediction; - g_list_foreach (root->classifications, gst_inference_print_class, - (gpointer) category); -} diff --git a/gst-libs/gst/r2inference/gstinferencepostprocess.c b/gst-libs/gst/r2inference/gstinferencepostprocess.c index d1eba609..27a7d8a1 100644 --- a/gst-libs/gst/r2inference/gstinferencepostprocess.c +++ b/gst-libs/gst/r2inference/gstinferencepostprocess.c @@ -252,64 +252,61 @@ GstInferencePrediction * gst_create_prediction_from_box (GstVideoInference * vi, BBox * box) { GstInferencePrediction *predict = NULL; - Classification *c = NULL; + GstInferenceClassification *c = NULL; g_return_val_if_fail (vi != NULL, NULL); g_return_val_if_fail (box != NULL, NULL); predict = gst_inference_prediction_new (); predict->bbox.x = box->x; - predict->bbox.y = box->x; + predict->bbox.y = box->y; predict->bbox.width = box->width; predict->bbox.height = box->height; - c = g_malloc0 (sizeof (Classification)); - c->class_id = box->label; - c->class_prob = box->prob; - c->class_label = NULL; - c->num_classes = 1; - c->classes_probs = &c->class_prob; - + c = gst_inference_classification_new_full (box->label, box->prob, NULL, 1, + &box->prob, NULL); gst_inference_prediction_append_classification (predict, c); return predict; } -Classification * +GstInferenceClassification * gst_create_class_from_prediction (GstVideoInference * vi, const gpointer prediction, gsize predsize) { - Classification *new_class = NULL; gdouble max = -1; gint index = 0; + gdouble *probs = NULL; + gint num_classes = 0; + const gchar *label = NULL; + gchar **labels = NULL; g_return_val_if_fail (vi != NULL, NULL); - /* Create new Classification */ - new_class = g_malloc (sizeof (Classification)); - /* Fill Classification with inference output */ - new_class->num_classes = predsize / sizeof (gfloat); - new_class->classes_probs = - g_malloc (new_class->num_classes * sizeof (gdouble)); - for (gint i = 0; i < new_class->num_classes; ++i) { - new_class->classes_probs[i] = (gdouble) ((gfloat *) prediction)[i]; + num_classes = predsize / sizeof (gfloat); + + /* FIXME: This is just dumb */ + if (sizeof (gfloat) != sizeof (gdouble)) { + probs = g_malloc (num_classes * sizeof (gdouble)); + + for (gint i = 0; i < num_classes; ++i) { + probs[i] = (gdouble) ((gfloat *) prediction)[i]; + } + } else { + probs = (gdouble *) prediction; } /* Obtain the highest probability and set it as class */ - for (gint i = 0; i < new_class->num_classes; ++i) { - gfloat current = ((gfloat *) prediction)[i]; + for (gint i = 0; i < num_classes; ++i) { + gdouble current = probs[i]; if (current > max) { max = current; index = i; } } - new_class->class_id = index; - new_class->class_prob = max; - /* TODO: Fill class label based on labels file */ - new_class->class_label = NULL; - - return new_class; + return gst_inference_classification_new_full (index, max, label, num_classes, + probs, labels); } static void diff --git a/gst-libs/gst/r2inference/gstinferencepostprocess.h b/gst-libs/gst/r2inference/gstinferencepostprocess.h index 26b9920c..fbcf126f 100644 --- a/gst-libs/gst/r2inference/gstinferencepostprocess.h +++ b/gst-libs/gst/r2inference/gstinferencepostprocess.h @@ -84,7 +84,7 @@ GstInferencePrediction *gst_create_prediction_from_box (GstVideoInference * vi, * \param prediction Value of the prediction * \param predsize Size of the prediction */ -Classification *gst_create_class_from_prediction (GstVideoInference * vi, +GstInferenceClassification *gst_create_class_from_prediction (GstVideoInference * vi, const gpointer prediction, gsize predsize); G_END_DECLS diff --git a/gst-libs/gst/r2inference/gstinferenceprediction.c b/gst-libs/gst/r2inference/gstinferenceprediction.c index 8be6512f..44ae86df 100644 --- a/gst-libs/gst/r2inference/gstinferenceprediction.c +++ b/gst-libs/gst/r2inference/gstinferenceprediction.c @@ -24,12 +24,12 @@ static gchar *prediction_children_to_string (GstInferencePrediction * self, static gchar *prediction_classes_to_string (GstInferencePrediction * self, gint level); +static GstInferenceClassification + * classification_copy (GstInferenceClassification * from, gpointer data); + static void bounding_box_reset (BoundingBox * bbox); static gchar *bounding_box_to_string (BoundingBox * bbox, gint level); -static void classification_free (GList ** classification); -static gchar *classification_to_string (Classification * c, gint level); - static void node_get_children (GNode * node, gpointer data); static gpointer node_copy (gconstpointer node, gpointer data); static gboolean node_assign (GNode * node, gpointer data); @@ -79,6 +79,12 @@ gst_inference_prediction_append (GstInferencePrediction * self, g_node_append (self->predictions, child->predictions); } +static GstInferenceClassification * +classification_copy (GstInferenceClassification * from, gpointer data) +{ + return gst_inference_classification_copy (from); +} + static GstInferencePrediction * prediction_copy (const GstInferencePrediction * self) { @@ -92,6 +98,10 @@ prediction_copy (const GstInferencePrediction * self) other->enabled = self->enabled; other->bbox = self->bbox; + other->classifications = + g_list_copy_deep (self->classifications, (GCopyFunc) classification_copy, + NULL); + return other; } @@ -129,22 +139,6 @@ gst_inference_prediction_copy (const GstInferencePrediction * self) return (GstInferencePrediction *) other->data; } -static gchar * -classification_to_string (Classification * c, gint level) -{ - gint indent = level * 2; - - g_return_val_if_fail (c, NULL); - - return g_strdup_printf ("{\n" - "%*s ID : %u\n" - "%*s Label : %s\n" - "%*s Probability : %f\n" - "%*s}", - indent, "", c->class_id, - indent, "", c->class_label, indent, "", c->class_prob, indent, ""); -} - static gchar * bounding_box_to_string (BoundingBox * bbox, gint level) { @@ -200,8 +194,8 @@ prediction_classes_to_string (GstInferencePrediction * self, gint level) string = g_string_new (NULL); for (iter = self->classifications; iter != NULL; iter = g_list_next (iter)) { - Classification *c = (Classification *) iter->data; - gchar *sclass = classification_to_string (c, level + 1); + GstInferenceClassification *c = (GstInferenceClassification *) iter->data; + gchar *sclass = gst_inference_classification_to_string (c, level + 1); g_string_append_printf (string, "%s, ", sclass); g_free (sclass); @@ -311,13 +305,6 @@ prediction_reset (GstInferencePrediction * self) self->predictions = g_node_new (self); } -static void -classification_free (GList ** classifications) -{ - g_list_free_full (*classifications, g_free); - *classifications = NULL; -} - static void prediction_free (GstInferencePrediction * self) { @@ -330,7 +317,9 @@ prediction_free (GstInferencePrediction * self) gst_inference_prediction_unref (child); } - classification_free (&self->classifications); + g_list_free_full (self->classifications, + (GDestroyNotify) gst_inference_classification_unref); + self->classifications = NULL; if (self->predictions) { g_node_destroy (self->predictions); @@ -340,7 +329,7 @@ prediction_free (GstInferencePrediction * self) void gst_inference_prediction_append_classification (GstInferencePrediction * self, - Classification * c) + GstInferenceClassification * c) { g_return_if_fail (self); g_return_if_fail (c); diff --git a/gst-libs/gst/r2inference/gstinferenceprediction.h b/gst-libs/gst/r2inference/gstinferenceprediction.h index d18fb282..970ffa34 100644 --- a/gst-libs/gst/r2inference/gstinferenceprediction.h +++ b/gst-libs/gst/r2inference/gstinferenceprediction.h @@ -12,26 +12,13 @@ #ifndef __GST_INFERENCE_PREDICTION__ #define __GST_INFERENCE_PREDICTION__ -#include +#include G_BEGIN_DECLS typedef struct _BoundingBox BoundingBox; typedef struct _GstInferencePrediction GstInferencePrediction; -/** - * Classification to be assigned to an object - */ -typedef struct _Classification Classification; -struct _Classification -{ - gint class_id; - gdouble class_prob; - gchar *class_label; - gint num_classes; - gdouble *classes_probs; -}; - /** * Size and coordinates of an image region */ @@ -66,7 +53,8 @@ void gst_inference_prediction_unref (GstInferencePrediction * self); gchar * gst_inference_prediction_to_string (GstInferencePrediction * self); void gst_inference_prediction_append (GstInferencePrediction * self, GstInferencePrediction * child); GSList * gst_inference_prediction_get_children (GstInferencePrediction * self); -void gst_inference_prediction_append_classification (GstInferencePrediction * self, Classification * c); +void gst_inference_prediction_append_classification (GstInferencePrediction * self, + GstInferenceClassification * c); G_END_DECLS From 25c3d475ee4740c8423458ef16e37d5593c548d0 Mon Sep 17 00:00:00 2001 From: Michael Gruner Date: Fri, 27 Dec 2019 15:42:47 -0600 Subject: [PATCH 052/188] Handle legacy and new metas simultaneously --- ext/r2inference/gstfacenetv1.c | 7 +- ext/r2inference/gstinceptionv1.c | 6 +- ext/r2inference/gstinceptionv2.c | 34 +++- ext/r2inference/gstinceptionv3.c | 7 +- ext/r2inference/gstinceptionv4.c | 7 +- ext/r2inference/gstmobilenetv2.c | 7 +- ext/r2inference/gstresnet50v1.c | 7 +- ext/r2inference/gsttinyyolov2.c | 31 +++- ext/r2inference/gsttinyyolov3.c | 11 +- gst-libs/gst/r2inference/gstinferencemeta.c | 34 +++- gst-libs/gst/r2inference/gstvideoinference.c | 154 ++++++------------- gst-libs/gst/r2inference/gstvideoinference.h | 5 +- 12 files changed, 165 insertions(+), 145 deletions(-) diff --git a/ext/r2inference/gstfacenetv1.c b/ext/r2inference/gstfacenetv1.c index 258fa2e4..aa413e42 100644 --- a/ext/r2inference/gstfacenetv1.c +++ b/ext/r2inference/gstfacenetv1.c @@ -62,7 +62,7 @@ static void gst_facenetv1_finalize (GObject * object); static gboolean gst_facenetv1_preprocess (GstVideoInference * vi, GstVideoFrame * inframe, GstVideoFrame * outframe); static gboolean gst_facenetv1_postprocess (GstVideoInference * vi, - const gpointer prediction, gsize predsize, GstMeta * meta_model, + const gpointer prediction, gsize predsize, GstMeta * meta_model[2], GstVideoInfo * info_model, gboolean * valid_prediction); static gboolean gst_facenetv1_start (GstVideoInference * vi); static gboolean gst_facenetv1_stop (GstVideoInference * vi); @@ -206,11 +206,12 @@ gst_facenetv1_preprocess (GstVideoInference * vi, static gboolean gst_facenetv1_postprocess (GstVideoInference * vi, const gpointer prediction, - gsize predsize, GstMeta * meta_model, GstVideoInfo * info_model, + gsize predsize, GstMeta * meta_model[2], GstVideoInfo * info_model, gboolean * valid_prediction) { - GstClassificationMeta *class_meta = (GstClassificationMeta *) meta_model; + GstClassificationMeta *class_meta = (GstClassificationMeta *) meta_model[0]; GstDebugLevel gst_debug_level = GST_LEVEL_LOG; + GST_LOG_OBJECT (vi, "Postprocess"); gst_fill_classification_meta (class_meta, prediction, predsize); diff --git a/ext/r2inference/gstinceptionv1.c b/ext/r2inference/gstinceptionv1.c index aaed7dd8..a362af8d 100644 --- a/ext/r2inference/gstinceptionv1.c +++ b/ext/r2inference/gstinceptionv1.c @@ -57,7 +57,7 @@ GST_DEBUG_CATEGORY_STATIC (gst_inceptionv1_debug_category); static gboolean gst_inceptionv1_preprocess (GstVideoInference * vi, GstVideoFrame * inframe, GstVideoFrame * outframe); static gboolean gst_inceptionv1_postprocess (GstVideoInference * vi, - const gpointer prediction, gsize predsize, GstMeta * meta_model, + const gpointer prediction, gsize predsize, GstMeta * meta_model[2], GstVideoInfo * info_model, gboolean * valid_prediction); static gboolean gst_inceptionv1_start (GstVideoInference * vi); static gboolean gst_inceptionv1_stop (GstVideoInference * vi); @@ -145,10 +145,10 @@ gst_inceptionv1_preprocess (GstVideoInference * vi, static gboolean gst_inceptionv1_postprocess (GstVideoInference * vi, const gpointer prediction, - gsize predsize, GstMeta * meta_model, GstVideoInfo * info_model, + gsize predsize, GstMeta * meta_model[2], GstVideoInfo * info_model, gboolean * valid_prediction) { - GstClassificationMeta *class_meta = (GstClassificationMeta *) meta_model; + GstClassificationMeta *class_meta = (GstClassificationMeta *) meta_model[0]; GstDebugLevel gst_debug_level = GST_LEVEL_LOG; GST_LOG_OBJECT (vi, "Postprocess"); diff --git a/ext/r2inference/gstinceptionv2.c b/ext/r2inference/gstinceptionv2.c index be38511e..5c8bd285 100644 --- a/ext/r2inference/gstinceptionv2.c +++ b/ext/r2inference/gstinceptionv2.c @@ -65,10 +65,12 @@ static void gst_inceptionv2_finalize (GObject * object); static gboolean gst_inceptionv2_preprocess (GstVideoInference * vi, GstVideoFrame * inframe, GstVideoFrame * outframe); static gboolean gst_inceptionv2_postprocess (GstVideoInference * vi, + const gpointer prediction, gsize predsize, GstMeta * meta_model[2], + GstVideoInfo * info_model, gboolean * valid_prediction); +static gboolean gst_inceptionv2_postprocess_old (GstVideoInference * vi, const gpointer prediction, gsize predsize, GstMeta * meta_model, GstVideoInfo * info_model, gboolean * valid_prediction); -static gboolean -gst_inceptionv2_postprocess_meta (GstVideoInference * vi, +static gboolean gst_inceptionv2_postprocess_new (GstVideoInference * vi, const gpointer prediction, gsize predsize, GstMeta * meta_model, GstVideoInfo * info_model, gboolean * valid_prediction); static gboolean gst_inceptionv2_start (GstVideoInference * vi); @@ -144,8 +146,6 @@ gst_inceptionv2_class_init (GstInceptionv2Class * klass) vi_class->stop = GST_DEBUG_FUNCPTR (gst_inceptionv2_stop); vi_class->preprocess = GST_DEBUG_FUNCPTR (gst_inceptionv2_preprocess); vi_class->postprocess = GST_DEBUG_FUNCPTR (gst_inceptionv2_postprocess); - vi_class->postprocess_meta = - GST_DEBUG_FUNCPTR (gst_inceptionv2_postprocess_meta); vi_class->inference_meta_info = gst_classification_meta_get_info (); } @@ -217,12 +217,13 @@ gst_inceptionv2_preprocess (GstVideoInference * vi, } static gboolean -gst_inceptionv2_postprocess (GstVideoInference * vi, const gpointer prediction, - gsize predsize, GstMeta * meta_model, GstVideoInfo * info_model, - gboolean * valid_prediction) +gst_inceptionv2_postprocess_old (GstVideoInference * vi, + const gpointer prediction, gsize predsize, GstMeta * meta_model, + GstVideoInfo * info_model, gboolean * valid_prediction) { GstClassificationMeta *class_meta = (GstClassificationMeta *) meta_model; GstDebugLevel gst_debug_level = GST_LEVEL_LOG; + GST_LOG_OBJECT (vi, "Postprocess"); gst_fill_classification_meta (class_meta, prediction, predsize); @@ -235,7 +236,7 @@ gst_inceptionv2_postprocess (GstVideoInference * vi, const gpointer prediction, } static gboolean -gst_inceptionv2_postprocess_meta (GstVideoInference * vi, +gst_inceptionv2_postprocess_new (GstVideoInference * vi, const gpointer prediction, gsize predsize, GstMeta * meta_model, GstVideoInfo * info_model, gboolean * valid_prediction) { @@ -265,6 +266,23 @@ gst_inceptionv2_postprocess_meta (GstVideoInference * vi, return TRUE; } +static gboolean +gst_inceptionv2_postprocess (GstVideoInference * vi, const gpointer prediction, + gsize predsize, GstMeta * meta_model[2], GstVideoInfo * info_model, + gboolean * valid_prediction) +{ + gboolean ret = TRUE; + + ret &= + gst_inceptionv2_postprocess_old (vi, prediction, predsize, meta_model[0], + info_model, valid_prediction); + ret &= + gst_inceptionv2_postprocess_new (vi, prediction, predsize, meta_model[1], + info_model, valid_prediction); + + return ret; +} + static gboolean gst_inceptionv2_start (GstVideoInference * vi) { diff --git a/ext/r2inference/gstinceptionv3.c b/ext/r2inference/gstinceptionv3.c index 582906f3..2a10cf1f 100644 --- a/ext/r2inference/gstinceptionv3.c +++ b/ext/r2inference/gstinceptionv3.c @@ -56,7 +56,7 @@ GST_DEBUG_CATEGORY_STATIC (gst_inceptionv3_debug_category); static gboolean gst_inceptionv3_preprocess (GstVideoInference * vi, GstVideoFrame * inframe, GstVideoFrame * outframe); static gboolean gst_inceptionv3_postprocess (GstVideoInference * vi, - const gpointer prediction, gsize predsize, GstMeta * meta_model, + const gpointer prediction, gsize predsize, GstMeta * meta_model[2], GstVideoInfo * info_model, gboolean * valid_prediction); static gboolean gst_inceptionv3_start (GstVideoInference * vi); static gboolean gst_inceptionv3_stop (GstVideoInference * vi); @@ -144,11 +144,12 @@ gst_inceptionv3_preprocess (GstVideoInference * vi, static gboolean gst_inceptionv3_postprocess (GstVideoInference * vi, const gpointer prediction, - gsize predsize, GstMeta * meta_model, GstVideoInfo * info_model, + gsize predsize, GstMeta * meta_model[2], GstVideoInfo * info_model, gboolean * valid_prediction) { - GstClassificationMeta *class_meta = (GstClassificationMeta *) meta_model; + GstClassificationMeta *class_meta = (GstClassificationMeta *) meta_model[0]; GstDebugLevel gst_debug_level = GST_LEVEL_LOG; + GST_LOG_OBJECT (vi, "Postprocess"); gst_fill_classification_meta (class_meta, prediction, predsize); diff --git a/ext/r2inference/gstinceptionv4.c b/ext/r2inference/gstinceptionv4.c index eae66ca0..28471ea5 100644 --- a/ext/r2inference/gstinceptionv4.c +++ b/ext/r2inference/gstinceptionv4.c @@ -64,7 +64,7 @@ static void gst_inceptionv4_finalize (GObject * object); static gboolean gst_inceptionv4_preprocess (GstVideoInference * vi, GstVideoFrame * inframe, GstVideoFrame * outframe); static gboolean gst_inceptionv4_postprocess (GstVideoInference * vi, - const gpointer prediction, gsize predsize, GstMeta * meta_model, + const gpointer prediction, gsize predsize, GstMeta * meta_model[2], GstVideoInfo * info_model, gboolean * valid_prediction); static gboolean gst_inceptionv4_start (GstVideoInference * vi); static gboolean gst_inceptionv4_stop (GstVideoInference * vi); @@ -211,11 +211,12 @@ gst_inceptionv4_preprocess (GstVideoInference * vi, static gboolean gst_inceptionv4_postprocess (GstVideoInference * vi, const gpointer prediction, - gsize predsize, GstMeta * meta_model, GstVideoInfo * info_model, + gsize predsize, GstMeta * meta_model[2], GstVideoInfo * info_model, gboolean * valid_prediction) { - GstClassificationMeta *class_meta = (GstClassificationMeta *) meta_model; + GstClassificationMeta *class_meta = (GstClassificationMeta *) meta_model[0]; GstDebugLevel gst_debug_level = GST_LEVEL_LOG; + GST_LOG_OBJECT (vi, "Postprocess"); gst_fill_classification_meta (class_meta, prediction, predsize); diff --git a/ext/r2inference/gstmobilenetv2.c b/ext/r2inference/gstmobilenetv2.c index 05fa37da..d78a1187 100644 --- a/ext/r2inference/gstmobilenetv2.c +++ b/ext/r2inference/gstmobilenetv2.c @@ -55,7 +55,7 @@ GST_DEBUG_CATEGORY_STATIC (gst_mobilenetv2_debug_category); static gboolean gst_mobilenetv2_preprocess (GstVideoInference * vi, GstVideoFrame * inframe, GstVideoFrame * outframe); static gboolean gst_mobilenetv2_postprocess (GstVideoInference * vi, - const gpointer prediction, gsize predsize, GstMeta * meta_model, + const gpointer prediction, gsize predsize, GstMeta * meta_model[2], GstVideoInfo * info_model, gboolean * valid_prediction); static gboolean gst_mobilenetv2_start (GstVideoInference * vi); static gboolean gst_mobilenetv2_stop (GstVideoInference * vi); @@ -143,11 +143,12 @@ gst_mobilenetv2_preprocess (GstVideoInference * vi, static gboolean gst_mobilenetv2_postprocess (GstVideoInference * vi, const gpointer prediction, - gsize predsize, GstMeta * meta_model, GstVideoInfo * info_model, + gsize predsize, GstMeta * meta_model[2], GstVideoInfo * info_model, gboolean * valid_prediction) { - GstClassificationMeta *class_meta = (GstClassificationMeta *) meta_model; + GstClassificationMeta *class_meta = (GstClassificationMeta *) meta_model[0]; GstDebugLevel gst_debug_level = GST_LEVEL_LOG; + GST_LOG_OBJECT (vi, "Postprocess"); gst_fill_classification_meta (class_meta, prediction, predsize); diff --git a/ext/r2inference/gstresnet50v1.c b/ext/r2inference/gstresnet50v1.c index 9c86edf7..dd24d2d1 100644 --- a/ext/r2inference/gstresnet50v1.c +++ b/ext/r2inference/gstresnet50v1.c @@ -59,7 +59,7 @@ GST_DEBUG_CATEGORY_STATIC (gst_resnet50v1_debug_category); static gboolean gst_resnet50v1_preprocess (GstVideoInference * vi, GstVideoFrame * inframe, GstVideoFrame * outframe); static gboolean gst_resnet50v1_postprocess (GstVideoInference * vi, - const gpointer prediction, gsize predsize, GstMeta * meta_model, + const gpointer prediction, gsize predsize, GstMeta * meta_model[2], GstVideoInfo * info_model, gboolean * valid_prediction); static gboolean gst_resnet50v1_start (GstVideoInference * vi); static gboolean gst_resnet50v1_stop (GstVideoInference * vi); @@ -149,11 +149,12 @@ gst_resnet50v1_preprocess (GstVideoInference * vi, static gboolean gst_resnet50v1_postprocess (GstVideoInference * vi, const gpointer prediction, - gsize predsize, GstMeta * meta_model, GstVideoInfo * info_model, + gsize predsize, GstMeta * meta_model[2], GstVideoInfo * info_model, gboolean * valid_prediction) { - GstClassificationMeta *class_meta = (GstClassificationMeta *) meta_model; + GstClassificationMeta *class_meta = (GstClassificationMeta *) meta_model[0]; GstDebugLevel gst_debug_level = GST_LEVEL_LOG; + GST_LOG_OBJECT (vi, "Postprocess"); gst_fill_classification_meta (class_meta, prediction, predsize); diff --git a/ext/r2inference/gsttinyyolov2.c b/ext/r2inference/gsttinyyolov2.c index 13db3b5d..9b59141e 100644 --- a/ext/r2inference/gsttinyyolov2.c +++ b/ext/r2inference/gsttinyyolov2.c @@ -81,10 +81,14 @@ static gboolean gst_tinyyolov2_preprocess (GstVideoInference * vi, GstVideoFrame * inframe, GstVideoFrame * outframe); static gboolean gst_tinyyolov2_postprocess (GstVideoInference * vi, const gpointer prediction, - gsize predsize, GstMeta * meta_model, GstVideoInfo * info_model, + gsize predsize, GstMeta * meta_model[2], GstVideoInfo * info_model, gboolean * valid_prediction); static gboolean -gst_tinyyolov2_postprocess_meta (GstVideoInference * vi, +gst_tinyyolov2_postprocess_old (GstVideoInference * vi, + const gpointer prediction, gsize predsize, GstMeta * meta_model, + GstVideoInfo * info_model, gboolean * valid_prediction); +static gboolean +gst_tinyyolov2_postprocess_new (GstVideoInference * vi, const gpointer prediction, gsize predsize, GstMeta * meta_model, GstVideoInfo * info_model, gboolean * valid_prediction); static gboolean gst_tinyyolov2_start (GstVideoInference * vi); @@ -184,8 +188,6 @@ gst_tinyyolov2_class_init (GstTinyyolov2Class * klass) vi_class->stop = GST_DEBUG_FUNCPTR (gst_tinyyolov2_stop); vi_class->preprocess = GST_DEBUG_FUNCPTR (gst_tinyyolov2_preprocess); vi_class->postprocess = GST_DEBUG_FUNCPTR (gst_tinyyolov2_postprocess); - vi_class->postprocess_meta = - GST_DEBUG_FUNCPTR (gst_tinyyolov2_postprocess_meta); vi_class->inference_meta_info = gst_detection_meta_get_info (); } @@ -286,8 +288,25 @@ gst_tinyyolov2_preprocess (GstVideoInference * vi, static gboolean gst_tinyyolov2_postprocess (GstVideoInference * vi, const gpointer prediction, - gsize predsize, GstMeta * meta_model, GstVideoInfo * info_model, + gsize predsize, GstMeta * meta_model[2], GstVideoInfo * info_model, gboolean * valid_prediction) +{ + gboolean ret = TRUE; + + ret &= + gst_tinyyolov2_postprocess_old (vi, prediction, predsize, meta_model[0], + info_model, valid_prediction); + ret &= + gst_tinyyolov2_postprocess_new (vi, prediction, predsize, meta_model[1], + info_model, valid_prediction); + + return TRUE; +} + +static gboolean +gst_tinyyolov2_postprocess_old (GstVideoInference * vi, + const gpointer prediction, gsize predsize, GstMeta * meta_model, + GstVideoInfo * info_model, gboolean * valid_prediction) { GstTinyyolov2 *tinyyolov2; GstDetectionMeta *detect_meta = (GstDetectionMeta *) meta_model; @@ -307,7 +326,7 @@ gst_tinyyolov2_postprocess (GstVideoInference * vi, const gpointer prediction, } static gboolean -gst_tinyyolov2_postprocess_meta (GstVideoInference * vi, +gst_tinyyolov2_postprocess_new (GstVideoInference * vi, const gpointer prediction, gsize predsize, GstMeta * meta_model, GstVideoInfo * info_model, gboolean * valid_prediction) { diff --git a/ext/r2inference/gsttinyyolov3.c b/ext/r2inference/gsttinyyolov3.c index ff6378dc..08cad77f 100644 --- a/ext/r2inference/gsttinyyolov3.c +++ b/ext/r2inference/gsttinyyolov3.c @@ -73,7 +73,7 @@ static gboolean gst_tinyyolov3_preprocess (GstVideoInference * vi, GstVideoFrame * inframe, GstVideoFrame * outframe); static gboolean gst_tinyyolov3_postprocess (GstVideoInference * vi, const gpointer prediction, - gsize predsize, GstMeta * meta_model, GstVideoInfo * info_model, + gsize predsize, GstMeta * meta_model[2], GstVideoInfo * info_model, gboolean * valid_prediction); static gboolean gst_tinyyolov3_start (GstVideoInference * vi); static gboolean gst_tinyyolov3_stop (GstVideoInference * vi); @@ -246,14 +246,15 @@ gst_tinyyolov3_preprocess (GstVideoInference * vi, static gboolean gst_tinyyolov3_postprocess (GstVideoInference * vi, const gpointer prediction, - gsize predsize, GstMeta * meta_model, GstVideoInfo * info_model, + gsize predsize, GstMeta * meta_model[2], GstVideoInfo * info_model, gboolean * valid_prediction) { - GstTinyyolov3 *tinyyolov3; - GstDetectionMeta *detect_meta = (GstDetectionMeta *) meta_model; + GstTinyyolov3 *tinyyolov3 = GST_TINYYOLOV3 (vi); + GstDetectionMeta *detect_meta = (GstDetectionMeta *) meta_model[0]; + GST_LOG_OBJECT (vi, "Postprocess"); + detect_meta->num_boxes = 0; - tinyyolov3 = GST_TINYYOLOV3 (vi); gst_create_boxes_float (vi, prediction, valid_prediction, &detect_meta->boxes, &detect_meta->num_boxes, diff --git a/gst-libs/gst/r2inference/gstinferencemeta.c b/gst-libs/gst/r2inference/gstinferencemeta.c index d5a8b79a..2fdced2b 100644 --- a/gst-libs/gst/r2inference/gstinferencemeta.c +++ b/gst-libs/gst/r2inference/gstinferencemeta.c @@ -17,6 +17,8 @@ static gboolean gst_inference_meta_init (GstMeta * meta, gpointer params, GstBuffer * buffer); static void gst_inference_meta_free (GstMeta * meta, GstBuffer * buffer); +static gboolean gst_inference_meta_transform (GstBuffer * transbuf, + GstMeta * meta, GstBuffer * buffer, GQuark type, gpointer data); static gboolean gst_classification_meta_init (GstMeta * meta, gpointer params, GstBuffer * buffer); static void gst_classification_meta_free (GstMeta * meta, GstBuffer * buffer); @@ -61,7 +63,7 @@ gst_inference_meta_get_info (void) const GstMetaInfo *meta = gst_meta_register (GST_INFERENCE_META_API_TYPE, "GstInferenceMeta", sizeof (GstInferenceMeta), gst_inference_meta_init, gst_inference_meta_free, - NULL); + gst_inference_meta_transform); g_once_init_leave (&inference_meta_info, meta); } return inference_meta_info; @@ -186,6 +188,36 @@ gst_classification_meta_free (GstMeta * meta, GstBuffer * buffer) } } +static gboolean +gst_inference_meta_transform (GstBuffer * dest, GstMeta * meta, + GstBuffer * buffer, GQuark type, gpointer data) +{ + GstInferenceMeta *dmeta, *smeta; + + g_return_val_if_fail (dest, FALSE); + g_return_val_if_fail (meta, FALSE); + g_return_val_if_fail (buffer, FALSE); + + GST_LOG ("Transforming inference metadata"); + + if (!GST_META_TRANSFORM_IS_COPY (type)) { + GST_WARNING ("inference meta only supports copying as per now"); + } + + smeta = (GstInferenceMeta *) meta; + dmeta = + (GstInferenceMeta *) gst_buffer_add_meta (dest, GST_INFERENCE_META_INFO, + NULL); + if (!dmeta) { + GST_ERROR ("Unable to add meta to buffer"); + return FALSE; + } + + dmeta->prediction = gst_inference_prediction_copy (smeta->prediction); + + return TRUE; +} + static gboolean gst_detection_meta_init (GstMeta * meta, gpointer params, GstBuffer * buffer) { diff --git a/gst-libs/gst/r2inference/gstvideoinference.c b/gst-libs/gst/r2inference/gstvideoinference.c index d5c28d73..80b457c8 100644 --- a/gst-libs/gst/r2inference/gstvideoinference.c +++ b/gst-libs/gst/r2inference/gstvideoinference.c @@ -62,6 +62,10 @@ enum PROP_NEW_META }; +GQuark _size_quark; +GQuark _orientation_quark; +GQuark _scale_quark; +GQuark _copy_quark; typedef struct _GstVideoInferencePad GstVideoInferencePad; struct _GstVideoInferencePad @@ -242,6 +246,12 @@ gst_video_inference_class_init (GstVideoInferenceClass * klass) klass->stop = NULL; klass->preprocess = NULL; klass->postprocess = NULL; + + _size_quark = g_quark_from_static_string (GST_META_TAG_VIDEO_SIZE_STR); + _orientation_quark = + g_quark_from_static_string (GST_META_TAG_VIDEO_ORIENTATION_STR); + _scale_quark = gst_video_meta_transform_scale_get_quark (); + _copy_quark = g_quark_from_static_string ("gst-copy"); } static void @@ -787,7 +797,9 @@ video_inference_prepare_postprocess (const GstMetaInfo * meta_info, if (out_meta) { g_return_val_if_fail (gst_buffer_is_writable (buffer), FALSE); - *out_meta = gst_buffer_add_meta (buffer, meta_info, NULL); + out_meta[0] = gst_buffer_add_meta (buffer, meta_info, NULL); + out_meta[1] = + gst_buffer_add_meta (buffer, gst_inference_meta_get_info (), NULL); } flags = (GstMapFlags) (GST_MAP_READ | GST_VIDEO_FRAME_MAP_FLAG_NO_REF); @@ -822,11 +834,6 @@ video_inference_transform_meta (GstBuffer * buffer_model, { GstMeta *meta_bypass = NULL; const GstMetaInfo *info; - GQuark size_quark = g_quark_from_static_string (GST_META_TAG_VIDEO_SIZE_STR); - GQuark orientation_quark = - g_quark_from_static_string (GST_META_TAG_VIDEO_ORIENTATION_STR); - GQuark scale_quark = gst_video_meta_transform_scale_get_quark (); - GQuark copy_quark = g_quark_from_static_string ("gst-copy"); g_return_val_if_fail (buffer_model, NULL); g_return_val_if_fail (info_model, NULL); @@ -838,15 +845,15 @@ video_inference_transform_meta (GstBuffer * buffer_model, info = meta_model->info; - if (gst_meta_api_type_has_tag (info->api, size_quark) || - gst_meta_api_type_has_tag (info->api, orientation_quark)) { + if (gst_meta_api_type_has_tag (info->api, _size_quark) || + gst_meta_api_type_has_tag (info->api, _orientation_quark)) { GstVideoMetaTransform trans = { info_model, info_bypass }; info->transform_func (buffer_bypass, meta_model, buffer_model, - scale_quark, &trans); + _scale_quark, &trans); } else { info->transform_func (buffer_bypass, meta_model, buffer_model, - copy_quark, NULL); + _copy_quark, NULL); } meta_bypass = gst_buffer_get_meta (buffer_bypass, info->api); @@ -860,9 +867,8 @@ gst_video_inference_postprocess (GstVideoInference * self, GstVideoInferencePad * pad_model, GstBuffer * buffer_bypass, GstVideoInferencePad * pad_bypass) { - GstVideoInferencePrivate *priv = GST_VIDEO_INFERENCE_PRIVATE (self); - GstMeta *meta_model = NULL; - GstMeta *meta_bypass = NULL; + GstMeta *meta_model[2] = { NULL }; + GstMeta *meta_bypass[2] = { NULL }; GstVideoFrame frame_model; GstVideoFrame frame_bypass; GstVideoInfo *info_model = NULL; @@ -879,89 +885,46 @@ gst_video_inference_postprocess (GstVideoInference * self, info_model = &(pad_model->info); info_bypass = pad_bypass ? &(pad_bypass->info) : NULL; - /* This conditional contains some duplicated code - that will be removed once the old metas implementation - is removed. For now let's keep it in that way to - avoid breaking the old method. - */ - if (priv->use_new_meta) { - /* Subclass didn't implement a post-process, dont fail, just ignore */ - if (NULL == klass->postprocess_meta) { - return TRUE; - } - - meta_model = - gst_buffer_get_meta (buffer_model, gst_inference_meta_api_get_type ()); - if (!meta_model) { - /* Create New Inference Meta */ - meta_model = - gst_buffer_add_meta (buffer_model, priv->inference_meta_info, NULL); - } - - /* Prepare postprocess */ - if (!video_inference_prepare_postprocess (priv->inference_meta_info, - buffer_model, info_model, &frame_model, NULL)) { - return FALSE; - } - - if (!video_inference_prepare_postprocess (priv->inference_meta_info, - buffer_bypass, info_bypass, &frame_bypass, NULL)) { - return FALSE; - } - - /* Subclass Processing */ - if (!klass->postprocess_meta (self, prediction_data, prediction_size, - meta_model, info_model, &pred_valid)) { - GST_ELEMENT_ERROR (self, STREAM, FAILED, - ("Subclass failed at preprocess"), (NULL)); - return FALSE; - } - if (pred_valid) { - GstVideoFrame *pbpass = buffer_bypass ? &frame_bypass : NULL; - - /* TODO: Transfer model meta to bypass */ + /* Subclass didn't implement a post-process, dont fail, just ignore */ + if (NULL == klass->postprocess) { + return TRUE; + } - g_signal_emit (self, gst_video_inference_signals[NEW_INFERENCE_SIGNAL], 0, - meta_model, &frame_model, meta_bypass, pbpass); - } - } else { - /* Subclass didn't implement a post-process, dont fail, just ignore */ - if (NULL == klass->postprocess) { - return TRUE; - } + if (!video_inference_prepare_postprocess (klass->inference_meta_info, + buffer_model, info_model, &frame_model, meta_model)) { + return FALSE; + } - if (!video_inference_prepare_postprocess (klass->inference_meta_info, - buffer_model, info_model, &frame_model, &meta_model)) { - return FALSE; - } + if (!video_inference_prepare_postprocess (klass->inference_meta_info, + buffer_bypass, info_bypass, &frame_bypass, NULL)) { + return FALSE; + } - if (!video_inference_prepare_postprocess (klass->inference_meta_info, - buffer_bypass, info_bypass, &frame_bypass, NULL)) { - return FALSE; - } + GST_LOG_OBJECT (self, "Calling frame postprocess"); + if (!klass->postprocess (self, prediction_data, prediction_size, meta_model, + info_model, &pred_valid)) { + GST_ELEMENT_ERROR (self, STREAM, FAILED, + ("Subclass failed at postprocess"), (NULL)); + return FALSE; + } - GST_LOG_OBJECT (self, "Calling frame postprocess"); - if (!klass->postprocess (self, prediction_data, prediction_size, meta_model, - info_model, &pred_valid)) { - GST_ELEMENT_ERROR (self, STREAM, FAILED, - ("Subclass failed at preprocess"), (NULL)); - return FALSE; - } + if (pred_valid) { + GstVideoFrame *pbpass = buffer_bypass ? &frame_bypass : NULL; - if (pred_valid) { - GstVideoFrame *pbpass = buffer_bypass ? &frame_bypass : NULL; + meta_bypass[0] = + video_inference_transform_meta (buffer_model, info_model, meta_model[0], + buffer_bypass, info_bypass); - meta_bypass = - video_inference_transform_meta (buffer_model, info_model, meta_model, - buffer_bypass, info_bypass); - g_signal_emit (self, gst_video_inference_signals[NEW_PREDICTION_SIGNAL], - 0, meta_model, &frame_model, meta_bypass, pbpass); + meta_bypass[1] = + video_inference_transform_meta (buffer_model, info_model, meta_model[1], + buffer_bypass, info_bypass); - } else { - video_inference_remove_meta (buffer_model, meta_model); - video_inference_remove_meta (buffer_bypass, meta_bypass); - } + g_signal_emit (self, gst_video_inference_signals[NEW_PREDICTION_SIGNAL], + 0, meta_model[0], &frame_model, meta_bypass[0], pbpass); + } else { + video_inference_remove_meta (buffer_model, meta_model[0]); + video_inference_remove_meta (buffer_model, meta_model[1]); } video_inference_frame_unmap (buffer_model, &frame_model); @@ -1031,20 +994,6 @@ gst_video_inference_collected (GstCollectPads * pads, gpointer user_data) } if (buffer_model) { - if (priv->use_new_meta) { - GstMeta *meta_model = gst_buffer_get_meta (buffer_model, - gst_inference_meta_api_get_type ()); - if (meta_model) { - /* Check if root is enabled to be processed, if not, just forward buffer */ - GstInferenceMeta *inference_meta = (GstInferenceMeta *) meta_model; - GstInferencePrediction *root = inference_meta->prediction; - if (!root->enabled) { - GST_INFO_OBJECT (self, - "Current Prediction is not enabled, bypassing processing..."); - goto buffers_forward; - } - } - } /* Run preprocess and inference on the model and generate prediction */ if (!gst_video_inference_model_buffer_process (self, klass, priv, buffer_model, &prediction_data, &prediction_size)) { @@ -1061,7 +1010,6 @@ gst_video_inference_collected (GstCollectPads * pads, gpointer user_data) } } -buffers_forward: /* Forward buffer to model src pad */ ret = gst_video_inference_forward_buffer (self, buffer_model, priv->src_model); diff --git a/gst-libs/gst/r2inference/gstvideoinference.h b/gst-libs/gst/r2inference/gstvideoinference.h index 006a439d..c5ce63ac 100644 --- a/gst-libs/gst/r2inference/gstvideoinference.h +++ b/gst-libs/gst/r2inference/gstvideoinference.h @@ -39,10 +39,7 @@ struct _GstVideoInferenceClass gboolean (*preprocess) (GstVideoInference * self, GstVideoFrame * inframe, GstVideoFrame * outframe); gboolean (*postprocess) (GstVideoInference * self, - const gpointer prediction, gsize size, GstMeta * meta_model, - GstVideoInfo * info_model, gboolean * valid_prediction); - gboolean (*postprocess_meta) (GstVideoInference * self, - const gpointer prediction, gsize size, GstMeta * meta_model, + const gpointer prediction, gsize size, GstMeta * meta_model[2], GstVideoInfo * info_model, gboolean * valid_prediction); const GstMetaInfo *inference_meta_info; From b338c7239f52a2aef93c29e80218a46afe60e460 Mon Sep 17 00:00:00 2001 From: Michael Gruner Date: Fri, 27 Dec 2019 15:48:37 -0600 Subject: [PATCH 053/188] Initialize top prediction's bbox with image size --- gst-libs/gst/r2inference/gstvideoinference.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/gst-libs/gst/r2inference/gstvideoinference.c b/gst-libs/gst/r2inference/gstvideoinference.c index 80b457c8..644130b3 100644 --- a/gst-libs/gst/r2inference/gstvideoinference.c +++ b/gst-libs/gst/r2inference/gstvideoinference.c @@ -796,10 +796,16 @@ video_inference_prepare_postprocess (const GstMetaInfo * meta_info, } if (out_meta) { + GstInferenceMeta *imeta = NULL; + g_return_val_if_fail (gst_buffer_is_writable (buffer), FALSE); out_meta[0] = gst_buffer_add_meta (buffer, meta_info, NULL); out_meta[1] = gst_buffer_add_meta (buffer, gst_inference_meta_get_info (), NULL); + + imeta = (GstInferenceMeta *) out_meta[1]; + imeta->prediction->bbox.width = video_info->width; + imeta->prediction->bbox.height = video_info->height; } flags = (GstMapFlags) (GST_MAP_READ | GST_VIDEO_FRAME_MAP_FLAG_NO_REF); From f4f7e0b684d0907ba79c9e41205e7e070222b77a Mon Sep 17 00:00:00 2001 From: Michael Gruner Date: Fri, 27 Dec 2019 17:01:38 -0600 Subject: [PATCH 054/188] Implement inference prediction scaling --- gst-libs/gst/r2inference/gstinferencemeta.c | 25 ++++-- .../gst/r2inference/gstinferenceprediction.c | 77 +++++++++++++++++++ .../gst/r2inference/gstinferenceprediction.h | 3 + 3 files changed, 99 insertions(+), 6 deletions(-) diff --git a/gst-libs/gst/r2inference/gstinferencemeta.c b/gst-libs/gst/r2inference/gstinferencemeta.c index 2fdced2b..46d74387 100644 --- a/gst-libs/gst/r2inference/gstinferencemeta.c +++ b/gst-libs/gst/r2inference/gstinferencemeta.c @@ -200,10 +200,6 @@ gst_inference_meta_transform (GstBuffer * dest, GstMeta * meta, GST_LOG ("Transforming inference metadata"); - if (!GST_META_TRANSFORM_IS_COPY (type)) { - GST_WARNING ("inference meta only supports copying as per now"); - } - smeta = (GstInferenceMeta *) meta; dmeta = (GstInferenceMeta *) gst_buffer_add_meta (dest, GST_INFERENCE_META_INFO, @@ -213,9 +209,26 @@ gst_inference_meta_transform (GstBuffer * dest, GstMeta * meta, return FALSE; } - dmeta->prediction = gst_inference_prediction_copy (smeta->prediction); + gst_inference_prediction_unref (dmeta->prediction); - return TRUE; + if (GST_META_TRANSFORM_IS_COPY (type)) { + GST_LOG ("Copy detection metadata"); + + dmeta->prediction = gst_inference_prediction_copy (smeta->prediction); + return TRUE; + } + + if (GST_VIDEO_META_TRANSFORM_IS_SCALE (type)) { + GstVideoMetaTransform *trans = (GstVideoMetaTransform *) data; + dmeta->prediction = + gst_inference_prediction_scale (smeta->prediction, trans->out_info, + trans->in_info); + return TRUE; + } + + /* Invalid transformation */ + gst_buffer_remove_meta (dest, (GstMeta *) dmeta); + return FALSE; } static gboolean diff --git a/gst-libs/gst/r2inference/gstinferenceprediction.c b/gst-libs/gst/r2inference/gstinferenceprediction.c index 44ae86df..1ae661c9 100644 --- a/gst-libs/gst/r2inference/gstinferenceprediction.c +++ b/gst-libs/gst/r2inference/gstinferenceprediction.c @@ -23,6 +23,8 @@ static gchar *prediction_children_to_string (GstInferencePrediction * self, gint level); static gchar *prediction_classes_to_string (GstInferencePrediction * self, gint level); +static GstInferencePrediction *prediction_scale (const GstInferencePrediction * + self, GstVideoInfo * to, GstVideoInfo * from); static GstInferenceClassification * classification_copy (GstInferenceClassification * from, gpointer data); @@ -32,8 +34,16 @@ static gchar *bounding_box_to_string (BoundingBox * bbox, gint level); static void node_get_children (GNode * node, gpointer data); static gpointer node_copy (gconstpointer node, gpointer data); +static gpointer node_scale (gconstpointer node, gpointer data); static gboolean node_assign (GNode * node, gpointer data); +typedef struct _PredictionScaleData PredictionScaleData; +struct _PredictionScaleData +{ + GstVideoInfo *from; + GstVideoInfo *to; +}; + GstInferencePrediction * gst_inference_prediction_new (void) { @@ -336,3 +346,70 @@ gst_inference_prediction_append_classification (GstInferencePrediction * self, self->classifications = g_list_append (self->classifications, c); } + +static GstInferencePrediction * +prediction_scale (const GstInferencePrediction * self, GstVideoInfo * to, + GstVideoInfo * from) +{ + GstInferencePrediction *dest = NULL; + gint fw, fh, tw, th; + gdouble hfactor, vfactor; + + g_return_val_if_fail (self, NULL); + g_return_val_if_fail (to, NULL); + g_return_val_if_fail (from, NULL); + + dest = gst_inference_prediction_new (); + + fw = GST_VIDEO_INFO_WIDTH (from); + tw = GST_VIDEO_INFO_WIDTH (to); + fh = GST_VIDEO_INFO_HEIGHT (from); + th = GST_VIDEO_INFO_HEIGHT (to); + + g_return_val_if_fail (fw, FALSE); + g_return_val_if_fail (fh, FALSE); + + hfactor = tw * 1.0 / fw; + vfactor = th * 1.0 / fh; + + dest->bbox.x = self->bbox.x * hfactor; + dest->bbox.y = self->bbox.y * vfactor; + + dest->bbox.width = self->bbox.width * hfactor; + dest->bbox.height = self->bbox.height * vfactor; + + GST_LOG ("scaled bbox: %dx%d@%dx%d -> %dx%d@%dx%d", + self->bbox.x, self->bbox.y, self->bbox.width, + self->bbox.height, dest->bbox.x, dest->bbox.y, + dest->bbox.width, dest->bbox.height); + + return dest; +} + +static gpointer +node_scale (gconstpointer node, gpointer data) +{ + const GstInferencePrediction *self = (GstInferencePrediction *) node; + PredictionScaleData *sdata = (PredictionScaleData *) data; + + return prediction_scale (self, sdata->to, sdata->from); +} + +GstInferencePrediction * +gst_inference_prediction_scale (GstInferencePrediction * self, + GstVideoInfo * to, GstVideoInfo * from) +{ + GNode *other = NULL; + PredictionScaleData data = {.from = from,.to = to }; + + g_return_val_if_fail (self, NULL); + g_return_val_if_fail (to, NULL); + g_return_val_if_fail (from, NULL); + + other = g_node_copy_deep (self->predictions, node_scale, &data); + + /* Now finish assigning the nodes to the predictions */ + g_node_traverse (other, G_IN_ORDER, G_TRAVERSE_ALL, -1, node_assign, NULL); + + return (GstInferencePrediction *) other->data; +} diff --git a/gst-libs/gst/r2inference/gstinferenceprediction.h b/gst-libs/gst/r2inference/gstinferenceprediction.h index 970ffa34..453a4857 100644 --- a/gst-libs/gst/r2inference/gstinferenceprediction.h +++ b/gst-libs/gst/r2inference/gstinferenceprediction.h @@ -13,6 +13,7 @@ #define __GST_INFERENCE_PREDICTION__ #include +#include G_BEGIN_DECLS @@ -55,6 +56,8 @@ void gst_inference_prediction_append (GstInferencePrediction * self, GstInferenc GSList * gst_inference_prediction_get_children (GstInferencePrediction * self); void gst_inference_prediction_append_classification (GstInferencePrediction * self, GstInferenceClassification * c); +GstInferencePrediction * gst_inference_prediction_scale (GstInferencePrediction * self, + GstVideoInfo * to, GstVideoInfo * from); G_END_DECLS From d8ecacfb0924b0ce67dd1a0cc5564e061a17198b Mon Sep 17 00:00:00 2001 From: Michael Gruner Date: Mon, 6 Jan 2020 08:59:16 -0600 Subject: [PATCH 055/188] Assign a unique ID to each prediction --- .../gst/r2inference/gstinferenceprediction.c | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/gst-libs/gst/r2inference/gstinferenceprediction.c b/gst-libs/gst/r2inference/gstinferenceprediction.c index 1ae661c9..bf635a52 100644 --- a/gst-libs/gst/r2inference/gstinferenceprediction.c +++ b/gst-libs/gst/r2inference/gstinferenceprediction.c @@ -37,6 +37,8 @@ static gpointer node_copy (gconstpointer node, gpointer data); static gpointer node_scale (gconstpointer node, gpointer data); static gboolean node_assign (GNode * node, gpointer data); +static guint64 get_new_id (void); + typedef struct _PredictionScaleData PredictionScaleData; struct _PredictionScaleData { @@ -44,6 +46,21 @@ struct _PredictionScaleData GstVideoInfo *to; }; +static guint64 +get_new_id (void) +{ + static guint64 _id = 0; + static GMutex _id_mutex; + static guint64 ret = 0; + + g_mutex_lock (&_id_mutex); + ret = _id++; + g_mutex_unlock (&_id_mutex); + + return ret; +} + + GstInferencePrediction * gst_inference_prediction_new (void) { @@ -304,7 +321,7 @@ prediction_reset (GstInferencePrediction * self) { g_return_if_fail (self); - self->id = 0; + self->id = get_new_id (); self->enabled = FALSE; bounding_box_reset (&self->bbox); From 40f4d99d3257bb4558aa4b3b0e8f1d6ff5ebb5d3 Mon Sep 17 00:00:00 2001 From: Michael Gruner Date: Mon, 6 Jan 2020 09:17:01 -0600 Subject: [PATCH 056/188] Remove unused new-meta property --- gst-libs/gst/r2inference/gstvideoinference.c | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/gst-libs/gst/r2inference/gstvideoinference.c b/gst-libs/gst/r2inference/gstvideoinference.c index 644130b3..0883e63d 100644 --- a/gst-libs/gst/r2inference/gstvideoinference.c +++ b/gst-libs/gst/r2inference/gstvideoinference.c @@ -45,7 +45,6 @@ GST_DEBUG_CATEGORY_STATIC (gst_video_inference_debug_category); #define GST_CAT_DEFAULT gst_video_inference_debug_category #define DEFAULT_MODEL_LOCATION NULL -#define DEFAULT_NEW_META FALSE enum { @@ -59,7 +58,6 @@ enum PROP_0, PROP_BACKEND, PROP_MODEL_LOCATION, - PROP_NEW_META }; GQuark _size_quark; @@ -91,7 +89,6 @@ struct _GstVideoInferencePrivate GstBackend *backend; gchar *model_location; - gboolean use_new_meta; }; /* GObject methods */ @@ -227,11 +224,6 @@ gst_video_inference_class_init (GstVideoInferenceClass * klass) "Path to the model to use", DEFAULT_MODEL_LOCATION, G_PARAM_READWRITE)); - g_object_class_install_property (oclass, PROP_NEW_META, - g_param_spec_boolean ("new-meta", "Use new Meta", - "Use new inference meta instead of old metas implementation", - DEFAULT_NEW_META, G_PARAM_READWRITE)); - gst_video_inference_signals[NEW_PREDICTION_SIGNAL] = g_signal_new ("new-prediction", G_TYPE_FROM_CLASS (klass), G_SIGNAL_RUN_FIRST, 0, NULL, NULL, NULL, G_TYPE_NONE, 4, G_TYPE_POINTER, @@ -267,7 +259,6 @@ gst_video_inference_init (GstVideoInference * self) priv->sink_model = NULL; priv->src_model = NULL; priv->inference_meta_info = gst_inference_meta_get_info (); - priv->use_new_meta = DEFAULT_NEW_META; priv->cpads = gst_collect_pads_new (); gst_collect_pads_set_function (priv->cpads, gst_video_inference_collected, @@ -310,11 +301,6 @@ gst_video_inference_set_property (GObject * object, } GST_OBJECT_UNLOCK (self); break; - case PROP_NEW_META: - GST_OBJECT_LOCK (self); - priv->use_new_meta = g_value_get_boolean (value); - GST_OBJECT_UNLOCK (self); - break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); break; @@ -337,9 +323,6 @@ gst_video_inference_get_property (GObject * object, case PROP_MODEL_LOCATION: g_value_set_string (value, priv->model_location); break; - case PROP_NEW_META: - g_value_set_boolean (value, priv->use_new_meta); - break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); break; From e0454a7381de83a101ee279b9ecdcdb4cd5d8351 Mon Sep 17 00:00:00 2001 From: Michael Gruner Date: Wed, 8 Jan 2020 12:00:30 -0600 Subject: [PATCH 057/188] Document prediction and classification modules --- .../r2inference/gstinferenceclassification.h | 85 +++++++++++- .../gst/r2inference/gstinferenceprediction.h | 123 +++++++++++++++++- 2 files changed, 204 insertions(+), 4 deletions(-) diff --git a/gst-libs/gst/r2inference/gstinferenceclassification.h b/gst-libs/gst/r2inference/gstinferenceclassification.h index 388f990c..167acfa3 100644 --- a/gst-libs/gst/r2inference/gstinferenceclassification.h +++ b/gst-libs/gst/r2inference/gstinferenceclassification.h @@ -17,13 +17,24 @@ G_BEGIN_DECLS /** - * Classification to be assigned to an object + * GstInferenceClassification: + * @class_id: the numerical id associated to the assigned class + * @class_prob: the resulting probability of the assigned + * class. Typically between 0 and 1 + * @class_label: the label associated to this class or NULL if not + * available + * @num_classes: the amount of classes of the entire prediction + * @probabilities: the entire array of probabilities of the prediction + * @labels: the entire array of labels of the prediction or NULL if + * not available */ typedef struct _GstInferenceClassification GstInferenceClassification; struct _GstInferenceClassification { + /**/ GstMiniObject base; + /**/ gint class_id; gdouble class_prob; gchar *class_label; @@ -32,15 +43,85 @@ struct _GstInferenceClassification gchar **labels; }; +/** + * gst_inference_classification_new: + * + * Creates a new GstInferenceClassification. + * + * Returns: A newly allocated and initialized GstInferenceClassification. + */ GstInferenceClassification * gst_inference_classification_new (void); + +/** + * gst_inference_classification_new_full: + * @class_id: the numerical id associated to the assigned class + * @class_prob: the resulting probability of the assigned + * class. Typically between 0 and 1 + * @class_label: the label associated to this class or NULL if not + * available. A copy of the label is made if available. + * @num_classes: the amount of classes of the entire prediction + * @probabilities: the entire array of probabilities of the + * prediction. A copy of the array is made. + * @labels: the entire array of labels of the prediction or NULL if + * not available. A copy is made, if available. + * + * Creates a new GstInferenceClassification and assigns its members. + * + * Returns: A newly allocated and initialized GstInferenceClassification. + */ GstInferenceClassification * gst_inference_classification_new_full (gint class_id, gdouble class_prob, const gchar * class_label, gint num_classes, const gdouble * probabilities, gchar ** labels); + +/** + * gst_inference_classification_reset: + * @self: the classification to reset + * + * Clears a classification, effectively freeing all associated memory. + */ void gst_inference_classification_reset (GstInferenceClassification * self); + +/** + * gst_inference_classification_copy: + * @self: the classification to copy + * + * Copies a classification into a newly allocated one. This is a deep + * copy, meaning that all arrays are copied as well. No pointers are + * shared. + * + * Returns: a newly allocated copy of the original classification + */ GstInferenceClassification * gst_inference_classification_copy (const GstInferenceClassification * self); -void gst_inference_classification_free (GstInferenceClassification * self); + +/** + * gst_inference_classification_ref: + * @self: the classification to ref + * + * Increase the reference counter of the classification. + * + * Returns: the same classification, for convenience purposes. + */ GstInferenceClassification * gst_inference_classification_ref (GstInferenceClassification * self); + +/** + * gst_inference_classification_unref: + * @self: the classification to unref + * + * Decreases the reference counter of the classification. When the + * reference counter hits zero, the classification is freed. + */ void gst_inference_classification_unref (GstInferenceClassification * self); + +/** + * gst_inference_classification_to_string: + * @self: the classification to serialize + * + * Serializes the classification into a JSON-like string. The full + * arrays are not included. Free this string after usage using + * g_free() + * + * Returns: a string representing the classification. + */ gchar * gst_inference_classification_to_string (GstInferenceClassification * self, gint level); G_END_DECLS diff --git a/gst-libs/gst/r2inference/gstinferenceprediction.h b/gst-libs/gst/r2inference/gstinferenceprediction.h index 453a4857..b75b3b2f 100644 --- a/gst-libs/gst/r2inference/gstinferenceprediction.h +++ b/gst-libs/gst/r2inference/gstinferenceprediction.h @@ -21,7 +21,15 @@ typedef struct _BoundingBox BoundingBox; typedef struct _GstInferencePrediction GstInferencePrediction; /** - * Size and coordinates of an image region + * BoundingBox: + * @x: horizontal coordinate of the upper left position of the + * bounding box in pixels + * @y: vertical coordinate of the upper left position of the bounding + * box in pixels + * @width: width of the bounding box in pixels + * @height: height of the bounding box in pixels + * + * Size and coordinates of a prediction region */ struct _BoundingBox { @@ -32,12 +40,26 @@ struct _BoundingBox }; /** + * GstInferencePrediction: + * @id: unique id for this specific prediction + * @enabled: flag indicating wether or not this prediction should be + * used for further inference + * @bbox: the BoundingBox for this specific prediction + * @classifications: a linked list of GstInfereferenceClassification + * associated to this prediction + * @predictions: a n-ary tree of child predictions within this + * specific prediction. It is recommended to to access the tree + * directly, but to use this module's API to interact with the + * children. + * * Abstraction that represents a prediction */ struct _GstInferencePrediction { + /**/ GstMiniObject base; + /**/ guint64 id; gboolean enabled; BoundingBox bbox; @@ -45,17 +67,114 @@ struct _GstInferencePrediction GNode * predictions; }; +/** + * gst_inference_prediction_new: + * + * Creates a new GstInferencePrediction. + * + * Returns: A newly allocated and initialized GstInferencePrediction. + */ GstInferencePrediction * gst_inference_prediction_new (void); + +/** + * gst_inference_prediction_reset: + * @self: the prediction to reset + * + * Clears a prediction, effectively removing al children and resetting + * all members. + */ void gst_inference_prediction_reset (GstInferencePrediction * self); + +/** + * gst_inference_prediction_copy: + * @self: the prediction to copy + * + * Copies a prediction into a newly allocated one. This is a deep + * copy, meaning that all children and classifications are copied as + * well. No references are shared. + * + * Returns: a newly allocated copy of the original prediction + */ GstInferencePrediction * gst_inference_prediction_copy (const GstInferencePrediction * self); -void gst_inference_prediction_free (GstInferencePrediction * self); + +/** + * gst_inference_prediction_ref: + * @self: the prediction to ref + * + * Increase the reference counter of the prediction. + * + * Returns: the same prediction, for convenience purposes. + */ GstInferencePrediction * gst_inference_prediction_ref (GstInferencePrediction * self); + +/** + * gst_inference_prediction_unref: + * @self: the prediction to unref + * + * Decreases the reference counter of the prediction. When the + * reference counter hits zero, the prediction is freed. + */ void gst_inference_prediction_unref (GstInferencePrediction * self); + +/** + * gst_inference_prediction_to_string: + * @self: the prediction to serialize + * + * Serializes the prediction along with it's classifications and + * children into a JSON-like string. Free this string after usage + * using g_free() + * + * Returns: a string representing the prediction. + */ gchar * gst_inference_prediction_to_string (GstInferencePrediction * self); + +/** + * gst_inference_prediction_append: + * @self: the parent prediction + * @child: the prediction to append as a child + * + * Append a new prediction as part of the parent prediction + * children. The parent takes ownership, use + * gst_inference_prediction_ref() if you wish to keep a reference. + */ void gst_inference_prediction_append (GstInferencePrediction * self, GstInferencePrediction * child); + +/** + * gst_inference_prediction_get_children: + * @self: the parent prediction + * + * Gets a list of the immediate children of the current prediction. In + * other words, the children of these childrens are not returned. The + * references of these children are still owned by the parent. + * + * Returns: A linked list of the child predictions. + */ GSList * gst_inference_prediction_get_children (GstInferencePrediction * self); + +/** + * gst_inference_prediction_append_classification: + * @self: the parent prediction + * @c: the classification to append to the prediction + * + * A new GstInferenceClassification to associate with this + * prediction. The prediction takes ownership of the classification + */ void gst_inference_prediction_append_classification (GstInferencePrediction * self, GstInferenceClassification * c); + +/** + * gst_inference_prediction_scale: + * @self: the original prediction + * @to: the resulting image size + * @from: the original image size + * + * Modifies the BoundingBox associated with this prediction (and all + * its children) to scale to the new image size. This is typically + * used by the GstMeta subsystem automatically and not for public + * usage. + * + * Returns: a newly allocated and scaled prediction. + */ GstInferencePrediction * gst_inference_prediction_scale (GstInferencePrediction * self, GstVideoInfo * to, GstVideoInfo * from); From f07d4b47d67a92e4c2ae7281bee11d6d45807c7e Mon Sep 17 00:00:00 2001 From: Michael Gruner Date: Wed, 8 Jan 2020 12:02:20 -0600 Subject: [PATCH 058/188] Improve id variable naming --- gst-libs/gst/r2inference/gstinferenceprediction.c | 6 +++--- gst-libs/gst/r2inference/gstinferenceprediction.h | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/gst-libs/gst/r2inference/gstinferenceprediction.c b/gst-libs/gst/r2inference/gstinferenceprediction.c index bf635a52..d10978ec 100644 --- a/gst-libs/gst/r2inference/gstinferenceprediction.c +++ b/gst-libs/gst/r2inference/gstinferenceprediction.c @@ -121,7 +121,7 @@ prediction_copy (const GstInferencePrediction * self) other = gst_inference_prediction_new (); - other->id = self->id; + other->prediction_id = self->prediction_id; other->enabled = self->enabled; other->bbox = self->bbox; @@ -257,7 +257,7 @@ prediction_to_string (GstInferencePrediction * self, gint level) "%*s %s\n" "%*s ]\n" "%*s}", - indent, "", self->id, + indent, "", self->prediction_id, indent, "", self->enabled ? "True" : "False", indent, "", bbox, indent, "", indent, "", classes, indent, "", @@ -321,7 +321,7 @@ prediction_reset (GstInferencePrediction * self) { g_return_if_fail (self); - self->id = get_new_id (); + self->prediction_id = get_new_id (); self->enabled = FALSE; bounding_box_reset (&self->bbox); diff --git a/gst-libs/gst/r2inference/gstinferenceprediction.h b/gst-libs/gst/r2inference/gstinferenceprediction.h index b75b3b2f..82f6f699 100644 --- a/gst-libs/gst/r2inference/gstinferenceprediction.h +++ b/gst-libs/gst/r2inference/gstinferenceprediction.h @@ -41,7 +41,7 @@ struct _BoundingBox /** * GstInferencePrediction: - * @id: unique id for this specific prediction + * @prediction_id: unique id for this specific prediction * @enabled: flag indicating wether or not this prediction should be * used for further inference * @bbox: the BoundingBox for this specific prediction @@ -60,7 +60,7 @@ struct _GstInferencePrediction GstMiniObject base; /**/ - guint64 id; + guint64 prediction_id; gboolean enabled; BoundingBox bbox; GList * classifications; From 92d80c7dec7bf779a30db4bac0b9fd24f477eec7 Mon Sep 17 00:00:00 2001 From: Michael Gruner Date: Wed, 8 Jan 2020 12:08:02 -0600 Subject: [PATCH 059/188] Use named constants instead of magic numbers --- .../r2inference/gstinferenceclassification.c | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/gst-libs/gst/r2inference/gstinferenceclassification.c b/gst-libs/gst/r2inference/gstinferenceclassification.c index 1c4bb9f9..364077b9 100644 --- a/gst-libs/gst/r2inference/gstinferenceclassification.c +++ b/gst-libs/gst/r2inference/gstinferenceclassification.c @@ -11,6 +11,13 @@ #include "gstinferenceclassification.h" +#define DEFAULT_CLASS_ID -1 +#define DEFAULT_CLASS_PROB 0.0f +#define DEFAULT_CLASS_LABEL NULL +#define DEFAULT_NUM_CLASSES 0 +#define DEFAULT_PROBABILITIES NULL +#define DEFAULT_LABELS NULL + static GType gst_inference_classification_get_type (void); GST_DEFINE_MINI_OBJECT_TYPE (GstInferenceClassification, gst_inference_classification); @@ -25,24 +32,24 @@ classification_reset (GstInferenceClassification * self) { g_return_if_fail (self); - self->class_id = -1; - self->class_prob = 0.0f; - self->num_classes = 0; + self->class_id = DEFAULT_CLASS_ID; + self->class_prob = DEFAULT_CLASS_PROB; + self->num_classes = DEFAULT_NUM_CLASSES; if (self->class_label) { g_free (self->class_label); } - self->class_label = NULL; + self->class_label = DEFAULT_CLASS_LABEL; if (self->probabilities) { g_free (self->probabilities); } - self->probabilities = NULL; + self->probabilities = DEFAULT_PROBABILITIES; if (self->labels) { g_strfreev (self->labels); } - self->labels = NULL; + self->labels = DEFAULT_LABELS; } GstInferenceClassification * From 21fd543ef199460226e95878cdb9224c63b752e3 Mon Sep 17 00:00:00 2001 From: Michael Gruner Date: Wed, 8 Jan 2020 12:17:53 -0600 Subject: [PATCH 060/188] Add missing pointer guards --- ext/r2inference/gsttinyyolov2.c | 9 +++++++++ gst-libs/gst/r2inference/gstinferenceprediction.c | 6 ++++++ 2 files changed, 15 insertions(+) diff --git a/ext/r2inference/gsttinyyolov2.c b/ext/r2inference/gsttinyyolov2.c index 9b59141e..877f2290 100644 --- a/ext/r2inference/gsttinyyolov2.c +++ b/ext/r2inference/gsttinyyolov2.c @@ -293,6 +293,12 @@ gst_tinyyolov2_postprocess (GstVideoInference * vi, const gpointer prediction, { gboolean ret = TRUE; + g_return_val_if_fail (vi, FALSE); + g_return_val_if_fail (prediction, FALSE); + g_return_val_if_fail (meta_model, FALSE); + g_return_val_if_fail (info_model, FALSE); + g_return_val_if_fail (valid_prediction, FALSE); + ret &= gst_tinyyolov2_postprocess_old (vi, prediction, predsize, meta_model[0], info_model, valid_prediction); @@ -310,6 +316,9 @@ gst_tinyyolov2_postprocess_old (GstVideoInference * vi, { GstTinyyolov2 *tinyyolov2; GstDetectionMeta *detect_meta = (GstDetectionMeta *) meta_model; + + g_return_val_if_fail (detect_meta, FALSE); + GST_LOG_OBJECT (vi, "Postprocess"); detect_meta->num_boxes = 0; tinyyolov2 = GST_TINYYOLOV2 (vi); diff --git a/gst-libs/gst/r2inference/gstinferenceprediction.c b/gst-libs/gst/r2inference/gstinferenceprediction.c index d10978ec..a432b54a 100644 --- a/gst-libs/gst/r2inference/gstinferenceprediction.c +++ b/gst-libs/gst/r2inference/gstinferenceprediction.c @@ -137,6 +137,9 @@ node_copy (gconstpointer node, gpointer data) { GstInferencePrediction *self = (GstInferencePrediction *) node; + g_return_val_if_fail (node, NULL); + g_return_val_if_fail (data, NULL); + return prediction_copy (self); } @@ -145,6 +148,9 @@ node_assign (GNode * node, gpointer data) { GstInferencePrediction *pred = (GstInferencePrediction *) node->data; + g_return_val_if_fail (node, FALSE); + g_return_val_if_fail (data, FALSE); + pred->predictions = node; return FALSE; From e5ee453eaeac10101a9e7212bf80e3fd357eb264 Mon Sep 17 00:00:00 2001 From: Michael Gruner Date: Thu, 9 Jan 2020 11:19:49 -0600 Subject: [PATCH 061/188] Print 64 bit integers in a portable way --- gst-libs/gst/r2inference/gstinferenceprediction.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gst-libs/gst/r2inference/gstinferenceprediction.c b/gst-libs/gst/r2inference/gstinferenceprediction.c index a432b54a..897e6258 100644 --- a/gst-libs/gst/r2inference/gstinferenceprediction.c +++ b/gst-libs/gst/r2inference/gstinferenceprediction.c @@ -253,7 +253,7 @@ prediction_to_string (GstInferencePrediction * self, gint level) children = prediction_children_to_string (self, level + 1); prediction = g_strdup_printf ("{\n" - "%*s id : %llu,\n" + "%*s id : %" G_GUINT64_FORMAT ",\n" "%*s enabled : %s,\n" "%*s bbox : %s,\n" "%*s classes : [\n" From ab1e8c581b73cd9637c3061b27ef12eaedadf7df Mon Sep 17 00:00:00 2001 From: rrcarlosrodriguez Date: Fri, 17 Jan 2020 09:23:19 -0600 Subject: [PATCH 062/188] Add missing header dependency definition --- gst-libs/gst/r2inference/gstinferenceclassification.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gst-libs/gst/r2inference/gstinferenceclassification.c b/gst-libs/gst/r2inference/gstinferenceclassification.c index 364077b9..2887ce33 100644 --- a/gst-libs/gst/r2inference/gstinferenceclassification.c +++ b/gst-libs/gst/r2inference/gstinferenceclassification.c @@ -11,6 +11,8 @@ #include "gstinferenceclassification.h" +#include + #define DEFAULT_CLASS_ID -1 #define DEFAULT_CLASS_PROB 0.0f #define DEFAULT_CLASS_LABEL NULL From 2a23318ec96565228911393343318cd23fb41073 Mon Sep 17 00:00:00 2001 From: rrcarlosrodriguez Date: Fri, 17 Jan 2020 10:42:15 -0600 Subject: [PATCH 063/188] Update License to LGPL --- COPYING | 13 -------- configure.ac | 2 +- ext/opencv/gstclassificationoverlay.cc | 3 +- ext/opencv/gstclassificationoverlay.h | 2 +- ext/opencv/gstdetectionoverlay.cc | 3 +- ext/opencv/gstdetectionoverlay.h | 2 +- ext/opencv/gstembeddingoverlay.cc | 3 +- ext/opencv/gstembeddingoverlay.h | 2 +- ext/opencv/gstinferenceoverlay.c | 2 +- ext/r2inference/gstfacenetv1.c | 2 +- ext/r2inference/gstfacenetv1.h | 2 +- ext/r2inference/gstinceptionv1.c | 2 +- ext/r2inference/gstinceptionv1.h | 2 +- ext/r2inference/gstinceptionv2.c | 2 +- ext/r2inference/gstinceptionv2.h | 2 +- ext/r2inference/gstinceptionv3.c | 2 +- ext/r2inference/gstinceptionv3.h | 2 +- ext/r2inference/gstinceptionv4.c | 2 +- ext/r2inference/gstinceptionv4.h | 2 +- ext/r2inference/gstinference.c | 2 +- ext/r2inference/gstmobilenetv2.c | 2 +- ext/r2inference/gstmobilenetv2.h | 2 +- ext/r2inference/gstresnet50v1.c | 2 +- ext/r2inference/gstresnet50v1.h | 2 +- ext/r2inference/gsttinyyolov2.c | 2 +- ext/r2inference/gsttinyyolov2.h | 2 +- ext/r2inference/gsttinyyolov3.c | 2 +- ext/r2inference/gsttinyyolov3.h | 2 +- gst-libs/gst/opencv/gstinferenceoverlay.c | 2 +- gst-libs/gst/opencv/gstinferenceoverlay.h | 2 +- gst-libs/gst/r2inference/gstbackend.cc | 2 +- gst-libs/gst/r2inference/gstbackend.h | 2 +- gst-libs/gst/r2inference/gstbackendsubclass.h | 3 +- gst-libs/gst/r2inference/gstchildinspector.c | 2 +- gst-libs/gst/r2inference/gstchildinspector.h | 2 +- .../gst/r2inference/gstinferencebackends.cc | 2 +- .../gst/r2inference/gstinferencebackends.h | 2 +- .../r2inference/gstinferenceclassification.c | 28 +++++++++++------ .../r2inference/gstinferenceclassification.h | 28 +++++++++++------ gst-libs/gst/r2inference/gstinferencedebug.c | 3 +- gst-libs/gst/r2inference/gstinferencedebug.h | 3 +- gst-libs/gst/r2inference/gstinferencemeta.c | 28 +++++++++++------ gst-libs/gst/r2inference/gstinferencemeta.h | 28 +++++++++++------ .../gst/r2inference/gstinferencepostprocess.c | 3 +- .../gst/r2inference/gstinferencepostprocess.h | 2 +- .../gst/r2inference/gstinferenceprediction.c | 28 +++++++++++------ .../gst/r2inference/gstinferenceprediction.h | 30 ++++++++++++------- .../gst/r2inference/gstinferencepreprocess.c | 3 +- .../gst/r2inference/gstinferencepreprocess.h | 2 +- gst-libs/gst/r2inference/gstncsdk.cc | 2 +- gst-libs/gst/r2inference/gstncsdk.h | 2 +- gst-libs/gst/r2inference/gsttensorflow.cc | 2 +- gst-libs/gst/r2inference/gsttensorflow.h | 2 +- gst-libs/gst/r2inference/gstvideoinference.c | 2 +- gst-libs/gst/r2inference/gstvideoinference.h | 2 +- gst/inferencecrop/cropelement.cc | 2 +- gst/inferencecrop/cropelement.h | 2 +- gst/inferencecrop/gstdetectioncrop.cc | 2 +- gst/inferencecrop/gstdetectioncrop.h | 2 +- gst/inferencecrop/videocrop.cc | 2 +- gst/inferencecrop/videocrop.h | 2 +- .../process/preprocess_functions_utils.c | 2 +- .../process/preprocess_functions_utils.h | 3 +- ...st_gst_fill_classification_meta_function.c | 3 +- .../process/test_gst_normalize_function.c | 3 +- .../test_gst_pixel_to_float_function.c | 3 +- .../process/test_gst_subtract_mean_function.c | 3 +- tests/examples/classification/customlogic.cc | 27 +++++++++++------ tests/examples/classification/customlogic.h | 27 +++++++++++------ .../classification/gstclassification.c | 27 +++++++++++------ tests/examples/detection/customlogic.cc | 27 +++++++++++------ tests/examples/detection/customlogic.h | 27 +++++++++++------ tests/examples/detection/gstdetection.c | 27 +++++++++++------ tests/examples/embedding/customlogic.cc | 27 +++++++++++------ tests/examples/embedding/customlogic.h | 27 +++++++++++------ tests/examples/embedding/gstembedding.c | 27 +++++++++++------ 76 files changed, 350 insertions(+), 209 deletions(-) diff --git a/COPYING b/COPYING index ee391442..fb6e92e3 100644 --- a/COPYING +++ b/COPYING @@ -1,16 +1,3 @@ -GstInference copyright (C) 2019 RidgeRun LLC - -This GStreamer plug-in is free software; you can redistribute it -and/or modify it under the terms of the GNU Lesser General Public -License version 2.1 as published by the Free Software Foundation. - -This GStreamer plug-in is distributed in the hope that it will be -useful, but WITHOUT ANY WARRANTY; without even the implied -warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -See the GNU Lesser General Public License below for more details. - ----------------------------------------------------------------------- - GNU LESSER GENERAL PUBLIC LICENSE Version 2.1, February 1999 diff --git a/configure.ac b/configure.ac index f85dc696..674a1464 100644 --- a/configure.ac +++ b/configure.ac @@ -195,7 +195,7 @@ AC_ARG_ENABLE(Bsymbolic, dnl *** set variables based on configure arguments *** dnl set license and copyright notice -GST_LICENSE="Proprietary" +GST_LICENSE="LGPL" AC_DEFINE_UNQUOTED(GST_LICENSE, "$GST_LICENSE", [GStreamer Inference license]) AC_SUBST(GST_LICENSE) diff --git a/ext/opencv/gstclassificationoverlay.cc b/ext/opencv/gstclassificationoverlay.cc index 456a49a0..6c6dd7dc 100644 --- a/ext/opencv/gstclassificationoverlay.cc +++ b/ext/opencv/gstclassificationoverlay.cc @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2018 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public @@ -18,6 +18,7 @@ * Boston, MA 02111-1307, USA. * */ + #ifdef HAVE_CONFIG_H #include "config.h" #endif diff --git a/ext/opencv/gstclassificationoverlay.h b/ext/opencv/gstclassificationoverlay.h index 214554cd..7fd8f759 100644 --- a/ext/opencv/gstclassificationoverlay.h +++ b/ext/opencv/gstclassificationoverlay.h @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2019 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public diff --git a/ext/opencv/gstdetectionoverlay.cc b/ext/opencv/gstdetectionoverlay.cc index 164a1bbb..b68784e1 100644 --- a/ext/opencv/gstdetectionoverlay.cc +++ b/ext/opencv/gstdetectionoverlay.cc @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2018 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public @@ -18,6 +18,7 @@ * Boston, MA 02111-1307, USA. * */ + #ifdef HAVE_CONFIG_H #include "config.h" #endif diff --git a/ext/opencv/gstdetectionoverlay.h b/ext/opencv/gstdetectionoverlay.h index ee1b8629..61587274 100644 --- a/ext/opencv/gstdetectionoverlay.h +++ b/ext/opencv/gstdetectionoverlay.h @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2019 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public diff --git a/ext/opencv/gstembeddingoverlay.cc b/ext/opencv/gstembeddingoverlay.cc index 2e9f3f93..56933d74 100644 --- a/ext/opencv/gstembeddingoverlay.cc +++ b/ext/opencv/gstembeddingoverlay.cc @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2019 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public @@ -18,6 +18,7 @@ * Boston, MA 02111-1307, USA. * */ + #ifdef HAVE_CONFIG_H #include "config.h" #endif diff --git a/ext/opencv/gstembeddingoverlay.h b/ext/opencv/gstembeddingoverlay.h index 4915435d..a1751149 100644 --- a/ext/opencv/gstembeddingoverlay.h +++ b/ext/opencv/gstembeddingoverlay.h @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2019 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public diff --git a/ext/opencv/gstinferenceoverlay.c b/ext/opencv/gstinferenceoverlay.c index 52b8129c..47ad56cc 100644 --- a/ext/opencv/gstinferenceoverlay.c +++ b/ext/opencv/gstinferenceoverlay.c @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2019 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public diff --git a/ext/r2inference/gstfacenetv1.c b/ext/r2inference/gstfacenetv1.c index aa413e42..903ec654 100644 --- a/ext/r2inference/gstfacenetv1.c +++ b/ext/r2inference/gstfacenetv1.c @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2019 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public diff --git a/ext/r2inference/gstfacenetv1.h b/ext/r2inference/gstfacenetv1.h index 7ed9cb05..c91989b8 100644 --- a/ext/r2inference/gstfacenetv1.h +++ b/ext/r2inference/gstfacenetv1.h @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2019 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public diff --git a/ext/r2inference/gstinceptionv1.c b/ext/r2inference/gstinceptionv1.c index a362af8d..0be9e5e2 100644 --- a/ext/r2inference/gstinceptionv1.c +++ b/ext/r2inference/gstinceptionv1.c @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2019 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public diff --git a/ext/r2inference/gstinceptionv1.h b/ext/r2inference/gstinceptionv1.h index 412143ae..6018f30a 100644 --- a/ext/r2inference/gstinceptionv1.h +++ b/ext/r2inference/gstinceptionv1.h @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2018 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public diff --git a/ext/r2inference/gstinceptionv2.c b/ext/r2inference/gstinceptionv2.c index 5c8bd285..b0e42b99 100644 --- a/ext/r2inference/gstinceptionv2.c +++ b/ext/r2inference/gstinceptionv2.c @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2019 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public diff --git a/ext/r2inference/gstinceptionv2.h b/ext/r2inference/gstinceptionv2.h index ba4aa4dc..8b444e79 100644 --- a/ext/r2inference/gstinceptionv2.h +++ b/ext/r2inference/gstinceptionv2.h @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2018 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public diff --git a/ext/r2inference/gstinceptionv3.c b/ext/r2inference/gstinceptionv3.c index 2a10cf1f..8201ed2a 100644 --- a/ext/r2inference/gstinceptionv3.c +++ b/ext/r2inference/gstinceptionv3.c @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2019 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public diff --git a/ext/r2inference/gstinceptionv3.h b/ext/r2inference/gstinceptionv3.h index 1978acc0..e504624b 100644 --- a/ext/r2inference/gstinceptionv3.h +++ b/ext/r2inference/gstinceptionv3.h @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2019 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public diff --git a/ext/r2inference/gstinceptionv4.c b/ext/r2inference/gstinceptionv4.c index 28471ea5..74e1d791 100644 --- a/ext/r2inference/gstinceptionv4.c +++ b/ext/r2inference/gstinceptionv4.c @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2019 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public diff --git a/ext/r2inference/gstinceptionv4.h b/ext/r2inference/gstinceptionv4.h index f4726f9b..f5a4a450 100644 --- a/ext/r2inference/gstinceptionv4.h +++ b/ext/r2inference/gstinceptionv4.h @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2019 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public diff --git a/ext/r2inference/gstinference.c b/ext/r2inference/gstinference.c index d53be1d8..1422cb96 100644 --- a/ext/r2inference/gstinference.c +++ b/ext/r2inference/gstinference.c @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2019 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public diff --git a/ext/r2inference/gstmobilenetv2.c b/ext/r2inference/gstmobilenetv2.c index d78a1187..412e099a 100644 --- a/ext/r2inference/gstmobilenetv2.c +++ b/ext/r2inference/gstmobilenetv2.c @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2019 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public diff --git a/ext/r2inference/gstmobilenetv2.h b/ext/r2inference/gstmobilenetv2.h index ce78f9be..6e2d24cd 100644 --- a/ext/r2inference/gstmobilenetv2.h +++ b/ext/r2inference/gstmobilenetv2.h @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2018 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public diff --git a/ext/r2inference/gstresnet50v1.c b/ext/r2inference/gstresnet50v1.c index dd24d2d1..60aabf41 100644 --- a/ext/r2inference/gstresnet50v1.c +++ b/ext/r2inference/gstresnet50v1.c @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2019 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public diff --git a/ext/r2inference/gstresnet50v1.h b/ext/r2inference/gstresnet50v1.h index 83390c58..9cb41a01 100644 --- a/ext/r2inference/gstresnet50v1.h +++ b/ext/r2inference/gstresnet50v1.h @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2019 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public diff --git a/ext/r2inference/gsttinyyolov2.c b/ext/r2inference/gsttinyyolov2.c index 877f2290..c97855c5 100644 --- a/ext/r2inference/gsttinyyolov2.c +++ b/ext/r2inference/gsttinyyolov2.c @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2018 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public diff --git a/ext/r2inference/gsttinyyolov2.h b/ext/r2inference/gsttinyyolov2.h index f67904b2..434f1740 100644 --- a/ext/r2inference/gsttinyyolov2.h +++ b/ext/r2inference/gsttinyyolov2.h @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2018 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public diff --git a/ext/r2inference/gsttinyyolov3.c b/ext/r2inference/gsttinyyolov3.c index 08cad77f..05f7dfab 100644 --- a/ext/r2inference/gsttinyyolov3.c +++ b/ext/r2inference/gsttinyyolov3.c @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2019 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public diff --git a/ext/r2inference/gsttinyyolov3.h b/ext/r2inference/gsttinyyolov3.h index 08f3328e..ab7b087a 100644 --- a/ext/r2inference/gsttinyyolov3.h +++ b/ext/r2inference/gsttinyyolov3.h @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2019 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public diff --git a/gst-libs/gst/opencv/gstinferenceoverlay.c b/gst-libs/gst/opencv/gstinferenceoverlay.c index 6c6bfae3..ca079bf4 100644 --- a/gst-libs/gst/opencv/gstinferenceoverlay.c +++ b/gst-libs/gst/opencv/gstinferenceoverlay.c @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2019 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public diff --git a/gst-libs/gst/opencv/gstinferenceoverlay.h b/gst-libs/gst/opencv/gstinferenceoverlay.h index b586bc6c..11362b5e 100644 --- a/gst-libs/gst/opencv/gstinferenceoverlay.h +++ b/gst-libs/gst/opencv/gstinferenceoverlay.h @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2019 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public diff --git a/gst-libs/gst/r2inference/gstbackend.cc b/gst-libs/gst/r2inference/gstbackend.cc index 48f088db..7c84517c 100644 --- a/gst-libs/gst/r2inference/gstbackend.cc +++ b/gst-libs/gst/r2inference/gstbackend.cc @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2018 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public diff --git a/gst-libs/gst/r2inference/gstbackend.h b/gst-libs/gst/r2inference/gstbackend.h index 933bb21f..eb354813 100644 --- a/gst-libs/gst/r2inference/gstbackend.h +++ b/gst-libs/gst/r2inference/gstbackend.h @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2018 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public diff --git a/gst-libs/gst/r2inference/gstbackendsubclass.h b/gst-libs/gst/r2inference/gstbackendsubclass.h index 37aef953..8cef2dac 100644 --- a/gst-libs/gst/r2inference/gstbackendsubclass.h +++ b/gst-libs/gst/r2inference/gstbackendsubclass.h @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2019 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public @@ -18,6 +18,7 @@ * Boston, MA 02111-1307, USA. * */ + #ifndef __GST_BACKENDSUBCLASS_H__ #define __GST_BACKENDSUBCLASS_H__ diff --git a/gst-libs/gst/r2inference/gstchildinspector.c b/gst-libs/gst/r2inference/gstchildinspector.c index 932c69fd..280ba639 100644 --- a/gst-libs/gst/r2inference/gstchildinspector.c +++ b/gst-libs/gst/r2inference/gstchildinspector.c @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2018 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public diff --git a/gst-libs/gst/r2inference/gstchildinspector.h b/gst-libs/gst/r2inference/gstchildinspector.h index 28392601..6b0166d3 100644 --- a/gst-libs/gst/r2inference/gstchildinspector.h +++ b/gst-libs/gst/r2inference/gstchildinspector.h @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2018 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public diff --git a/gst-libs/gst/r2inference/gstinferencebackends.cc b/gst-libs/gst/r2inference/gstinferencebackends.cc index 324d23c9..d171f172 100644 --- a/gst-libs/gst/r2inference/gstinferencebackends.cc +++ b/gst-libs/gst/r2inference/gstinferencebackends.cc @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2018 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public diff --git a/gst-libs/gst/r2inference/gstinferencebackends.h b/gst-libs/gst/r2inference/gstinferencebackends.h index 8d8bc8c9..da3ab634 100644 --- a/gst-libs/gst/r2inference/gstinferencebackends.h +++ b/gst-libs/gst/r2inference/gstinferencebackends.h @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2018 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public diff --git a/gst-libs/gst/r2inference/gstinferenceclassification.c b/gst-libs/gst/r2inference/gstinferenceclassification.c index 2887ce33..d72d208f 100644 --- a/gst-libs/gst/r2inference/gstinferenceclassification.c +++ b/gst-libs/gst/r2inference/gstinferenceclassification.c @@ -1,13 +1,23 @@ -/* Copyright (C) 2019 RidgeRun, LLC (http://www.ridgerun.com) - * All Rights Reserved. +/* + * GStreamer + * Copyright (C) 2018-2020 RidgeRun * - * The contents of this software are proprietary and confidential to RidgeRun, - * LLC. No part of this program may be photocopied, reproduced or translated - * into another programming language without prior written consent of - * RidgeRun, LLC. The user is free to modify the source code after obtaining - * a software license from RidgeRun. All source code changes must be provided - * back to RidgeRun without any encumbrance. -*/ + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + */ #include "gstinferenceclassification.h" diff --git a/gst-libs/gst/r2inference/gstinferenceclassification.h b/gst-libs/gst/r2inference/gstinferenceclassification.h index 167acfa3..19f8ad6b 100644 --- a/gst-libs/gst/r2inference/gstinferenceclassification.h +++ b/gst-libs/gst/r2inference/gstinferenceclassification.h @@ -1,13 +1,23 @@ -/* Copyright (C) 2019 RidgeRun, LLC (http://www.ridgerun.com) - * All Rights Reserved. +/* + * GStreamer + * Copyright (C) 2018-2020 RidgeRun * - * The contents of this software are proprietary and confidential to RidgeRun, - * LLC. No part of this program may be photocopied, reproduced or translated - * into another programming language without prior written consent of - * RidgeRun, LLC. The user is free to modify the source code after obtaining - * a software license from RidgeRun. All source code changes must be provided - * back to RidgeRun without any encumbrance. -*/ + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + */ #ifndef __GST_INFERENCE_CLASSIFICATION__ #define __GST_INFERENCE_CLASSIFICATION__ diff --git a/gst-libs/gst/r2inference/gstinferencedebug.c b/gst-libs/gst/r2inference/gstinferencedebug.c index 2a0cfdeb..35970970 100644 --- a/gst-libs/gst/r2inference/gstinferencedebug.c +++ b/gst-libs/gst/r2inference/gstinferencedebug.c @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2019 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public @@ -18,6 +18,7 @@ * Boston, MA 02111-1307, USA. * */ + #include "gstinferencedebug.h" void diff --git a/gst-libs/gst/r2inference/gstinferencedebug.h b/gst-libs/gst/r2inference/gstinferencedebug.h index b34ace39..9c88d745 100644 --- a/gst-libs/gst/r2inference/gstinferencedebug.h +++ b/gst-libs/gst/r2inference/gstinferencedebug.h @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2019 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public @@ -18,6 +18,7 @@ * Boston, MA 02111-1307, USA. * */ + #ifndef GST_INFERENCE_DEBUG_H #define GST_INFERENCE_DEBUG_H diff --git a/gst-libs/gst/r2inference/gstinferencemeta.c b/gst-libs/gst/r2inference/gstinferencemeta.c index 46d74387..8811a275 100644 --- a/gst-libs/gst/r2inference/gstinferencemeta.c +++ b/gst-libs/gst/r2inference/gstinferencemeta.c @@ -1,13 +1,23 @@ -/* Copyright (C) 2019 RidgeRun, LLC (http://www.ridgerun.com) - * All Rights Reserved. +/* + * GStreamer + * Copyright (C) 2018-2020 RidgeRun * - * The contents of this software are proprietary and confidential to RidgeRun, - * LLC. No part of this program may be photocopied, reproduced or translated - * into another programming language without prior written consent of - * RidgeRun, LLC. The user is free to modify the source code after obtaining - * a software license from RidgeRun. All source code changes must be provided - * back to RidgeRun without any encumbrance. -*/ + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + */ #include "gstinferencemeta.h" diff --git a/gst-libs/gst/r2inference/gstinferencemeta.h b/gst-libs/gst/r2inference/gstinferencemeta.h index 415ed180..7d23f707 100644 --- a/gst-libs/gst/r2inference/gstinferencemeta.h +++ b/gst-libs/gst/r2inference/gstinferencemeta.h @@ -1,13 +1,23 @@ -/* Copyright (C) 2019 RidgeRun, LLC (http://www.ridgerun.com) - * All Rights Reserved. +/* + * GStreamer + * Copyright (C) 2018-2020 RidgeRun * - * The contents of this software are proprietary and confidential to RidgeRun, - * LLC. No part of this program may be photocopied, reproduced or translated - * into another programming language without prior written consent of - * RidgeRun, LLC. The user is free to modify the source code after obtaining - * a software license from RidgeRun. All source code changes must be provided - * back to RidgeRun without any encumbrance. -*/ + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + */ #ifndef GST_INFERENCE_META_H #define GST_INFERENCE_META_H diff --git a/gst-libs/gst/r2inference/gstinferencepostprocess.c b/gst-libs/gst/r2inference/gstinferencepostprocess.c index 27a7d8a1..90a3bc40 100644 --- a/gst-libs/gst/r2inference/gstinferencepostprocess.c +++ b/gst-libs/gst/r2inference/gstinferencepostprocess.c @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2019 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public @@ -18,6 +18,7 @@ * Boston, MA 02111-1307, USA. * */ + #include "gstinferencepostprocess.h" #include #include diff --git a/gst-libs/gst/r2inference/gstinferencepostprocess.h b/gst-libs/gst/r2inference/gstinferencepostprocess.h index fbcf126f..a3db89b5 100644 --- a/gst-libs/gst/r2inference/gstinferencepostprocess.h +++ b/gst-libs/gst/r2inference/gstinferencepostprocess.h @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2019 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public diff --git a/gst-libs/gst/r2inference/gstinferenceprediction.c b/gst-libs/gst/r2inference/gstinferenceprediction.c index 897e6258..e1334cd4 100644 --- a/gst-libs/gst/r2inference/gstinferenceprediction.c +++ b/gst-libs/gst/r2inference/gstinferenceprediction.c @@ -1,13 +1,23 @@ -/* Copyright (C) 2019 RidgeRun, LLC (http://www.ridgerun.com) - * All Rights Reserved. +/* + * GStreamer + * Copyright (C) 2018-2020 RidgeRun * - * The contents of this software are proprietary and confidential to RidgeRun, - * LLC. No part of this program may be photocopied, reproduced or translated - * into another programming language without prior written consent of - * RidgeRun, LLC. The user is free to modify the source code after obtaining - * a software license from RidgeRun. All source code changes must be provided - * back to RidgeRun without any encumbrance. -*/ + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + */ #include "gstinferenceprediction.h" diff --git a/gst-libs/gst/r2inference/gstinferenceprediction.h b/gst-libs/gst/r2inference/gstinferenceprediction.h index 82f6f699..c0262a60 100644 --- a/gst-libs/gst/r2inference/gstinferenceprediction.h +++ b/gst-libs/gst/r2inference/gstinferenceprediction.h @@ -1,13 +1,23 @@ -/* Copyright (C) 2019 RidgeRun, LLC (http://www.ridgerun.com) - * All Rights Reserved. - * - * The contents of this software are proprietary and confidential to RidgeRun, - * LLC. No part of this program may be photocopied, reproduced or translated - * into another programming language without prior written consent of - * RidgeRun, LLC. The user is free to modify the source code after obtaining - * a software license from RidgeRun. All source code changes must be provided - * back to RidgeRun without any encumbrance. -*/ +/* + * GStreamer + * Copyright (C) 2018-2020 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + */ #ifndef __GST_INFERENCE_PREDICTION__ #define __GST_INFERENCE_PREDICTION__ diff --git a/gst-libs/gst/r2inference/gstinferencepreprocess.c b/gst-libs/gst/r2inference/gstinferencepreprocess.c index 2f78bf08..0312f3d5 100644 --- a/gst-libs/gst/r2inference/gstinferencepreprocess.c +++ b/gst-libs/gst/r2inference/gstinferencepreprocess.c @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2019 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public @@ -18,6 +18,7 @@ * Boston, MA 02111-1307, USA. * */ + #include "gstinferencepreprocess.h" #include diff --git a/gst-libs/gst/r2inference/gstinferencepreprocess.h b/gst-libs/gst/r2inference/gstinferencepreprocess.h index 6337e611..a186f5f8 100644 --- a/gst-libs/gst/r2inference/gstinferencepreprocess.h +++ b/gst-libs/gst/r2inference/gstinferencepreprocess.h @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2019 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public diff --git a/gst-libs/gst/r2inference/gstncsdk.cc b/gst-libs/gst/r2inference/gstncsdk.cc index ec797872..cc2eff89 100644 --- a/gst-libs/gst/r2inference/gstncsdk.cc +++ b/gst-libs/gst/r2inference/gstncsdk.cc @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2018 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public diff --git a/gst-libs/gst/r2inference/gstncsdk.h b/gst-libs/gst/r2inference/gstncsdk.h index 08689b68..17b0ff02 100644 --- a/gst-libs/gst/r2inference/gstncsdk.h +++ b/gst-libs/gst/r2inference/gstncsdk.h @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2018 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public diff --git a/gst-libs/gst/r2inference/gsttensorflow.cc b/gst-libs/gst/r2inference/gsttensorflow.cc index 889cedf7..f00eeeb6 100644 --- a/gst-libs/gst/r2inference/gsttensorflow.cc +++ b/gst-libs/gst/r2inference/gsttensorflow.cc @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2019 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public diff --git a/gst-libs/gst/r2inference/gsttensorflow.h b/gst-libs/gst/r2inference/gsttensorflow.h index 793eb6dc..a8594457 100644 --- a/gst-libs/gst/r2inference/gsttensorflow.h +++ b/gst-libs/gst/r2inference/gsttensorflow.h @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2019 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public diff --git a/gst-libs/gst/r2inference/gstvideoinference.c b/gst-libs/gst/r2inference/gstvideoinference.c index 0883e63d..184ef795 100644 --- a/gst-libs/gst/r2inference/gstvideoinference.c +++ b/gst-libs/gst/r2inference/gstvideoinference.c @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2018 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public diff --git a/gst-libs/gst/r2inference/gstvideoinference.h b/gst-libs/gst/r2inference/gstvideoinference.h index c5ce63ac..f216d376 100644 --- a/gst-libs/gst/r2inference/gstvideoinference.h +++ b/gst-libs/gst/r2inference/gstvideoinference.h @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2018 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public diff --git a/gst/inferencecrop/cropelement.cc b/gst/inferencecrop/cropelement.cc index fd3d3640..325d6a89 100644 --- a/gst/inferencecrop/cropelement.cc +++ b/gst/inferencecrop/cropelement.cc @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2019 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public diff --git a/gst/inferencecrop/cropelement.h b/gst/inferencecrop/cropelement.h index ee43a50d..7fbcf561 100644 --- a/gst/inferencecrop/cropelement.h +++ b/gst/inferencecrop/cropelement.h @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2019 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public diff --git a/gst/inferencecrop/gstdetectioncrop.cc b/gst/inferencecrop/gstdetectioncrop.cc index 5873a2d6..f0daaa25 100644 --- a/gst/inferencecrop/gstdetectioncrop.cc +++ b/gst/inferencecrop/gstdetectioncrop.cc @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2019 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public diff --git a/gst/inferencecrop/gstdetectioncrop.h b/gst/inferencecrop/gstdetectioncrop.h index 5b812224..cb577819 100644 --- a/gst/inferencecrop/gstdetectioncrop.h +++ b/gst/inferencecrop/gstdetectioncrop.h @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2019 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public diff --git a/gst/inferencecrop/videocrop.cc b/gst/inferencecrop/videocrop.cc index f1e82e5c..db80e9ae 100644 --- a/gst/inferencecrop/videocrop.cc +++ b/gst/inferencecrop/videocrop.cc @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2019 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public diff --git a/gst/inferencecrop/videocrop.h b/gst/inferencecrop/videocrop.h index 76a94f84..7d71ed71 100644 --- a/gst/inferencecrop/videocrop.h +++ b/gst/inferencecrop/videocrop.h @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2019 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public diff --git a/tests/check/process/preprocess_functions_utils.c b/tests/check/process/preprocess_functions_utils.c index 99463064..f4cc1245 100644 --- a/tests/check/process/preprocess_functions_utils.c +++ b/tests/check/process/preprocess_functions_utils.c @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2019 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public diff --git a/tests/check/process/preprocess_functions_utils.h b/tests/check/process/preprocess_functions_utils.h index 2f257e5d..5e99ff1c 100644 --- a/tests/check/process/preprocess_functions_utils.h +++ b/tests/check/process/preprocess_functions_utils.h @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2019 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public @@ -18,6 +18,7 @@ * Boston, MA 02111-1307, USA. * */ + #ifndef __PREPROCESS_FUNCTIONS_UTILS_H__ #define __PREPROCESS_FUNCTIONS_UTILS_H__ diff --git a/tests/check/process/test_gst_fill_classification_meta_function.c b/tests/check/process/test_gst_fill_classification_meta_function.c index f3428aa1..8b245f45 100644 --- a/tests/check/process/test_gst_fill_classification_meta_function.c +++ b/tests/check/process/test_gst_fill_classification_meta_function.c @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2019 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public @@ -18,6 +18,7 @@ * Boston, MA 02111-1307, USA. * */ + #include #include "gst/r2inference/gstinferencepostprocess.h" #include "gst/r2inference/gstinferencemeta.h" diff --git a/tests/check/process/test_gst_normalize_function.c b/tests/check/process/test_gst_normalize_function.c index 83b6248d..bb51b7a8 100644 --- a/tests/check/process/test_gst_normalize_function.c +++ b/tests/check/process/test_gst_normalize_function.c @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2019 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public @@ -18,6 +18,7 @@ * Boston, MA 02111-1307, USA. * */ + #include #include "preprocess_functions_utils.c" #include "gst/r2inference/gstinferencepreprocess.h" diff --git a/tests/check/process/test_gst_pixel_to_float_function.c b/tests/check/process/test_gst_pixel_to_float_function.c index fe5cefac..002c8820 100644 --- a/tests/check/process/test_gst_pixel_to_float_function.c +++ b/tests/check/process/test_gst_pixel_to_float_function.c @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2019 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public @@ -18,6 +18,7 @@ * Boston, MA 02111-1307, USA. * */ + #include #include "preprocess_functions_utils.c" #include "gst/r2inference/gstinferencepreprocess.h" diff --git a/tests/check/process/test_gst_subtract_mean_function.c b/tests/check/process/test_gst_subtract_mean_function.c index f55a9fb7..dc5e8156 100644 --- a/tests/check/process/test_gst_subtract_mean_function.c +++ b/tests/check/process/test_gst_subtract_mean_function.c @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2019 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public @@ -18,6 +18,7 @@ * Boston, MA 02111-1307, USA. * */ + #include #include "preprocess_functions_utils.c" #include "gst/r2inference/gstinferencepreprocess.h" diff --git a/tests/examples/classification/customlogic.cc b/tests/examples/classification/customlogic.cc index 7ac6217b..c19343e0 100644 --- a/tests/examples/classification/customlogic.cc +++ b/tests/examples/classification/customlogic.cc @@ -1,13 +1,22 @@ -/* GstClassification - * Copyright (C) 2019 RidgeRun - * All Rights Reserved. +/* + * GStreamer + * Copyright (C) 2018-2020 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. * - *The contents of this software are proprietary and confidential to RidgeRun, - *LLC. No part of this program may be photocopied, reproduced or translated - *into another programming language without prior written consent of - *RidgeRun, LLC. The user is free to modify the source code after obtaining - *a software license from RidgeRun. All source code changes must be provided - *back to RidgeRun without any encumbrance. */ #include "customlogic.h" diff --git a/tests/examples/classification/customlogic.h b/tests/examples/classification/customlogic.h index a6a3fcb1..45f3f1f0 100644 --- a/tests/examples/classification/customlogic.h +++ b/tests/examples/classification/customlogic.h @@ -1,13 +1,22 @@ -/* GstClassification - * Copyright (C) 2019 RidgeRun - * All Rights Reserved. +/* + * GStreamer + * Copyright (C) 2018-2020 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. * - *The contents of this software are proprietary and confidential to RidgeRun, - *LLC. No part of this program may be photocopied, reproduced or translated - *into another programming language without prior written consent of - *RidgeRun, LLC. The user is free to modify the source code after obtaining - *a software license from RidgeRun. All source code changes must be provided - *back to RidgeRun without any encumbrance. */ #ifndef __CLASSIFICATION_CUSTOM_LOGIC__ diff --git a/tests/examples/classification/gstclassification.c b/tests/examples/classification/gstclassification.c index 211731b6..4b5ffbb0 100644 --- a/tests/examples/classification/gstclassification.c +++ b/tests/examples/classification/gstclassification.c @@ -1,13 +1,22 @@ -/* GstClassification - * Copyright (C) 2019 RidgeRun - * All Rights Reserved. +/* + * GStreamer + * Copyright (C) 2018-2020 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. * - *The contents of this software are proprietary and confidential to RidgeRun, - *LLC. No part of this program may be photocopied, reproduced or translated - *into another programming language without prior written consent of - *RidgeRun, LLC. The user is free to modify the source code after obtaining - *a software license from RidgeRun. All source code changes must be provided - *back to RidgeRun without any encumbrance. */ #ifdef HAVE_CONFIG_H diff --git a/tests/examples/detection/customlogic.cc b/tests/examples/detection/customlogic.cc index bf1b41d6..fa554cff 100644 --- a/tests/examples/detection/customlogic.cc +++ b/tests/examples/detection/customlogic.cc @@ -1,13 +1,22 @@ -/* GstClassification - * Copyright (C) 2019 RidgeRun - * All Rights Reserved. +/* + * GStreamer + * Copyright (C) 2018-2020 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. * - *The contents of this software are proprietary and confidential to RidgeRun, - *LLC. No part of this program may be photocopied, reproduced or translated - *into another programming language without prior written consent of - *RidgeRun, LLC. The user is free to modify the source code after obtaining - *a software license from RidgeRun. All source code changes must be provided - *back to RidgeRun without any encumbrance. */ #include "customlogic.h" diff --git a/tests/examples/detection/customlogic.h b/tests/examples/detection/customlogic.h index d7af047c..afeedbf4 100644 --- a/tests/examples/detection/customlogic.h +++ b/tests/examples/detection/customlogic.h @@ -1,13 +1,22 @@ -/* GstDetection - * Copyright (C) 2019 RidgeRun - * All Rights Reserved. +/* + * GStreamer + * Copyright (C) 2018-2020 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. * - *The contents of this software are proprietary and confidential to RidgeRun, - *LLC. No part of this program may be photocopied, reproduced or translated - *into another programming language without prior written consent of - *RidgeRun, LLC. The user is free to modify the source code after obtaining - *a software license from RidgeRun. All source code changes must be provided - *back to RidgeRun without any encumbrance. */ #ifndef __DETECTION_CUSTOM_LOGIC__ diff --git a/tests/examples/detection/gstdetection.c b/tests/examples/detection/gstdetection.c index cf24155b..793aaac7 100644 --- a/tests/examples/detection/gstdetection.c +++ b/tests/examples/detection/gstdetection.c @@ -1,13 +1,22 @@ -/* GstDetection - * Copyright (C) 2019 RidgeRun - * All Rights Reserved. +/* + * GStreamer + * Copyright (C) 2018-2020 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. * - *The contents of this software are proprietary and confidential to RidgeRun, - *LLC. No part of this program may be photocopied, reproduced or translated - *into another programming language without prior written consent of - *RidgeRun, LLC. The user is free to modify the source code after obtaining - *a software license from RidgeRun. All source code changes must be provided - *back to RidgeRun without any encumbrance. */ #ifdef HAVE_CONFIG_H diff --git a/tests/examples/embedding/customlogic.cc b/tests/examples/embedding/customlogic.cc index 4088cf5f..a26f7302 100644 --- a/tests/examples/embedding/customlogic.cc +++ b/tests/examples/embedding/customlogic.cc @@ -1,13 +1,22 @@ -/* GstEmbedding - * Copyright (C) 2019 RidgeRun - * All Rights Reserved. +/* + * GStreamer + * Copyright (C) 2018-2020 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. * - *The contents of this software are proprietary and confidential to RidgeRun, - *LLC. No part of this program may be photocopied, reproduced or translated - *into another programming language without prior written consent of - *RidgeRun, LLC. The user is free to modify the source code after obtaining - *a software license from RidgeRun. All source code changes must be provided - *back to RidgeRun without any encumbrance. */ #include "customlogic.h" diff --git a/tests/examples/embedding/customlogic.h b/tests/examples/embedding/customlogic.h index 2cbdf048..17b4a189 100644 --- a/tests/examples/embedding/customlogic.h +++ b/tests/examples/embedding/customlogic.h @@ -1,13 +1,22 @@ -/* GstEmbedding - * Copyright (C) 2019 RidgeRun - * All Rights Reserved. +/* + * GStreamer + * Copyright (C) 2018-2020 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. * - *The contents of this software are proprietary and confidential to RidgeRun, - *LLC. No part of this program may be photocopied, reproduced or translated - *into another programming language without prior written consent of - *RidgeRun, LLC. The user is free to modify the source code after obtaining - *a software license from RidgeRun. All source code changes must be provided - *back to RidgeRun without any encumbrance. */ #ifndef __EMBEDDING_CUSTOM_LOGIC__ diff --git a/tests/examples/embedding/gstembedding.c b/tests/examples/embedding/gstembedding.c index e17b7335..dbb9ff8c 100644 --- a/tests/examples/embedding/gstembedding.c +++ b/tests/examples/embedding/gstembedding.c @@ -1,13 +1,22 @@ -/* GstEmbedding - * Copyright (C) 2019 RidgeRun - * All Rights Reserved. +/* + * GStreamer + * Copyright (C) 2018-2020 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. * - *The contents of this software are proprietary and confidential to RidgeRun, - *LLC. No part of this program may be photocopied, reproduced or translated - *into another programming language without prior written consent of - *RidgeRun, LLC. The user is free to modify the source code after obtaining - *a software license from RidgeRun. All source code changes must be provided - *back to RidgeRun without any encumbrance. */ #ifdef HAVE_CONFIG_H From d6815e31b46d68f54224defe06b2d5c9d04f2221 Mon Sep 17 00:00:00 2001 From: c-trejos Date: Thu, 19 Dec 2019 11:23:53 -0600 Subject: [PATCH 064/188] Add filter class property --- gst/inferencefilter/gstinferencefilter.cc | 253 ++++++++++++++++++++++ gst/inferencefilter/gstinferencefilter.h | 34 +++ 2 files changed, 287 insertions(+) create mode 100644 gst/inferencefilter/gstinferencefilter.cc create mode 100644 gst/inferencefilter/gstinferencefilter.h diff --git a/gst/inferencefilter/gstinferencefilter.cc b/gst/inferencefilter/gstinferencefilter.cc new file mode 100644 index 00000000..96cdaca8 --- /dev/null +++ b/gst/inferencefilter/gstinferencefilter.cc @@ -0,0 +1,253 @@ +/* + * GStreamer + * Copyright (C) 2019 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + */ + +/** + * SECTION:element-gstinferencefilter + * + * The inferencefilter element modifies metadata on input buffer to select + * which prediction on the inference meta should be processed by modifying + * its enable property. + * + * + * Example launch line + * |[ + * gst-launch-1.0 v4l2src device=$CAMERA ! "video/x-raw, width=1280, height=720" ! videoconvert ! tee name=t \ + t. ! videoscale ! queue ! net.sink_model t. ! queue ! net.sink_bypass \ + tinyyolov2 name=net model-location=$MODEL_LOCATION backend=tensorflow backend::input-layer=$INPUT_LAYER \ + backend::output-layer=$OUTPUT_LAYER net.src_model ! inferencefilter filter-class=1 ! fakesink + * ]| + * + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "gstinferencefilter.h" + +#include +#include + + +GST_DEBUG_CATEGORY_STATIC (gst_inferencefilter_debug_category); +#define GST_CAT_DEFAULT gst_inferencefilter_debug_category + +#define GST_INFERENCEFILTER_PROPERTY_FLAGS (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS) +#define PROP_FILTER_CLASS_LABEL_DEFAULT -1 +#define PROP_FILTER_CLASS_LABEL_MIN -1 + +/* prototypes */ + +static void gst_inferencefilter_set_property (GObject * object, + guint property_id, const GValue * value, GParamSpec * pspec); +static void gst_inferencefilter_get_property (GObject * object, + guint property_id, GValue * value, GParamSpec * pspec); +static gboolean gst_inferencefilter_start (GstBaseTransform * trans); +static gboolean gst_inferencefilter_stop (GstBaseTransform * trans); +static void gst_inferencefilter_finalize (GObject * object); + +static gboolean gst_inferencefilter_transform_meta (GstBaseTransform * trans, + GstBuffer * outbuf, GstMeta * meta, GstBuffer * inbuf); +static GstFlowReturn gst_inferencefilter_transform_ip (GstBaseTransform * trans, + GstBuffer * buf); + +enum +{ + PROP_0, + PROP_FILTER_CLASS_LABEL, +}; + +struct _GstInferencefilter +{ + GstBaseTransform base_inferencefilter; + gint filter_class; +}; + + +/* pad templates */ + +static GstStaticPadTemplate gst_inferencefilter_src_template = +GST_STATIC_PAD_TEMPLATE ("src", + GST_PAD_SRC, + GST_PAD_ALWAYS, + GST_STATIC_CAPS_ANY + ); + +static GstStaticPadTemplate gst_inferencefilter_sink_template = +GST_STATIC_PAD_TEMPLATE ("sink", + GST_PAD_SINK, + GST_PAD_ALWAYS, + GST_STATIC_CAPS_ANY + ); + + +/* class initialization */ + +G_DEFINE_TYPE_WITH_CODE (GstInferencefilter, gst_inferencefilter, + GST_TYPE_BASE_TRANSFORM, + GST_DEBUG_CATEGORY_INIT (gst_inferencefilter_debug_category, + "inferencefilter", 0, "debug category for inferencefilter element")); + +static void +gst_inferencefilter_class_init (GstInferencefilterClass * klass) +{ + GObjectClass *gobject_class = G_OBJECT_CLASS (klass); + GstBaseTransformClass *base_transform_class = + GST_BASE_TRANSFORM_CLASS (klass); + + /* Setting up pads and setting metadata should be moved to + base_class_init if you intend to subclass this class. */ + gst_element_class_add_static_pad_template (GST_ELEMENT_CLASS (klass), + &gst_inferencefilter_src_template); + gst_element_class_add_static_pad_template (GST_ELEMENT_CLASS (klass), + &gst_inferencefilter_sink_template); + + gst_element_class_set_static_metadata (GST_ELEMENT_CLASS (klass), + "Inference Filter", "Generic", "Enables/disables specific classes contained on the inference metadata to be processed", + ""); + + gobject_class->set_property = gst_inferencefilter_set_property; + gobject_class->get_property = gst_inferencefilter_get_property; + + base_transform_class->start = GST_DEBUG_FUNCPTR (gst_inferencefilter_start); + base_transform_class->stop = GST_DEBUG_FUNCPTR (gst_inferencefilter_stop); + gobject_class->finalize = gst_inferencefilter_finalize; + + g_object_class_install_property (gobject_class, PROP_FILTER_CLASS_LABEL, + g_param_spec_int ("filter-class", "filter-class", "Filter class", PROP_FILTER_CLASS_LABEL_MIN, G_MAXINT, + PROP_FILTER_CLASS_LABEL_DEFAULT, GST_INFERENCEFILTER_PROPERTY_FLAGS)); + base_transform_class->transform_meta = + GST_DEBUG_FUNCPTR (gst_inferencefilter_transform_meta); + base_transform_class->transform_ip = + GST_DEBUG_FUNCPTR (gst_inferencefilter_transform_ip); + +} + +static void +gst_inferencefilter_init (GstInferencefilter * inferencefilter) +{ + inferencefilter->filter_class = -1; +} + +void +gst_inferencefilter_set_property (GObject * object, guint property_id, + const GValue * value, GParamSpec * pspec) +{ + GstInferencefilter *inferencefilter = GST_INFERENCEFILTER (object); + + GST_DEBUG_OBJECT (inferencefilter, "set_property"); + + switch (property_id) { + case PROP_FILTER_CLASS_LABEL: + GST_OBJECT_LOCK (inferencefilter); + inferencefilter->filter_class = g_value_get_int (value); + GST_OBJECT_UNLOCK(inferencefilter); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); + break; + } +} + +void +gst_inferencefilter_get_property (GObject * object, guint property_id, + GValue * value, GParamSpec * pspec) +{ + GstInferencefilter *inferencefilter = GST_INFERENCEFILTER (object); + + GST_DEBUG_OBJECT (inferencefilter, "get_property"); + + switch (property_id) { + case PROP_FILTER_CLASS_LABEL: + GST_OBJECT_LOCK (inferencefilter); + g_value_set_int (value, inferencefilter->filter_class); + GST_OBJECT_UNLOCK (inferencefilter); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); + break; + } +} + +/* states */ +static gboolean +gst_inferencefilter_start (GstBaseTransform * trans) +{ + GstInferencefilter *inferencefilter = GST_INFERENCEFILTER (trans); + + GST_DEBUG_OBJECT (inferencefilter, "start"); + + return TRUE; +} + +static gboolean +gst_inferencefilter_stop (GstBaseTransform * trans) +{ + GstInferencefilter *inferencefilter = GST_INFERENCEFILTER (trans); + + GST_DEBUG_OBJECT (inferencefilter, "stop"); + + return TRUE; +} + +void +gst_inferencefilter_finalize (GObject * object) +{ + GstInferencefilter *inferencefilter = GST_INFERENCEFILTER (object); + + GST_DEBUG_OBJECT (inferencefilter, "finalize"); + + G_OBJECT_CLASS (gst_inferencefilter_parent_class)->finalize (object); +} + + +static gboolean +gst_inferencefilter_transform_meta (GstBaseTransform * trans, + GstBuffer * outbuf, GstMeta * meta, GstBuffer * inbuf) +{ + GstInferencefilter *inferencefilter = GST_INFERENCEFILTER (trans); + GST_DEBUG_OBJECT (inferencefilter, "transform_ip"); + return TRUE; +} + +static GstFlowReturn +gst_inferencefilter_transform_ip (GstBaseTransform * trans, GstBuffer * buf) +{ + GstInferencefilter *inferencefilter = GST_INFERENCEFILTER (trans); + + GST_DEBUG_OBJECT (inferencefilter, "transform_ip"); + + return GST_FLOW_OK; +} + +static gboolean +plugin_init (GstPlugin * plugin) +{ + return gst_element_register (plugin, "inferencefilter", GST_RANK_NONE, + GST_TYPE_INFERENCEFILTER); +} + + +GST_PLUGIN_DEFINE (GST_VERSION_MAJOR, + GST_VERSION_MINOR, + inferencefilter, + "Enables/disables selected classes on inference meta to be processed", + plugin_init, VERSION, "LGPL", PACKAGE_NAME, GST_PACKAGE_ORIGIN) diff --git a/gst/inferencefilter/gstinferencefilter.h b/gst/inferencefilter/gstinferencefilter.h new file mode 100644 index 00000000..fc5f4520 --- /dev/null +++ b/gst/inferencefilter/gstinferencefilter.h @@ -0,0 +1,34 @@ +/* + * GStreamer + * Copyright (C) 2019 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + */ + + +#ifndef _GST_INFERENCEFILTER_H_ +#define _GST_INFERENCEFILTER_H_ + +#include + +G_BEGIN_DECLS +#define GST_TYPE_INFERENCEFILTER (gst_inferencefilter_get_type()) +G_DECLARE_FINAL_TYPE (GstInferencefilter, gst_inferencefilter, GST, + INFERENCEFILTER, GstBaseTransform) + +G_END_DECLS +#endif From 377a03823b41b15cf334b74d359f99421570f80e Mon Sep 17 00:00:00 2001 From: c-trejos Date: Thu, 19 Dec 2019 11:26:32 -0600 Subject: [PATCH 065/188] Add reset enable property --- gst/inferencefilter/gstinferencefilter.cc | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/gst/inferencefilter/gstinferencefilter.cc b/gst/inferencefilter/gstinferencefilter.cc index 96cdaca8..0425c6f4 100644 --- a/gst/inferencefilter/gstinferencefilter.cc +++ b/gst/inferencefilter/gstinferencefilter.cc @@ -53,6 +53,7 @@ GST_DEBUG_CATEGORY_STATIC (gst_inferencefilter_debug_category); #define GST_INFERENCEFILTER_PROPERTY_FLAGS (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS) #define PROP_FILTER_CLASS_LABEL_DEFAULT -1 #define PROP_FILTER_CLASS_LABEL_MIN -1 +#define PROP_RESET_ENABLE_DEFAULT FALSE /* prototypes */ @@ -73,12 +74,14 @@ enum { PROP_0, PROP_FILTER_CLASS_LABEL, + PROP_RESET_ENABLE, }; struct _GstInferencefilter { GstBaseTransform base_inferencefilter; gint filter_class; + gboolean reset_enable; }; @@ -133,7 +136,13 @@ gst_inferencefilter_class_init (GstInferencefilterClass * klass) g_object_class_install_property (gobject_class, PROP_FILTER_CLASS_LABEL, g_param_spec_int ("filter-class", "filter-class", "Filter class", PROP_FILTER_CLASS_LABEL_MIN, G_MAXINT, - PROP_FILTER_CLASS_LABEL_DEFAULT, GST_INFERENCEFILTER_PROPERTY_FLAGS)); + PROP_FILTER_CLASS_LABEL_DEFAULT, GST_INFERENCEFILTER_PROPERTY_FLAGS)); + g_object_class_install_property (gobject_class, + PROP_RESET_ENABLE, g_param_spec_boolean ("reset-enable", + "Reset enable", + "Enables all inference meta to be processed", + PROP_RESET_ENABLE_DEFAULT, GST_INFERENCEFILTER_PROPERTY_FLAGS)); + base_transform_class->transform_meta = GST_DEBUG_FUNCPTR (gst_inferencefilter_transform_meta); base_transform_class->transform_ip = @@ -145,6 +154,7 @@ static void gst_inferencefilter_init (GstInferencefilter * inferencefilter) { inferencefilter->filter_class = -1; + inferencefilter->reset_enable = FALSE; } void @@ -161,6 +171,11 @@ gst_inferencefilter_set_property (GObject * object, guint property_id, inferencefilter->filter_class = g_value_get_int (value); GST_OBJECT_UNLOCK(inferencefilter); break; + case PROP_RESET_ENABLE: + GST_OBJECT_LOCK (inferencefilter); + inferencefilter->reset_enable = g_value_get_boolean (value); + GST_OBJECT_UNLOCK(inferencefilter); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); break; @@ -181,6 +196,11 @@ gst_inferencefilter_get_property (GObject * object, guint property_id, g_value_set_int (value, inferencefilter->filter_class); GST_OBJECT_UNLOCK (inferencefilter); break; + case PROP_RESET_ENABLE: + GST_OBJECT_LOCK (inferencefilter); + g_value_set_boolean (value, inferencefilter->reset_enable); + GST_OBJECT_UNLOCK (inferencefilter); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); break; From 718ddad3b1238b37a29fe1820b65e5cde0cd4def Mon Sep 17 00:00:00 2001 From: c-trejos Date: Thu, 19 Dec 2019 12:27:45 -0600 Subject: [PATCH 066/188] Parse metadata to set enable property --- gst/inferencefilter/gstinferencefilter.cc | 63 ++++++++++++++++++----- 1 file changed, 49 insertions(+), 14 deletions(-) diff --git a/gst/inferencefilter/gstinferencefilter.cc b/gst/inferencefilter/gstinferencefilter.cc index 0425c6f4..3ecccf45 100644 --- a/gst/inferencefilter/gstinferencefilter.cc +++ b/gst/inferencefilter/gstinferencefilter.cc @@ -64,9 +64,7 @@ static void gst_inferencefilter_get_property (GObject * object, static gboolean gst_inferencefilter_start (GstBaseTransform * trans); static gboolean gst_inferencefilter_stop (GstBaseTransform * trans); static void gst_inferencefilter_finalize (GObject * object); - -static gboolean gst_inferencefilter_transform_meta (GstBaseTransform * trans, - GstBuffer * outbuf, GstMeta * meta, GstBuffer * inbuf); +static void gst_inferencefilter_filter_enable(GstInferencefilter * inferencefilter, GstInferencePrediction *rot, gint class_id, gboolean reset); static GstFlowReturn gst_inferencefilter_transform_ip (GstBaseTransform * trans, GstBuffer * buf); @@ -129,11 +127,11 @@ gst_inferencefilter_class_init (GstInferencefilterClass * klass) gobject_class->set_property = gst_inferencefilter_set_property; gobject_class->get_property = gst_inferencefilter_get_property; - + base_transform_class->start = GST_DEBUG_FUNCPTR (gst_inferencefilter_start); base_transform_class->stop = GST_DEBUG_FUNCPTR (gst_inferencefilter_stop); gobject_class->finalize = gst_inferencefilter_finalize; - + g_object_class_install_property (gobject_class, PROP_FILTER_CLASS_LABEL, g_param_spec_int ("filter-class", "filter-class", "Filter class", PROP_FILTER_CLASS_LABEL_MIN, G_MAXINT, PROP_FILTER_CLASS_LABEL_DEFAULT, GST_INFERENCEFILTER_PROPERTY_FLAGS)); @@ -238,24 +236,61 @@ gst_inferencefilter_finalize (GObject * object) G_OBJECT_CLASS (gst_inferencefilter_parent_class)->finalize (object); } - -static gboolean -gst_inferencefilter_transform_meta (GstBaseTransform * trans, - GstBuffer * outbuf, GstMeta * meta, GstBuffer * inbuf) +static void +gst_inferencefilter_filter_enable (GstInferencefilter * inferencefilter, GstInferencePrediction *root, gint class_id, gboolean reset) { - GstInferencefilter *inferencefilter = GST_INFERENCEFILTER (trans); - GST_DEBUG_OBJECT (inferencefilter, "transform_ip"); - return TRUE; + guint i; + GList *iter = NULL; + if (root == NULL) { + GST_ERROR_OBJECT (inferencefilter, "Invalid inference prediction."); + } + + if (root->classifications == NULL) { + GST_LOG_OBJECT (inferencefilter, "No inference classification on prediction."); + } + + for (iter = root->classifications; iter != NULL; iter = g_list_next(iter)) { + GstInferenceClassification *classification = (GstInferenceClassification *)iter->data; + if (classification->class_id == class_id || reset) { + GST_DEBUG_OBJECT(inferencefilter, "Enabling classification id %d", classification->class_id); + root->enabled = TRUE; + break; + } else { + GST_DEBUG_OBJECT(inferencefilter, "Disabling classification id %d", classification->class_id); + root->enabled = FALSE; + } + } + + for (i = 0; i < g_node_n_children(root->predictions) ; ++i) { + GstInferencePrediction *predict = (GstInferencePrediction*)g_node_nth_child (root->predictions,i)->data; + gst_inferencefilter_filter_enable (inferencefilter, predict, class_id, reset); + } } static GstFlowReturn gst_inferencefilter_transform_ip (GstBaseTransform * trans, GstBuffer * buf) { GstInferencefilter *inferencefilter = GST_INFERENCEFILTER (trans); - + GstInferenceMeta *meta; GST_DEBUG_OBJECT (inferencefilter, "transform_ip"); - return GST_FLOW_OK; + meta = (GstInferenceMeta *) gst_buffer_get_meta (buf, + GST_INFERENCE_META_API_TYPE); + + if (NULL == meta) { + GST_LOG_OBJECT (inferencefilter, "No inference meta found. Buffer passthrough."); + return GST_FLOW_OK; + } + + g_return_val_if_fail(meta->prediction, GST_FLOW_ERROR); + + if (inferencefilter->filter_class < 0 && !inferencefilter->reset_enable) { + GST_ERROR_OBJECT (inferencefilter, "Invalid filter-class value"); + return GST_FLOW_ERROR; + } else { + gst_inferencefilter_filter_enable (inferencefilter, meta->prediction, inferencefilter->filter_class, inferencefilter->reset_enable); + return GST_FLOW_OK; + } } static gboolean From 5cf87c7614c9b608a66c0ecf5be458e0819ccb44 Mon Sep 17 00:00:00 2001 From: c-trejos Date: Mon, 30 Dec 2019 14:39:41 -0600 Subject: [PATCH 067/188] Add inference filter to Makefile --- configure.ac | 1 + gst/Makefile.am | 2 +- gst/inferencefilter/Makefile.am | 35 +++++++++++++++++++++++++++++++++ 3 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 gst/inferencefilter/Makefile.am diff --git a/configure.ac b/configure.ac index 674a1464..12e297e9 100644 --- a/configure.ac +++ b/configure.ac @@ -383,6 +383,7 @@ ext/opencv/Makefile ext/r2inference/Makefile gst/Makefile gst/inferencecrop/Makefile +gst/inferencefilter/Makefile gst-libs/Makefile gst-libs/gst/Makefile gst-libs/gst/opencv/Makefile diff --git a/gst/Makefile.am b/gst/Makefile.am index bdebbbf5..c765c2e8 100644 --- a/gst/Makefile.am +++ b/gst/Makefile.am @@ -1 +1 @@ -SUBDIRS = inferencecrop +SUBDIRS = inferencecrop inferencefilter diff --git a/gst/inferencefilter/Makefile.am b/gst/inferencefilter/Makefile.am new file mode 100644 index 00000000..aa581af1 --- /dev/null +++ b/gst/inferencefilter/Makefile.am @@ -0,0 +1,35 @@ +plugin_LTLIBRARIES = libgstinferencefilter.la + +libgstinferencefilter_la_SOURCES = \ + gstinferencefilter.cc + +libgstinferencefilter_la_CFLAGS = \ + $(GST_CFLAGS) \ + $(GST_BASE_CFLAGS) \ + $(GST_PLUGINS_BASE_CFLAGS) \ + -I$(top_srcdir)/gst-libs \ + $(R2INFERENCE_CFLAGS) + +libgstinferencefilter_la_CXXFLAGS = \ + $(GST_CFLAGS) \ + $(GST_BASE_CFLAGS) \ + $(GST_PLUGINS_BASE_CFLAGS) \ + -I$(top_srcdir)/gst-libs \ + -std=c++11 \ + $(R2INFERENCE_CFLAGS) + +libgstinferencefilter_la_LIBADD = \ + $(GST_LIBS) \ + $(GST_BASE_LIBS) \ + $(GST_PLUGINS_BASE_LIBS) \ + $(R2INFERENCE_LIBS) \ + $(top_builddir)/gst-libs/gst/r2inference/libgstinference-@GST_API_VERSION@.la + +libgstinferencefilter_la_LDFLAGS = \ +$(GST_PLUGIN_LDFLAGS) + +libgstinferencefilter_la_LIBTOOLFLAGS = \ + $(GST_PLUGIN_LIBTOOLFLAGS) + +noinst_HEADERS = \ + gstinferencefilter.h From 4ef403b129993c9a21fba9a0b4c74d2d6b47dc92 Mon Sep 17 00:00:00 2001 From: c-trejos Date: Mon, 13 Jan 2020 17:07:25 -0600 Subject: [PATCH 068/188] Serialize inferencefilter struct access --- gst/inferencefilter/gstinferencefilter.cc | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/gst/inferencefilter/gstinferencefilter.cc b/gst/inferencefilter/gstinferencefilter.cc index 3ecccf45..3d268fac 100644 --- a/gst/inferencefilter/gstinferencefilter.cc +++ b/gst/inferencefilter/gstinferencefilter.cc @@ -271,6 +271,8 @@ static GstFlowReturn gst_inferencefilter_transform_ip (GstBaseTransform * trans, GstBuffer * buf) { GstInferencefilter *inferencefilter = GST_INFERENCEFILTER (trans); + gboolean reset = FALSE; + gint filter = -1; GstInferenceMeta *meta; GST_DEBUG_OBJECT (inferencefilter, "transform_ip"); @@ -284,12 +286,21 @@ gst_inferencefilter_transform_ip (GstBaseTransform * trans, GstBuffer * buf) g_return_val_if_fail(meta->prediction, GST_FLOW_ERROR); - if (inferencefilter->filter_class < 0 && !inferencefilter->reset_enable) { + GST_OBJECT_LOCK (inferencefilter); + reset = inferencefilter->reset_enable; + filter = inferencefilter->filter_class; + GST_OBJECT_UNLOCK (inferencefilter); + + if (filter < 0 && !reset) { GST_ERROR_OBJECT (inferencefilter, "Invalid filter-class value"); return GST_FLOW_ERROR; } else { - gst_inferencefilter_filter_enable (inferencefilter, meta->prediction, inferencefilter->filter_class, inferencefilter->reset_enable); - return GST_FLOW_OK; + if(meta->prediction == NULL) { + GST_LOG_OBJECT(inferencefilter, "Inferece meta has no predictions"); + return GST_FLOW_ERROR; + } + gst_inferencefilter_filter_enable (inferencefilter, meta->prediction, filter, reset); + return GST_FLOW_OK; } } From 8a51973f279539be36a151f9da478795854de48c Mon Sep 17 00:00:00 2001 From: c-trejos Date: Mon, 13 Jan 2020 17:34:30 -0600 Subject: [PATCH 069/188] Remove unused functions --- gst/inferencefilter/gstinferencefilter.cc | 40 ----------------------- 1 file changed, 40 deletions(-) diff --git a/gst/inferencefilter/gstinferencefilter.cc b/gst/inferencefilter/gstinferencefilter.cc index 3d268fac..2d6833b9 100644 --- a/gst/inferencefilter/gstinferencefilter.cc +++ b/gst/inferencefilter/gstinferencefilter.cc @@ -61,9 +61,6 @@ static void gst_inferencefilter_set_property (GObject * object, guint property_id, const GValue * value, GParamSpec * pspec); static void gst_inferencefilter_get_property (GObject * object, guint property_id, GValue * value, GParamSpec * pspec); -static gboolean gst_inferencefilter_start (GstBaseTransform * trans); -static gboolean gst_inferencefilter_stop (GstBaseTransform * trans); -static void gst_inferencefilter_finalize (GObject * object); static void gst_inferencefilter_filter_enable(GstInferencefilter * inferencefilter, GstInferencePrediction *rot, gint class_id, gboolean reset); static GstFlowReturn gst_inferencefilter_transform_ip (GstBaseTransform * trans, GstBuffer * buf); @@ -128,10 +125,6 @@ gst_inferencefilter_class_init (GstInferencefilterClass * klass) gobject_class->set_property = gst_inferencefilter_set_property; gobject_class->get_property = gst_inferencefilter_get_property; - base_transform_class->start = GST_DEBUG_FUNCPTR (gst_inferencefilter_start); - base_transform_class->stop = GST_DEBUG_FUNCPTR (gst_inferencefilter_stop); - gobject_class->finalize = gst_inferencefilter_finalize; - g_object_class_install_property (gobject_class, PROP_FILTER_CLASS_LABEL, g_param_spec_int ("filter-class", "filter-class", "Filter class", PROP_FILTER_CLASS_LABEL_MIN, G_MAXINT, PROP_FILTER_CLASS_LABEL_DEFAULT, GST_INFERENCEFILTER_PROPERTY_FLAGS)); @@ -141,8 +134,6 @@ gst_inferencefilter_class_init (GstInferencefilterClass * klass) "Enables all inference meta to be processed", PROP_RESET_ENABLE_DEFAULT, GST_INFERENCEFILTER_PROPERTY_FLAGS)); - base_transform_class->transform_meta = - GST_DEBUG_FUNCPTR (gst_inferencefilter_transform_meta); base_transform_class->transform_ip = GST_DEBUG_FUNCPTR (gst_inferencefilter_transform_ip); @@ -205,37 +196,6 @@ gst_inferencefilter_get_property (GObject * object, guint property_id, } } -/* states */ -static gboolean -gst_inferencefilter_start (GstBaseTransform * trans) -{ - GstInferencefilter *inferencefilter = GST_INFERENCEFILTER (trans); - - GST_DEBUG_OBJECT (inferencefilter, "start"); - - return TRUE; -} - -static gboolean -gst_inferencefilter_stop (GstBaseTransform * trans) -{ - GstInferencefilter *inferencefilter = GST_INFERENCEFILTER (trans); - - GST_DEBUG_OBJECT (inferencefilter, "stop"); - - return TRUE; -} - -void -gst_inferencefilter_finalize (GObject * object) -{ - GstInferencefilter *inferencefilter = GST_INFERENCEFILTER (object); - - GST_DEBUG_OBJECT (inferencefilter, "finalize"); - - G_OBJECT_CLASS (gst_inferencefilter_parent_class)->finalize (object); -} - static void gst_inferencefilter_filter_enable (GstInferencefilter * inferencefilter, GstInferencePrediction *root, gint class_id, gboolean reset) { From 2b68e79d0aef349963a99bbe81b53558f7fdb555 Mon Sep 17 00:00:00 2001 From: c-trejos Date: Tue, 14 Jan 2020 09:47:27 -0600 Subject: [PATCH 070/188] Fix indentation --- gst/inferencefilter/gstinferencefilter.cc | 67 +++++++++++++---------- 1 file changed, 38 insertions(+), 29 deletions(-) diff --git a/gst/inferencefilter/gstinferencefilter.cc b/gst/inferencefilter/gstinferencefilter.cc index 2d6833b9..ece1a2d9 100644 --- a/gst/inferencefilter/gstinferencefilter.cc +++ b/gst/inferencefilter/gstinferencefilter.cc @@ -61,7 +61,8 @@ static void gst_inferencefilter_set_property (GObject * object, guint property_id, const GValue * value, GParamSpec * pspec); static void gst_inferencefilter_get_property (GObject * object, guint property_id, GValue * value, GParamSpec * pspec); -static void gst_inferencefilter_filter_enable(GstInferencefilter * inferencefilter, GstInferencePrediction *rot, gint class_id, gboolean reset); +static void gst_inferencefilter_filter_enable (GstInferencefilter * + inferencefilter, GstInferencePrediction * rot, gint class_id, gboolean reset); static GstFlowReturn gst_inferencefilter_transform_ip (GstBaseTransform * trans, GstBuffer * buf); @@ -86,15 +87,13 @@ static GstStaticPadTemplate gst_inferencefilter_src_template = GST_STATIC_PAD_TEMPLATE ("src", GST_PAD_SRC, GST_PAD_ALWAYS, - GST_STATIC_CAPS_ANY - ); + GST_STATIC_CAPS_ANY); static GstStaticPadTemplate gst_inferencefilter_sink_template = GST_STATIC_PAD_TEMPLATE ("sink", GST_PAD_SINK, GST_PAD_ALWAYS, - GST_STATIC_CAPS_ANY - ); + GST_STATIC_CAPS_ANY); /* class initialization */ @@ -119,20 +118,21 @@ gst_inferencefilter_class_init (GstInferencefilterClass * klass) &gst_inferencefilter_sink_template); gst_element_class_set_static_metadata (GST_ELEMENT_CLASS (klass), - "Inference Filter", "Generic", "Enables/disables specific classes contained on the inference metadata to be processed", + "Inference Filter", "Generic", + "Enables/disables specific classes contained on the inference metadata to be processed", ""); gobject_class->set_property = gst_inferencefilter_set_property; gobject_class->get_property = gst_inferencefilter_get_property; g_object_class_install_property (gobject_class, PROP_FILTER_CLASS_LABEL, - g_param_spec_int ("filter-class", "filter-class", "Filter class", PROP_FILTER_CLASS_LABEL_MIN, G_MAXINT, - PROP_FILTER_CLASS_LABEL_DEFAULT, GST_INFERENCEFILTER_PROPERTY_FLAGS)); - g_object_class_install_property (gobject_class, - PROP_RESET_ENABLE, g_param_spec_boolean ("reset-enable", - "Reset enable", - "Enables all inference meta to be processed", - PROP_RESET_ENABLE_DEFAULT, GST_INFERENCEFILTER_PROPERTY_FLAGS)); + g_param_spec_int ("filter-class", "filter-class", "Filter class", + PROP_FILTER_CLASS_LABEL_MIN, G_MAXINT, + PROP_FILTER_CLASS_LABEL_DEFAULT, GST_INFERENCEFILTER_PROPERTY_FLAGS)); + g_object_class_install_property (gobject_class, PROP_RESET_ENABLE, + g_param_spec_boolean ("reset-enable", "Reset enable", + "Enables all inference meta to be processed", + PROP_RESET_ENABLE_DEFAULT, GST_INFERENCEFILTER_PROPERTY_FLAGS)); base_transform_class->transform_ip = GST_DEBUG_FUNCPTR (gst_inferencefilter_transform_ip); @@ -158,12 +158,12 @@ gst_inferencefilter_set_property (GObject * object, guint property_id, case PROP_FILTER_CLASS_LABEL: GST_OBJECT_LOCK (inferencefilter); inferencefilter->filter_class = g_value_get_int (value); - GST_OBJECT_UNLOCK(inferencefilter); + GST_OBJECT_UNLOCK (inferencefilter); break; case PROP_RESET_ENABLE: GST_OBJECT_LOCK (inferencefilter); inferencefilter->reset_enable = g_value_get_boolean (value); - GST_OBJECT_UNLOCK(inferencefilter); + GST_OBJECT_UNLOCK (inferencefilter); break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); @@ -197,7 +197,8 @@ gst_inferencefilter_get_property (GObject * object, guint property_id, } static void -gst_inferencefilter_filter_enable (GstInferencefilter * inferencefilter, GstInferencePrediction *root, gint class_id, gboolean reset) +gst_inferencefilter_filter_enable (GstInferencefilter * inferencefilter, + GstInferencePrediction * root, gint class_id, gboolean reset) { guint i; GList *iter = NULL; @@ -206,23 +207,29 @@ gst_inferencefilter_filter_enable (GstInferencefilter * inferencefilter, GstInfe } if (root->classifications == NULL) { - GST_LOG_OBJECT (inferencefilter, "No inference classification on prediction."); + GST_LOG_OBJECT (inferencefilter, + "No inference classification on prediction."); } - for (iter = root->classifications; iter != NULL; iter = g_list_next(iter)) { - GstInferenceClassification *classification = (GstInferenceClassification *)iter->data; + for (iter = root->classifications; iter != NULL; iter = g_list_next (iter)) { + GstInferenceClassification *classification = + (GstInferenceClassification *) iter->data; if (classification->class_id == class_id || reset) { - GST_DEBUG_OBJECT(inferencefilter, "Enabling classification id %d", classification->class_id); + GST_DEBUG_OBJECT (inferencefilter, "Enabling classification id %d", + classification->class_id); root->enabled = TRUE; break; } else { - GST_DEBUG_OBJECT(inferencefilter, "Disabling classification id %d", classification->class_id); + GST_DEBUG_OBJECT (inferencefilter, "Disabling classification id %d", + classification->class_id); root->enabled = FALSE; } } - for (i = 0; i < g_node_n_children(root->predictions) ; ++i) { - GstInferencePrediction *predict = (GstInferencePrediction*)g_node_nth_child (root->predictions,i)->data; + for (i = 0; i < g_node_n_children (root->predictions); ++i) { + GstInferencePrediction *predict = + (GstInferencePrediction *) g_node_nth_child (root->predictions, + i)->data; gst_inferencefilter_filter_enable (inferencefilter, predict, class_id, reset); } } @@ -237,10 +244,11 @@ gst_inferencefilter_transform_ip (GstBaseTransform * trans, GstBuffer * buf) GST_DEBUG_OBJECT (inferencefilter, "transform_ip"); meta = (GstInferenceMeta *) gst_buffer_get_meta (buf, - GST_INFERENCE_META_API_TYPE); + GST_INFERENCE_META_API_TYPE); if (NULL == meta) { - GST_LOG_OBJECT (inferencefilter, "No inference meta found. Buffer passthrough."); + GST_LOG_OBJECT (inferencefilter, + "No inference meta found. Buffer passthrough."); return GST_FLOW_OK; } @@ -255,12 +263,13 @@ gst_inferencefilter_transform_ip (GstBaseTransform * trans, GstBuffer * buf) GST_ERROR_OBJECT (inferencefilter, "Invalid filter-class value"); return GST_FLOW_ERROR; } else { - if(meta->prediction == NULL) { - GST_LOG_OBJECT(inferencefilter, "Inferece meta has no predictions"); + if (meta->prediction == NULL) { + GST_LOG_OBJECT (inferencefilter, "Inferece meta has no predictions"); return GST_FLOW_ERROR; } - gst_inferencefilter_filter_enable (inferencefilter, meta->prediction, filter, reset); - return GST_FLOW_OK; + gst_inferencefilter_filter_enable (inferencefilter, meta->prediction, + filter, reset); + return GST_FLOW_OK; } } From 359968412c68e79fef98bde22588085f7573db30 Mon Sep 17 00:00:00 2001 From: c-trejos Date: Tue, 14 Jan 2020 10:05:26 -0600 Subject: [PATCH 071/188] Add missing pointer guard --- gst/inferencefilter/gstinferencefilter.cc | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/gst/inferencefilter/gstinferencefilter.cc b/gst/inferencefilter/gstinferencefilter.cc index ece1a2d9..8b3f2d7c 100644 --- a/gst/inferencefilter/gstinferencefilter.cc +++ b/gst/inferencefilter/gstinferencefilter.cc @@ -202,14 +202,9 @@ gst_inferencefilter_filter_enable (GstInferencefilter * inferencefilter, { guint i; GList *iter = NULL; - if (root == NULL) { - GST_ERROR_OBJECT (inferencefilter, "Invalid inference prediction."); - } - if (root->classifications == NULL) { - GST_LOG_OBJECT (inferencefilter, - "No inference classification on prediction."); - } + g_return_if_fail(inferencefilter); + g_return_if_fail(root); for (iter = root->classifications; iter != NULL; iter = g_list_next (iter)) { GstInferenceClassification *classification = From 0b080381b135660e4f03be8175a5bb982f9eaee7 Mon Sep 17 00:00:00 2001 From: c-trejos Date: Fri, 17 Jan 2020 09:19:45 -0600 Subject: [PATCH 072/188] Use existing inference_prediction_get_children for iteration --- gst/inferencefilter/gstinferencefilter.cc | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/gst/inferencefilter/gstinferencefilter.cc b/gst/inferencefilter/gstinferencefilter.cc index 8b3f2d7c..d573878d 100644 --- a/gst/inferencefilter/gstinferencefilter.cc +++ b/gst/inferencefilter/gstinferencefilter.cc @@ -200,8 +200,8 @@ static void gst_inferencefilter_filter_enable (GstInferencefilter * inferencefilter, GstInferencePrediction * root, gint class_id, gboolean reset) { - guint i; GList *iter = NULL; + GSList *iter_child = NULL; g_return_if_fail(inferencefilter); g_return_if_fail(root); @@ -221,10 +221,9 @@ gst_inferencefilter_filter_enable (GstInferencefilter * inferencefilter, } } - for (i = 0; i < g_node_n_children (root->predictions); ++i) { + for (iter_child = gst_inference_prediction_get_children (root); iter_child != NULL; iter_child = g_slist_next (iter_child)) { GstInferencePrediction *predict = - (GstInferencePrediction *) g_node_nth_child (root->predictions, - i)->data; + (GstInferencePrediction *) iter_child->data; gst_inferencefilter_filter_enable (inferencefilter, predict, class_id, reset); } } From 34baef6cbc8d9ef4e22bed5593db7d052fdf3c69 Mon Sep 17 00:00:00 2001 From: c-trejos Date: Fri, 17 Jan 2020 09:51:50 -0600 Subject: [PATCH 073/188] Fix license header --- gst/inferencefilter/gstinferencefilter.cc | 2 +- gst/inferencefilter/gstinferencefilter.h | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/gst/inferencefilter/gstinferencefilter.cc b/gst/inferencefilter/gstinferencefilter.cc index d573878d..f4afd639 100644 --- a/gst/inferencefilter/gstinferencefilter.cc +++ b/gst/inferencefilter/gstinferencefilter.cc @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2019 RidgeRun + * Copyright (C) 2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public diff --git a/gst/inferencefilter/gstinferencefilter.h b/gst/inferencefilter/gstinferencefilter.h index fc5f4520..597e9dc5 100644 --- a/gst/inferencefilter/gstinferencefilter.h +++ b/gst/inferencefilter/gstinferencefilter.h @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2019 RidgeRun + * Copyright (C) 2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public @@ -19,7 +19,6 @@ * */ - #ifndef _GST_INFERENCEFILTER_H_ #define _GST_INFERENCEFILTER_H_ From ad580e260fa364c5dae92c59b08c42724bc6cd84 Mon Sep 17 00:00:00 2001 From: c-trejos Date: Fri, 17 Jan 2020 11:26:58 -0600 Subject: [PATCH 074/188] Switch inferencefilter from c++ to c --- gst/inferencefilter/Makefile.am | 10 +--------- ...gstinferencefilter.cc => gstinferencefilter.c} | 15 +++++++++------ 2 files changed, 10 insertions(+), 15 deletions(-) rename gst/inferencefilter/{gstinferencefilter.cc => gstinferencefilter.c} (97%) diff --git a/gst/inferencefilter/Makefile.am b/gst/inferencefilter/Makefile.am index aa581af1..84b68a81 100644 --- a/gst/inferencefilter/Makefile.am +++ b/gst/inferencefilter/Makefile.am @@ -1,7 +1,7 @@ plugin_LTLIBRARIES = libgstinferencefilter.la libgstinferencefilter_la_SOURCES = \ - gstinferencefilter.cc + gstinferencefilter.c libgstinferencefilter_la_CFLAGS = \ $(GST_CFLAGS) \ @@ -10,14 +10,6 @@ libgstinferencefilter_la_CFLAGS = \ -I$(top_srcdir)/gst-libs \ $(R2INFERENCE_CFLAGS) -libgstinferencefilter_la_CXXFLAGS = \ - $(GST_CFLAGS) \ - $(GST_BASE_CFLAGS) \ - $(GST_PLUGINS_BASE_CFLAGS) \ - -I$(top_srcdir)/gst-libs \ - -std=c++11 \ - $(R2INFERENCE_CFLAGS) - libgstinferencefilter_la_LIBADD = \ $(GST_LIBS) \ $(GST_BASE_LIBS) \ diff --git a/gst/inferencefilter/gstinferencefilter.cc b/gst/inferencefilter/gstinferencefilter.c similarity index 97% rename from gst/inferencefilter/gstinferencefilter.cc rename to gst/inferencefilter/gstinferencefilter.c index f4afd639..099a0d12 100644 --- a/gst/inferencefilter/gstinferencefilter.cc +++ b/gst/inferencefilter/gstinferencefilter.c @@ -62,7 +62,8 @@ static void gst_inferencefilter_set_property (GObject * object, static void gst_inferencefilter_get_property (GObject * object, guint property_id, GValue * value, GParamSpec * pspec); static void gst_inferencefilter_filter_enable (GstInferencefilter * - inferencefilter, GstInferencePrediction * rot, gint class_id, gboolean reset); + inferencefilter, GstInferencePrediction * rot, gint class_id, + gboolean reset); static GstFlowReturn gst_inferencefilter_transform_ip (GstBaseTransform * trans, GstBuffer * buf); @@ -203,8 +204,8 @@ gst_inferencefilter_filter_enable (GstInferencefilter * inferencefilter, GList *iter = NULL; GSList *iter_child = NULL; - g_return_if_fail(inferencefilter); - g_return_if_fail(root); + g_return_if_fail (inferencefilter); + g_return_if_fail (root); for (iter = root->classifications; iter != NULL; iter = g_list_next (iter)) { GstInferenceClassification *classification = @@ -221,10 +222,12 @@ gst_inferencefilter_filter_enable (GstInferencefilter * inferencefilter, } } - for (iter_child = gst_inference_prediction_get_children (root); iter_child != NULL; iter_child = g_slist_next (iter_child)) { + for (iter_child = gst_inference_prediction_get_children (root); + iter_child != NULL; iter_child = g_slist_next (iter_child)) { GstInferencePrediction *predict = (GstInferencePrediction *) iter_child->data; - gst_inferencefilter_filter_enable (inferencefilter, predict, class_id, reset); + gst_inferencefilter_filter_enable (inferencefilter, predict, class_id, + reset); } } @@ -246,7 +249,7 @@ gst_inferencefilter_transform_ip (GstBaseTransform * trans, GstBuffer * buf) return GST_FLOW_OK; } - g_return_val_if_fail(meta->prediction, GST_FLOW_ERROR); + g_return_val_if_fail (meta->prediction, GST_FLOW_ERROR); GST_OBJECT_LOCK (inferencefilter); reset = inferencefilter->reset_enable; From 1114eb14b9d157d91e7ae8031838e7c694885bcd Mon Sep 17 00:00:00 2001 From: c-trejos Date: Fri, 17 Jan 2020 11:41:56 -0600 Subject: [PATCH 075/188] Disable filter when filter_class is -1 --- gst/inferencefilter/gstinferencefilter.c | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/gst/inferencefilter/gstinferencefilter.c b/gst/inferencefilter/gstinferencefilter.c index 099a0d12..7e2891b4 100644 --- a/gst/inferencefilter/gstinferencefilter.c +++ b/gst/inferencefilter/gstinferencefilter.c @@ -127,17 +127,15 @@ gst_inferencefilter_class_init (GstInferencefilterClass * klass) gobject_class->get_property = gst_inferencefilter_get_property; g_object_class_install_property (gobject_class, PROP_FILTER_CLASS_LABEL, - g_param_spec_int ("filter-class", "filter-class", "Filter class", - PROP_FILTER_CLASS_LABEL_MIN, G_MAXINT, + g_param_spec_int ("filter-class", "filter-class", + "Filter class (-1 = disabled)", PROP_FILTER_CLASS_LABEL_MIN, G_MAXINT, PROP_FILTER_CLASS_LABEL_DEFAULT, GST_INFERENCEFILTER_PROPERTY_FLAGS)); g_object_class_install_property (gobject_class, PROP_RESET_ENABLE, g_param_spec_boolean ("reset-enable", "Reset enable", "Enables all inference meta to be processed", PROP_RESET_ENABLE_DEFAULT, GST_INFERENCEFILTER_PROPERTY_FLAGS)); - base_transform_class->transform_ip = GST_DEBUG_FUNCPTR (gst_inferencefilter_transform_ip); - } static void @@ -257,17 +255,18 @@ gst_inferencefilter_transform_ip (GstBaseTransform * trans, GstBuffer * buf) GST_OBJECT_UNLOCK (inferencefilter); if (filter < 0 && !reset) { - GST_ERROR_OBJECT (inferencefilter, "Invalid filter-class value"); + GST_LOG_OBJECT (inferencefilter, "Filter disabled"); + return GST_FLOW_OK; + } + + if (meta->prediction == NULL) { + GST_LOG_OBJECT (inferencefilter, "Inferece meta has no predictions"); return GST_FLOW_ERROR; } else { - if (meta->prediction == NULL) { - GST_LOG_OBJECT (inferencefilter, "Inferece meta has no predictions"); - return GST_FLOW_ERROR; - } gst_inferencefilter_filter_enable (inferencefilter, meta->prediction, filter, reset); - return GST_FLOW_OK; } + return GST_FLOW_OK; } static gboolean From e03083828a96f92525b6635e13683069db88c4bb Mon Sep 17 00:00:00 2001 From: c-trejos Date: Fri, 17 Jan 2020 11:48:04 -0600 Subject: [PATCH 076/188] Remove unnecessary check --- gst/inferencefilter/gstinferencefilter.c | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/gst/inferencefilter/gstinferencefilter.c b/gst/inferencefilter/gstinferencefilter.c index 7e2891b4..e8b7ad5b 100644 --- a/gst/inferencefilter/gstinferencefilter.c +++ b/gst/inferencefilter/gstinferencefilter.c @@ -259,13 +259,8 @@ gst_inferencefilter_transform_ip (GstBaseTransform * trans, GstBuffer * buf) return GST_FLOW_OK; } - if (meta->prediction == NULL) { - GST_LOG_OBJECT (inferencefilter, "Inferece meta has no predictions"); - return GST_FLOW_ERROR; - } else { - gst_inferencefilter_filter_enable (inferencefilter, meta->prediction, - filter, reset); - } + gst_inferencefilter_filter_enable (inferencefilter, meta->prediction, + filter, reset); return GST_FLOW_OK; } From 540ab064e6cd375546e76cebceeb22f307604f9a Mon Sep 17 00:00:00 2001 From: tvlenin Date: Wed, 15 Jan 2020 08:25:11 -0600 Subject: [PATCH 077/188] Add new property to videoinference to read labels --- gst-libs/gst/r2inference/gstvideoinference.c | 31 +++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/gst-libs/gst/r2inference/gstvideoinference.c b/gst-libs/gst/r2inference/gstvideoinference.c index 184ef795..31b785c9 100644 --- a/gst-libs/gst/r2inference/gstvideoinference.c +++ b/gst-libs/gst/r2inference/gstvideoinference.c @@ -45,7 +45,8 @@ GST_DEBUG_CATEGORY_STATIC (gst_video_inference_debug_category); #define GST_CAT_DEFAULT gst_video_inference_debug_category #define DEFAULT_MODEL_LOCATION NULL - +#define DEFAULT_LABELS NULL +#define DEFAULT_NUM_LABELS 0 enum { NEW_PREDICTION_SIGNAL, @@ -58,6 +59,7 @@ enum PROP_0, PROP_BACKEND, PROP_MODEL_LOCATION, + PROP_LABELS, }; GQuark _size_quark; @@ -89,6 +91,10 @@ struct _GstVideoInferencePrivate GstBackend *backend; gchar *model_location; + + gchar *labels; + gchar **labels_list; + gint num_labels; }; /* GObject methods */ @@ -223,6 +229,10 @@ gst_video_inference_class_init (GstVideoInferenceClass * klass) g_param_spec_string ("model-location", "Model Location", "Path to the model to use", DEFAULT_MODEL_LOCATION, G_PARAM_READWRITE)); + g_object_class_install_property (oclass, PROP_LABELS, + g_param_spec_string ("labels", "labels", + "Semicolon separated string containing inference labels", + DEFAULT_LABELS, G_PARAM_READWRITE)); gst_video_inference_signals[NEW_PREDICTION_SIGNAL] = g_signal_new ("new-prediction", G_TYPE_FROM_CLASS (klass), @@ -251,6 +261,10 @@ gst_video_inference_init (GstVideoInference * self) { GstVideoInferencePrivate *priv = GST_VIDEO_INFERENCE_PRIVATE (self); + priv->labels = DEFAULT_LABELS; + priv->labels_list = DEFAULT_LABELS; + priv->num_labels = DEFAULT_NUM_LABELS; + priv->sink_bypass_data = NULL; priv->sink_model_data = NULL; @@ -301,6 +315,18 @@ gst_video_inference_set_property (GObject * object, } GST_OBJECT_UNLOCK (self); break; + case PROP_LABELS: + if (priv->labels != NULL) { + g_free (priv->labels); + } + if (priv->labels_list != NULL) { + g_strfreev (priv->labels_list); + } + priv->labels = g_value_dup_string (value); + priv->labels_list = g_strsplit (g_value_get_string (value), ";", 0); + priv->num_labels = g_strv_length (priv->labels_list); + GST_DEBUG_OBJECT (self, "Changed inference labels %s", priv->labels); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); break; @@ -323,6 +349,9 @@ gst_video_inference_get_property (GObject * object, case PROP_MODEL_LOCATION: g_value_set_string (value, priv->model_location); break; + case PROP_LABELS: + g_value_set_string (value, priv->labels); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); break; From 42089ee2f1e12adf959582787821a449c56d179f Mon Sep 17 00:00:00 2001 From: tvlenin Date: Wed, 15 Jan 2020 10:01:18 -0600 Subject: [PATCH 078/188] Add labels to new metadata --- ext/r2inference/gstfacenetv1.c | 5 +++-- ext/r2inference/gstinceptionv1.c | 5 +++-- ext/r2inference/gstinceptionv2.c | 5 +++-- ext/r2inference/gstinceptionv3.c | 5 +++-- ext/r2inference/gstinceptionv4.c | 5 +++-- ext/r2inference/gstmobilenetv2.c | 5 +++-- ext/r2inference/gstresnet50v1.c | 5 +++-- ext/r2inference/gsttinyyolov2.c | 17 +++++++++++------ ext/r2inference/gsttinyyolov3.c | 4 ++-- .../gst/r2inference/gstinferencepostprocess.c | 8 +++++--- .../gst/r2inference/gstinferencepostprocess.h | 2 +- gst-libs/gst/r2inference/gstvideoinference.c | 3 ++- gst-libs/gst/r2inference/gstvideoinference.h | 2 +- 13 files changed, 43 insertions(+), 28 deletions(-) diff --git a/ext/r2inference/gstfacenetv1.c b/ext/r2inference/gstfacenetv1.c index 903ec654..8a1af8e4 100644 --- a/ext/r2inference/gstfacenetv1.c +++ b/ext/r2inference/gstfacenetv1.c @@ -63,7 +63,8 @@ static gboolean gst_facenetv1_preprocess (GstVideoInference * vi, GstVideoFrame * inframe, GstVideoFrame * outframe); static gboolean gst_facenetv1_postprocess (GstVideoInference * vi, const gpointer prediction, gsize predsize, GstMeta * meta_model[2], - GstVideoInfo * info_model, gboolean * valid_prediction); + GstVideoInfo * info_model, gboolean * valid_prediction, + gchar ** labels_list, gint num_labels); static gboolean gst_facenetv1_start (GstVideoInference * vi); static gboolean gst_facenetv1_stop (GstVideoInference * vi); @@ -207,7 +208,7 @@ gst_facenetv1_preprocess (GstVideoInference * vi, static gboolean gst_facenetv1_postprocess (GstVideoInference * vi, const gpointer prediction, gsize predsize, GstMeta * meta_model[2], GstVideoInfo * info_model, - gboolean * valid_prediction) + gboolean * valid_prediction, gchar ** labels_list, gint num_labels) { GstClassificationMeta *class_meta = (GstClassificationMeta *) meta_model[0]; GstDebugLevel gst_debug_level = GST_LEVEL_LOG; diff --git a/ext/r2inference/gstinceptionv1.c b/ext/r2inference/gstinceptionv1.c index 0be9e5e2..4e3949a7 100644 --- a/ext/r2inference/gstinceptionv1.c +++ b/ext/r2inference/gstinceptionv1.c @@ -58,7 +58,8 @@ static gboolean gst_inceptionv1_preprocess (GstVideoInference * vi, GstVideoFrame * inframe, GstVideoFrame * outframe); static gboolean gst_inceptionv1_postprocess (GstVideoInference * vi, const gpointer prediction, gsize predsize, GstMeta * meta_model[2], - GstVideoInfo * info_model, gboolean * valid_prediction); + GstVideoInfo * info_model, gboolean * valid_prediction, + gchar ** labels_list, gint num_labels); static gboolean gst_inceptionv1_start (GstVideoInference * vi); static gboolean gst_inceptionv1_stop (GstVideoInference * vi); @@ -146,7 +147,7 @@ gst_inceptionv1_preprocess (GstVideoInference * vi, static gboolean gst_inceptionv1_postprocess (GstVideoInference * vi, const gpointer prediction, gsize predsize, GstMeta * meta_model[2], GstVideoInfo * info_model, - gboolean * valid_prediction) + gboolean * valid_prediction, gchar ** labels_list, gint num_labels) { GstClassificationMeta *class_meta = (GstClassificationMeta *) meta_model[0]; GstDebugLevel gst_debug_level = GST_LEVEL_LOG; diff --git a/ext/r2inference/gstinceptionv2.c b/ext/r2inference/gstinceptionv2.c index b0e42b99..82840541 100644 --- a/ext/r2inference/gstinceptionv2.c +++ b/ext/r2inference/gstinceptionv2.c @@ -66,7 +66,8 @@ static gboolean gst_inceptionv2_preprocess (GstVideoInference * vi, GstVideoFrame * inframe, GstVideoFrame * outframe); static gboolean gst_inceptionv2_postprocess (GstVideoInference * vi, const gpointer prediction, gsize predsize, GstMeta * meta_model[2], - GstVideoInfo * info_model, gboolean * valid_prediction); + GstVideoInfo * info_model, gboolean * valid_prediction, + gchar ** labels_list, gint num_labels); static gboolean gst_inceptionv2_postprocess_old (GstVideoInference * vi, const gpointer prediction, gsize predsize, GstMeta * meta_model, GstVideoInfo * info_model, gboolean * valid_prediction); @@ -269,7 +270,7 @@ gst_inceptionv2_postprocess_new (GstVideoInference * vi, static gboolean gst_inceptionv2_postprocess (GstVideoInference * vi, const gpointer prediction, gsize predsize, GstMeta * meta_model[2], GstVideoInfo * info_model, - gboolean * valid_prediction) + gboolean * valid_prediction, gchar ** labels_list, gint num_labels) { gboolean ret = TRUE; diff --git a/ext/r2inference/gstinceptionv3.c b/ext/r2inference/gstinceptionv3.c index 8201ed2a..741cdde0 100644 --- a/ext/r2inference/gstinceptionv3.c +++ b/ext/r2inference/gstinceptionv3.c @@ -57,7 +57,8 @@ static gboolean gst_inceptionv3_preprocess (GstVideoInference * vi, GstVideoFrame * inframe, GstVideoFrame * outframe); static gboolean gst_inceptionv3_postprocess (GstVideoInference * vi, const gpointer prediction, gsize predsize, GstMeta * meta_model[2], - GstVideoInfo * info_model, gboolean * valid_prediction); + GstVideoInfo * info_model, gboolean * valid_prediction, + gchar ** labels_list, gint num_labels); static gboolean gst_inceptionv3_start (GstVideoInference * vi); static gboolean gst_inceptionv3_stop (GstVideoInference * vi); @@ -145,7 +146,7 @@ gst_inceptionv3_preprocess (GstVideoInference * vi, static gboolean gst_inceptionv3_postprocess (GstVideoInference * vi, const gpointer prediction, gsize predsize, GstMeta * meta_model[2], GstVideoInfo * info_model, - gboolean * valid_prediction) + gboolean * valid_prediction, gchar ** labels_list, gint num_labels) { GstClassificationMeta *class_meta = (GstClassificationMeta *) meta_model[0]; GstDebugLevel gst_debug_level = GST_LEVEL_LOG; diff --git a/ext/r2inference/gstinceptionv4.c b/ext/r2inference/gstinceptionv4.c index 74e1d791..a6f0dba3 100644 --- a/ext/r2inference/gstinceptionv4.c +++ b/ext/r2inference/gstinceptionv4.c @@ -65,7 +65,8 @@ static gboolean gst_inceptionv4_preprocess (GstVideoInference * vi, GstVideoFrame * inframe, GstVideoFrame * outframe); static gboolean gst_inceptionv4_postprocess (GstVideoInference * vi, const gpointer prediction, gsize predsize, GstMeta * meta_model[2], - GstVideoInfo * info_model, gboolean * valid_prediction); + GstVideoInfo * info_model, gboolean * valid_prediction, + gchar ** labels_list, gint num_labels); static gboolean gst_inceptionv4_start (GstVideoInference * vi); static gboolean gst_inceptionv4_stop (GstVideoInference * vi); @@ -212,7 +213,7 @@ gst_inceptionv4_preprocess (GstVideoInference * vi, static gboolean gst_inceptionv4_postprocess (GstVideoInference * vi, const gpointer prediction, gsize predsize, GstMeta * meta_model[2], GstVideoInfo * info_model, - gboolean * valid_prediction) + gboolean * valid_prediction, gchar ** labels_list, gint num_labels) { GstClassificationMeta *class_meta = (GstClassificationMeta *) meta_model[0]; GstDebugLevel gst_debug_level = GST_LEVEL_LOG; diff --git a/ext/r2inference/gstmobilenetv2.c b/ext/r2inference/gstmobilenetv2.c index 412e099a..761d87e9 100644 --- a/ext/r2inference/gstmobilenetv2.c +++ b/ext/r2inference/gstmobilenetv2.c @@ -56,7 +56,8 @@ static gboolean gst_mobilenetv2_preprocess (GstVideoInference * vi, GstVideoFrame * inframe, GstVideoFrame * outframe); static gboolean gst_mobilenetv2_postprocess (GstVideoInference * vi, const gpointer prediction, gsize predsize, GstMeta * meta_model[2], - GstVideoInfo * info_model, gboolean * valid_prediction); + GstVideoInfo * info_model, gboolean * valid_prediction, + gchar ** labels_list, gint num_labels); static gboolean gst_mobilenetv2_start (GstVideoInference * vi); static gboolean gst_mobilenetv2_stop (GstVideoInference * vi); @@ -144,7 +145,7 @@ gst_mobilenetv2_preprocess (GstVideoInference * vi, static gboolean gst_mobilenetv2_postprocess (GstVideoInference * vi, const gpointer prediction, gsize predsize, GstMeta * meta_model[2], GstVideoInfo * info_model, - gboolean * valid_prediction) + gboolean * valid_prediction, gchar ** labels_list, gint num_labels) { GstClassificationMeta *class_meta = (GstClassificationMeta *) meta_model[0]; GstDebugLevel gst_debug_level = GST_LEVEL_LOG; diff --git a/ext/r2inference/gstresnet50v1.c b/ext/r2inference/gstresnet50v1.c index 60aabf41..be39c830 100644 --- a/ext/r2inference/gstresnet50v1.c +++ b/ext/r2inference/gstresnet50v1.c @@ -60,7 +60,8 @@ static gboolean gst_resnet50v1_preprocess (GstVideoInference * vi, GstVideoFrame * inframe, GstVideoFrame * outframe); static gboolean gst_resnet50v1_postprocess (GstVideoInference * vi, const gpointer prediction, gsize predsize, GstMeta * meta_model[2], - GstVideoInfo * info_model, gboolean * valid_prediction); + GstVideoInfo * info_model, gboolean * valid_prediction, + gchar ** labels_list, gint num_labels); static gboolean gst_resnet50v1_start (GstVideoInference * vi); static gboolean gst_resnet50v1_stop (GstVideoInference * vi); @@ -150,7 +151,7 @@ gst_resnet50v1_preprocess (GstVideoInference * vi, static gboolean gst_resnet50v1_postprocess (GstVideoInference * vi, const gpointer prediction, gsize predsize, GstMeta * meta_model[2], GstVideoInfo * info_model, - gboolean * valid_prediction) + gboolean * valid_prediction, gchar ** labels_list, gint num_labels) { GstClassificationMeta *class_meta = (GstClassificationMeta *) meta_model[0]; GstDebugLevel gst_debug_level = GST_LEVEL_LOG; diff --git a/ext/r2inference/gsttinyyolov2.c b/ext/r2inference/gsttinyyolov2.c index c97855c5..2529b012 100644 --- a/ext/r2inference/gsttinyyolov2.c +++ b/ext/r2inference/gsttinyyolov2.c @@ -82,7 +82,7 @@ static gboolean gst_tinyyolov2_preprocess (GstVideoInference * vi, static gboolean gst_tinyyolov2_postprocess (GstVideoInference * vi, const gpointer prediction, gsize predsize, GstMeta * meta_model[2], GstVideoInfo * info_model, - gboolean * valid_prediction); + gboolean * valid_prediction, gchar ** labels_list, gint num_labels); static gboolean gst_tinyyolov2_postprocess_old (GstVideoInference * vi, const gpointer prediction, gsize predsize, GstMeta * meta_model, @@ -90,7 +90,8 @@ gst_tinyyolov2_postprocess_old (GstVideoInference * vi, static gboolean gst_tinyyolov2_postprocess_new (GstVideoInference * vi, const gpointer prediction, gsize predsize, GstMeta * meta_model, - GstVideoInfo * info_model, gboolean * valid_prediction); + GstVideoInfo * info_model, gboolean * valid_prediction, + gchar ** labels_list, gint num_labels); static gboolean gst_tinyyolov2_start (GstVideoInference * vi); static gboolean gst_tinyyolov2_stop (GstVideoInference * vi); @@ -289,7 +290,7 @@ gst_tinyyolov2_preprocess (GstVideoInference * vi, static gboolean gst_tinyyolov2_postprocess (GstVideoInference * vi, const gpointer prediction, gsize predsize, GstMeta * meta_model[2], GstVideoInfo * info_model, - gboolean * valid_prediction) + gboolean * valid_prediction, gchar ** labels_list, gint num_labels) { gboolean ret = TRUE; @@ -298,13 +299,16 @@ gst_tinyyolov2_postprocess (GstVideoInference * vi, const gpointer prediction, g_return_val_if_fail (meta_model, FALSE); g_return_val_if_fail (info_model, FALSE); g_return_val_if_fail (valid_prediction, FALSE); + g_return_val_if_fail (labels_list, FALSE); + + g_print ("\n\n\n%s", labels_list[0]); ret &= gst_tinyyolov2_postprocess_old (vi, prediction, predsize, meta_model[0], info_model, valid_prediction); ret &= gst_tinyyolov2_postprocess_new (vi, prediction, predsize, meta_model[1], - info_model, valid_prediction); + info_model, valid_prediction, labels_list, num_labels); return TRUE; } @@ -337,7 +341,8 @@ gst_tinyyolov2_postprocess_old (GstVideoInference * vi, static gboolean gst_tinyyolov2_postprocess_new (GstVideoInference * vi, const gpointer prediction, gsize predsize, GstMeta * meta_model, - GstVideoInfo * info_model, gboolean * valid_prediction) + GstVideoInfo * info_model, gboolean * valid_prediction, + gchar ** labels_list, gint num_labels) { GstTinyyolov2 *tinyyolov2 = NULL; GstInferenceMeta *imeta = NULL; @@ -368,7 +373,7 @@ gst_tinyyolov2_postprocess_new (GstVideoInference * vi, for (i = 0; i < num_boxes; i++) { GstInferencePrediction *pred = - gst_create_prediction_from_box (vi, &boxes[i]); + gst_create_prediction_from_box (vi, &boxes[i], labels_list, num_labels); gst_inference_prediction_append (imeta->prediction, pred); } diff --git a/ext/r2inference/gsttinyyolov3.c b/ext/r2inference/gsttinyyolov3.c index 05f7dfab..d7406745 100644 --- a/ext/r2inference/gsttinyyolov3.c +++ b/ext/r2inference/gsttinyyolov3.c @@ -74,7 +74,7 @@ static gboolean gst_tinyyolov3_preprocess (GstVideoInference * vi, static gboolean gst_tinyyolov3_postprocess (GstVideoInference * vi, const gpointer prediction, gsize predsize, GstMeta * meta_model[2], GstVideoInfo * info_model, - gboolean * valid_prediction); + gboolean * valid_prediction, gchar ** labels_list, gint num_labels); static gboolean gst_tinyyolov3_start (GstVideoInference * vi); static gboolean gst_tinyyolov3_stop (GstVideoInference * vi); @@ -247,7 +247,7 @@ gst_tinyyolov3_preprocess (GstVideoInference * vi, static gboolean gst_tinyyolov3_postprocess (GstVideoInference * vi, const gpointer prediction, gsize predsize, GstMeta * meta_model[2], GstVideoInfo * info_model, - gboolean * valid_prediction) + gboolean * valid_prediction, gchar ** labels_list, gint num_labels) { GstTinyyolov3 *tinyyolov3 = GST_TINYYOLOV3 (vi); GstDetectionMeta *detect_meta = (GstDetectionMeta *) meta_model[0]; diff --git a/gst-libs/gst/r2inference/gstinferencepostprocess.c b/gst-libs/gst/r2inference/gstinferencepostprocess.c index 90a3bc40..989824a7 100644 --- a/gst-libs/gst/r2inference/gstinferencepostprocess.c +++ b/gst-libs/gst/r2inference/gstinferencepostprocess.c @@ -250,13 +250,15 @@ gst_create_boxes (GstVideoInference * vi, const gpointer prediction, } GstInferencePrediction * -gst_create_prediction_from_box (GstVideoInference * vi, BBox * box) +gst_create_prediction_from_box (GstVideoInference * vi, BBox * box, + gchar ** labels_list, gint num_labels) { GstInferencePrediction *predict = NULL; GstInferenceClassification *c = NULL; g_return_val_if_fail (vi != NULL, NULL); g_return_val_if_fail (box != NULL, NULL); + g_return_val_if_fail (labels_list != NULL, NULL); predict = gst_inference_prediction_new (); predict->bbox.x = box->x; @@ -264,8 +266,8 @@ gst_create_prediction_from_box (GstVideoInference * vi, BBox * box) predict->bbox.width = box->width; predict->bbox.height = box->height; - c = gst_inference_classification_new_full (box->label, box->prob, NULL, 1, - &box->prob, NULL); + c = gst_inference_classification_new_full (box->label, box->prob, NULL, + num_labels, &box->prob, labels_list); gst_inference_prediction_append_classification (predict, c); return predict; diff --git a/gst-libs/gst/r2inference/gstinferencepostprocess.h b/gst-libs/gst/r2inference/gstinferencepostprocess.h index a3db89b5..e24f388d 100644 --- a/gst-libs/gst/r2inference/gstinferencepostprocess.h +++ b/gst-libs/gst/r2inference/gstinferencepostprocess.h @@ -75,7 +75,7 @@ gboolean gst_create_boxes_float (GstVideoInference * vi, * \param vi Father object of every architecture * \param box Box used to fill Prediction */ -GstInferencePrediction *gst_create_prediction_from_box (GstVideoInference * vi, BBox * box); +GstInferencePrediction *gst_create_prediction_from_box (GstVideoInference * vi, BBox * box, gchar **labels_list, gint num_labels); /** * \brief Create Classification from prediction data diff --git a/gst-libs/gst/r2inference/gstvideoinference.c b/gst-libs/gst/r2inference/gstvideoinference.c index 31b785c9..60771996 100644 --- a/gst-libs/gst/r2inference/gstvideoinference.c +++ b/gst-libs/gst/r2inference/gstvideoinference.c @@ -892,6 +892,7 @@ gst_video_inference_postprocess (GstVideoInference * self, GstVideoInfo *info_model = NULL; GstVideoInfo *info_bypass = NULL; gboolean pred_valid = FALSE; + GstVideoInferencePrivate *priv = GST_VIDEO_INFERENCE_PRIVATE (self); g_return_val_if_fail (self, FALSE); g_return_val_if_fail (klass, FALSE); @@ -920,7 +921,7 @@ gst_video_inference_postprocess (GstVideoInference * self, GST_LOG_OBJECT (self, "Calling frame postprocess"); if (!klass->postprocess (self, prediction_data, prediction_size, meta_model, - info_model, &pred_valid)) { + info_model, &pred_valid, priv->labels_list, priv->num_labels)) { GST_ELEMENT_ERROR (self, STREAM, FAILED, ("Subclass failed at postprocess"), (NULL)); return FALSE; diff --git a/gst-libs/gst/r2inference/gstvideoinference.h b/gst-libs/gst/r2inference/gstvideoinference.h index f216d376..1dc610d8 100644 --- a/gst-libs/gst/r2inference/gstvideoinference.h +++ b/gst-libs/gst/r2inference/gstvideoinference.h @@ -40,7 +40,7 @@ struct _GstVideoInferenceClass GstVideoFrame * outframe); gboolean (*postprocess) (GstVideoInference * self, const gpointer prediction, gsize size, GstMeta * meta_model[2], - GstVideoInfo * info_model, gboolean * valid_prediction); + GstVideoInfo * info_model, gboolean * valid_prediction, gchar **labels_list, gint num_labels); const GstMetaInfo *inference_meta_info; From 74babbd436df17fc96fedc5aff91888fcaaefe28 Mon Sep 17 00:00:00 2001 From: tvlenin Date: Wed, 15 Jan 2020 10:16:33 -0600 Subject: [PATCH 079/188] Add labels to new meta to inceptionv2 --- ext/r2inference/gstinceptionv2.c | 10 ++++++---- ext/r2inference/gsttinyyolov2.c | 2 -- gst-libs/gst/r2inference/gstinferencepostprocess.c | 2 +- gst-libs/gst/r2inference/gstinferencepostprocess.h | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ext/r2inference/gstinceptionv2.c b/ext/r2inference/gstinceptionv2.c index 82840541..bc708482 100644 --- a/ext/r2inference/gstinceptionv2.c +++ b/ext/r2inference/gstinceptionv2.c @@ -73,7 +73,8 @@ static gboolean gst_inceptionv2_postprocess_old (GstVideoInference * vi, GstVideoInfo * info_model, gboolean * valid_prediction); static gboolean gst_inceptionv2_postprocess_new (GstVideoInference * vi, const gpointer prediction, gsize predsize, GstMeta * meta_model, - GstVideoInfo * info_model, gboolean * valid_prediction); + GstVideoInfo * info_model, gboolean * valid_prediction, + gchar ** labels_list); static gboolean gst_inceptionv2_start (GstVideoInference * vi); static gboolean gst_inceptionv2_stop (GstVideoInference * vi); @@ -239,7 +240,8 @@ gst_inceptionv2_postprocess_old (GstVideoInference * vi, static gboolean gst_inceptionv2_postprocess_new (GstVideoInference * vi, const gpointer prediction, gsize predsize, GstMeta * meta_model, - GstVideoInfo * info_model, gboolean * valid_prediction) + GstVideoInfo * info_model, gboolean * valid_prediction, + gchar ** labels_list) { GstInferenceMeta *imeta = NULL; GstInferenceClassification *c = NULL; @@ -259,7 +261,7 @@ gst_inceptionv2_postprocess_new (GstVideoInference * vi, return FALSE; } - c = gst_create_class_from_prediction (vi, prediction, predsize); + c = gst_create_class_from_prediction (vi, prediction, predsize, labels_list); gst_inference_prediction_append_classification (root, c); gst_inference_print_predictions (vi, gst_inceptionv2_debug_category, imeta); @@ -279,7 +281,7 @@ gst_inceptionv2_postprocess (GstVideoInference * vi, const gpointer prediction, info_model, valid_prediction); ret &= gst_inceptionv2_postprocess_new (vi, prediction, predsize, meta_model[1], - info_model, valid_prediction); + info_model, valid_prediction, labels_list); return ret; } diff --git a/ext/r2inference/gsttinyyolov2.c b/ext/r2inference/gsttinyyolov2.c index 2529b012..0ae6a142 100644 --- a/ext/r2inference/gsttinyyolov2.c +++ b/ext/r2inference/gsttinyyolov2.c @@ -301,8 +301,6 @@ gst_tinyyolov2_postprocess (GstVideoInference * vi, const gpointer prediction, g_return_val_if_fail (valid_prediction, FALSE); g_return_val_if_fail (labels_list, FALSE); - g_print ("\n\n\n%s", labels_list[0]); - ret &= gst_tinyyolov2_postprocess_old (vi, prediction, predsize, meta_model[0], info_model, valid_prediction); diff --git a/gst-libs/gst/r2inference/gstinferencepostprocess.c b/gst-libs/gst/r2inference/gstinferencepostprocess.c index 989824a7..9150b49f 100644 --- a/gst-libs/gst/r2inference/gstinferencepostprocess.c +++ b/gst-libs/gst/r2inference/gstinferencepostprocess.c @@ -275,7 +275,7 @@ gst_create_prediction_from_box (GstVideoInference * vi, BBox * box, GstInferenceClassification * gst_create_class_from_prediction (GstVideoInference * vi, - const gpointer prediction, gsize predsize) + const gpointer prediction, gsize predsize, gchar ** labels_list) { gdouble max = -1; gint index = 0; diff --git a/gst-libs/gst/r2inference/gstinferencepostprocess.h b/gst-libs/gst/r2inference/gstinferencepostprocess.h index e24f388d..ec8efb12 100644 --- a/gst-libs/gst/r2inference/gstinferencepostprocess.h +++ b/gst-libs/gst/r2inference/gstinferencepostprocess.h @@ -85,7 +85,7 @@ GstInferencePrediction *gst_create_prediction_from_box (GstVideoInference * vi, * \param predsize Size of the prediction */ GstInferenceClassification *gst_create_class_from_prediction (GstVideoInference * vi, - const gpointer prediction, gsize predsize); + const gpointer prediction, gsize predsize, gchar **labels_list); G_END_DECLS #endif From 2e615a33e81c481ffa3f6980f69042eac2de5e83 Mon Sep 17 00:00:00 2001 From: tvlenin Date: Wed, 15 Jan 2020 10:58:30 -0600 Subject: [PATCH 080/188] Support print metadata to debug with labels --- ext/r2inference/gsttinyyolov2.c | 1 - .../gst/r2inference/gstinferenceclassification.c | 13 ++++++++++--- gst-libs/gst/r2inference/gstinferencepostprocess.c | 8 +++++--- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/ext/r2inference/gsttinyyolov2.c b/ext/r2inference/gsttinyyolov2.c index 0ae6a142..77feb46d 100644 --- a/ext/r2inference/gsttinyyolov2.c +++ b/ext/r2inference/gsttinyyolov2.c @@ -299,7 +299,6 @@ gst_tinyyolov2_postprocess (GstVideoInference * vi, const gpointer prediction, g_return_val_if_fail (meta_model, FALSE); g_return_val_if_fail (info_model, FALSE); g_return_val_if_fail (valid_prediction, FALSE); - g_return_val_if_fail (labels_list, FALSE); ret &= gst_tinyyolov2_postprocess_old (vi, prediction, predsize, meta_model[0], diff --git a/gst-libs/gst/r2inference/gstinferenceclassification.c b/gst-libs/gst/r2inference/gstinferenceclassification.c index d72d208f..eba4abf2 100644 --- a/gst-libs/gst/r2inference/gstinferenceclassification.c +++ b/gst-libs/gst/r2inference/gstinferenceclassification.c @@ -177,16 +177,21 @@ gst_inference_classification_to_string (GstInferenceClassification * self, gint level) { gint indent = level * 2; + gchar *labels = g_strjoinv (";", self->labels); g_return_val_if_fail (self, NULL); return g_strdup_printf ("{\n" - "%*s Class : %d/%d\n" + "%*s Class : %d\n" "%*s Label : %s\n" "%*s Probability : %f\n" + "%*s Num_classes : %d\n" + "%*s Labels : %s\n" "%*s}", - indent, "", self->class_id, self->num_classes, - indent, "", self->class_label, indent, "", self->class_prob, indent, ""); + indent, "", self->class_id, + indent, "", self->class_label, + indent, "", self->class_prob, + indent, "", self->num_classes, indent, "", labels, indent, ""); } static void @@ -194,3 +199,5 @@ classification_free (GstInferenceClassification * self) { classification_reset (self); } + +//g_strjoin(";",self->labels) diff --git a/gst-libs/gst/r2inference/gstinferencepostprocess.c b/gst-libs/gst/r2inference/gstinferencepostprocess.c index 9150b49f..34be0872 100644 --- a/gst-libs/gst/r2inference/gstinferencepostprocess.c +++ b/gst-libs/gst/r2inference/gstinferencepostprocess.c @@ -255,10 +255,9 @@ gst_create_prediction_from_box (GstVideoInference * vi, BBox * box, { GstInferencePrediction *predict = NULL; GstInferenceClassification *c = NULL; - + gchar *label = NULL; g_return_val_if_fail (vi != NULL, NULL); g_return_val_if_fail (box != NULL, NULL); - g_return_val_if_fail (labels_list != NULL, NULL); predict = gst_inference_prediction_new (); predict->bbox.x = box->x; @@ -266,7 +265,10 @@ gst_create_prediction_from_box (GstVideoInference * vi, BBox * box, predict->bbox.width = box->width; predict->bbox.height = box->height; - c = gst_inference_classification_new_full (box->label, box->prob, NULL, + if (num_labels > box->label) { + label = labels_list[box->label]; + } + c = gst_inference_classification_new_full (box->label, box->prob, label, num_labels, &box->prob, labels_list); gst_inference_prediction_append_classification (predict, c); From f659c08950642e0ebed3b99c650ec0d59aaf31a9 Mon Sep 17 00:00:00 2001 From: tvlenin Date: Wed, 15 Jan 2020 11:38:24 -0600 Subject: [PATCH 081/188] Fix print metadata on inceptionv2 --- ext/r2inference/gstinceptionv2.c | 9 +++++---- gst-libs/gst/r2inference/gstinferencepostprocess.c | 9 ++++++--- gst-libs/gst/r2inference/gstinferencepostprocess.h | 2 +- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/ext/r2inference/gstinceptionv2.c b/ext/r2inference/gstinceptionv2.c index bc708482..e54323cb 100644 --- a/ext/r2inference/gstinceptionv2.c +++ b/ext/r2inference/gstinceptionv2.c @@ -74,7 +74,7 @@ static gboolean gst_inceptionv2_postprocess_old (GstVideoInference * vi, static gboolean gst_inceptionv2_postprocess_new (GstVideoInference * vi, const gpointer prediction, gsize predsize, GstMeta * meta_model, GstVideoInfo * info_model, gboolean * valid_prediction, - gchar ** labels_list); + gchar ** labels_list, gint num_labels); static gboolean gst_inceptionv2_start (GstVideoInference * vi); static gboolean gst_inceptionv2_stop (GstVideoInference * vi); @@ -241,7 +241,7 @@ static gboolean gst_inceptionv2_postprocess_new (GstVideoInference * vi, const gpointer prediction, gsize predsize, GstMeta * meta_model, GstVideoInfo * info_model, gboolean * valid_prediction, - gchar ** labels_list) + gchar ** labels_list, gint num_labels) { GstInferenceMeta *imeta = NULL; GstInferenceClassification *c = NULL; @@ -261,7 +261,8 @@ gst_inceptionv2_postprocess_new (GstVideoInference * vi, return FALSE; } - c = gst_create_class_from_prediction (vi, prediction, predsize, labels_list); + c = gst_create_class_from_prediction (vi, prediction, predsize, labels_list, + num_labels); gst_inference_prediction_append_classification (root, c); gst_inference_print_predictions (vi, gst_inceptionv2_debug_category, imeta); @@ -281,7 +282,7 @@ gst_inceptionv2_postprocess (GstVideoInference * vi, const gpointer prediction, info_model, valid_prediction); ret &= gst_inceptionv2_postprocess_new (vi, prediction, predsize, meta_model[1], - info_model, valid_prediction, labels_list); + info_model, valid_prediction, labels_list, num_labels); return ret; } diff --git a/gst-libs/gst/r2inference/gstinferencepostprocess.c b/gst-libs/gst/r2inference/gstinferencepostprocess.c index 34be0872..bb870b94 100644 --- a/gst-libs/gst/r2inference/gstinferencepostprocess.c +++ b/gst-libs/gst/r2inference/gstinferencepostprocess.c @@ -277,14 +277,14 @@ gst_create_prediction_from_box (GstVideoInference * vi, BBox * box, GstInferenceClassification * gst_create_class_from_prediction (GstVideoInference * vi, - const gpointer prediction, gsize predsize, gchar ** labels_list) + const gpointer prediction, gsize predsize, gchar ** labels_list, + gint num_labels) { gdouble max = -1; gint index = 0; gdouble *probs = NULL; gint num_classes = 0; const gchar *label = NULL; - gchar **labels = NULL; g_return_val_if_fail (vi != NULL, NULL); @@ -310,8 +310,11 @@ gst_create_class_from_prediction (GstVideoInference * vi, } } + if (num_labels > index) { + label = labels_list[index]; + } return gst_inference_classification_new_full (index, max, label, num_classes, - probs, labels); + probs, labels_list); } static void diff --git a/gst-libs/gst/r2inference/gstinferencepostprocess.h b/gst-libs/gst/r2inference/gstinferencepostprocess.h index ec8efb12..f9b23123 100644 --- a/gst-libs/gst/r2inference/gstinferencepostprocess.h +++ b/gst-libs/gst/r2inference/gstinferencepostprocess.h @@ -85,7 +85,7 @@ GstInferencePrediction *gst_create_prediction_from_box (GstVideoInference * vi, * \param predsize Size of the prediction */ GstInferenceClassification *gst_create_class_from_prediction (GstVideoInference * vi, - const gpointer prediction, gsize predsize, gchar **labels_list); + const gpointer prediction, gsize predsize, gchar **labels_list, gint num_labels); G_END_DECLS #endif From 89551e47fa7d4a8cec148bd33ba9d1c3e9a00eb0 Mon Sep 17 00:00:00 2001 From: tvlenin Date: Thu, 16 Jan 2020 11:14:07 -0600 Subject: [PATCH 082/188] Add documentation of new parameters --- gst-libs/gst/r2inference/gstinferencepostprocess.h | 4 ++++ gst-libs/gst/r2inference/gstvideoinference.c | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/gst-libs/gst/r2inference/gstinferencepostprocess.h b/gst-libs/gst/r2inference/gstinferencepostprocess.h index f9b23123..caa9dee7 100644 --- a/gst-libs/gst/r2inference/gstinferencepostprocess.h +++ b/gst-libs/gst/r2inference/gstinferencepostprocess.h @@ -74,6 +74,8 @@ gboolean gst_create_boxes_float (GstVideoInference * vi, * * \param vi Father object of every architecture * \param box Box used to fill Prediction + * \param labels_list List with all possible lables + * \param num_labels The number of posibble labels */ GstInferencePrediction *gst_create_prediction_from_box (GstVideoInference * vi, BBox * box, gchar **labels_list, gint num_labels); @@ -83,6 +85,8 @@ GstInferencePrediction *gst_create_prediction_from_box (GstVideoInference * vi, * \param vi Father object of every architecture * \param prediction Value of the prediction * \param predsize Size of the prediction + * \param labels_list List with all possible lables + * \param num_labels The number of posibble labels */ GstInferenceClassification *gst_create_class_from_prediction (GstVideoInference * vi, const gpointer prediction, gsize predsize, gchar **labels_list, gint num_labels); diff --git a/gst-libs/gst/r2inference/gstvideoinference.c b/gst-libs/gst/r2inference/gstvideoinference.c index 60771996..2744b99a 100644 --- a/gst-libs/gst/r2inference/gstvideoinference.c +++ b/gst-libs/gst/r2inference/gstvideoinference.c @@ -1227,6 +1227,10 @@ gst_video_inference_finalize (GObject * object) g_free (priv->model_location); priv->model_location = NULL; + priv->labels = NULL; + priv->labels_list = NULL; + + g_clear_object (&priv->backend); G_OBJECT_CLASS (gst_video_inference_parent_class)->finalize (object); From 6dee81247272b30784c74631f33cb8dc0d2d245e Mon Sep 17 00:00:00 2001 From: tvlenin Date: Thu, 16 Jan 2020 11:17:02 -0600 Subject: [PATCH 083/188] Protect pointer before use it --- gst-libs/gst/r2inference/gstinferenceclassification.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/gst-libs/gst/r2inference/gstinferenceclassification.c b/gst-libs/gst/r2inference/gstinferenceclassification.c index eba4abf2..da819e19 100644 --- a/gst-libs/gst/r2inference/gstinferenceclassification.c +++ b/gst-libs/gst/r2inference/gstinferenceclassification.c @@ -176,11 +176,14 @@ gchar * gst_inference_classification_to_string (GstInferenceClassification * self, gint level) { - gint indent = level * 2; - gchar *labels = g_strjoinv (";", self->labels); + gchar *labels = NULL; + gint indent = 0; g_return_val_if_fail (self, NULL); + labels = g_strjoinv (";", self->labels); + indent = level * 2; + return g_strdup_printf ("{\n" "%*s Class : %d\n" "%*s Label : %s\n" @@ -199,5 +202,3 @@ classification_free (GstInferenceClassification * self) { classification_reset (self); } - -//g_strjoin(";",self->labels) From 9badd6b7830045ad6556312bbd246c5095c82ac7 Mon Sep 17 00:00:00 2001 From: tvlenin Date: Thu, 16 Jan 2020 11:32:07 -0600 Subject: [PATCH 084/188] Remove labels from debug print --- gst-libs/gst/r2inference/gstinferenceclassification.c | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/gst-libs/gst/r2inference/gstinferenceclassification.c b/gst-libs/gst/r2inference/gstinferenceclassification.c index da819e19..31044264 100644 --- a/gst-libs/gst/r2inference/gstinferenceclassification.c +++ b/gst-libs/gst/r2inference/gstinferenceclassification.c @@ -176,25 +176,18 @@ gchar * gst_inference_classification_to_string (GstInferenceClassification * self, gint level) { - gchar *labels = NULL; - gint indent = 0; - + gint indent = level * 2; g_return_val_if_fail (self, NULL); - labels = g_strjoinv (";", self->labels); - indent = level * 2; - return g_strdup_printf ("{\n" "%*s Class : %d\n" "%*s Label : %s\n" "%*s Probability : %f\n" "%*s Num_classes : %d\n" - "%*s Labels : %s\n" "%*s}", indent, "", self->class_id, indent, "", self->class_label, - indent, "", self->class_prob, - indent, "", self->num_classes, indent, "", labels, indent, ""); + indent, "", self->class_prob, indent, "", self->num_classes, indent, ""); } static void From 2a65790ee6aca895402900fdb8371e2cf2729ec7 Mon Sep 17 00:00:00 2001 From: tvlenin Date: Fri, 17 Jan 2020 08:37:35 -0600 Subject: [PATCH 085/188] Fix format problems --- gst-libs/gst/r2inference/gstinferenceclassification.c | 5 +++-- gst-libs/gst/r2inference/gstinferencepostprocess.h | 3 ++- gst-libs/gst/r2inference/gstvideoinference.c | 2 -- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/gst-libs/gst/r2inference/gstinferenceclassification.c b/gst-libs/gst/r2inference/gstinferenceclassification.c index 31044264..da9a6cd0 100644 --- a/gst-libs/gst/r2inference/gstinferenceclassification.c +++ b/gst-libs/gst/r2inference/gstinferenceclassification.c @@ -183,11 +183,12 @@ gst_inference_classification_to_string (GstInferenceClassification * self, "%*s Class : %d\n" "%*s Label : %s\n" "%*s Probability : %f\n" - "%*s Num_classes : %d\n" + "%*s Classes : %d\n" "%*s}", indent, "", self->class_id, indent, "", self->class_label, - indent, "", self->class_prob, indent, "", self->num_classes, indent, ""); + indent, "", self->probabilities[0], indent, "", self->num_classes, indent, + ""); } static void diff --git a/gst-libs/gst/r2inference/gstinferencepostprocess.h b/gst-libs/gst/r2inference/gstinferencepostprocess.h index caa9dee7..7d28974a 100644 --- a/gst-libs/gst/r2inference/gstinferencepostprocess.h +++ b/gst-libs/gst/r2inference/gstinferencepostprocess.h @@ -77,7 +77,8 @@ gboolean gst_create_boxes_float (GstVideoInference * vi, * \param labels_list List with all possible lables * \param num_labels The number of posibble labels */ -GstInferencePrediction *gst_create_prediction_from_box (GstVideoInference * vi, BBox * box, gchar **labels_list, gint num_labels); +GstInferencePrediction *gst_create_prediction_from_box (GstVideoInference * vi, BBox * box, + gchar **labels_list, gint num_labels, const gdouble * probabilities); /** * \brief Create Classification from prediction data diff --git a/gst-libs/gst/r2inference/gstvideoinference.c b/gst-libs/gst/r2inference/gstvideoinference.c index 2744b99a..7d114cda 100644 --- a/gst-libs/gst/r2inference/gstvideoinference.c +++ b/gst-libs/gst/r2inference/gstvideoinference.c @@ -1226,11 +1226,9 @@ gst_video_inference_finalize (GObject * object) priv->sink_model_data = NULL; g_free (priv->model_location); priv->model_location = NULL; - priv->labels = NULL; priv->labels_list = NULL; - g_clear_object (&priv->backend); G_OBJECT_CLASS (gst_video_inference_parent_class)->finalize (object); From bfbe11f52b94f47704d4078a82749b67c7968d2d Mon Sep 17 00:00:00 2001 From: tvlenin Date: Fri, 17 Jan 2020 10:03:53 -0600 Subject: [PATCH 086/188] Add all probabilities to new meta --- ext/r2inference/gsttinyyolov2.c | 10 ++++++--- .../r2inference/gstinferenceclassification.c | 3 +-- .../gst/r2inference/gstinferencepostprocess.c | 21 ++++++++++++------- .../gst/r2inference/gstinferencepostprocess.h | 2 +- 4 files changed, 23 insertions(+), 13 deletions(-) diff --git a/ext/r2inference/gsttinyyolov2.c b/ext/r2inference/gsttinyyolov2.c index 77feb46d..b2c27f84 100644 --- a/ext/r2inference/gsttinyyolov2.c +++ b/ext/r2inference/gsttinyyolov2.c @@ -316,6 +316,8 @@ gst_tinyyolov2_postprocess_old (GstVideoInference * vi, GstVideoInfo * info_model, gboolean * valid_prediction) { GstTinyyolov2 *tinyyolov2; + gdouble *probabilities = NULL; + GstDetectionMeta *detect_meta = (GstDetectionMeta *) meta_model; g_return_val_if_fail (detect_meta, FALSE); @@ -326,7 +328,7 @@ gst_tinyyolov2_postprocess_old (GstVideoInference * vi, gst_create_boxes (vi, prediction, valid_prediction, &detect_meta->boxes, &detect_meta->num_boxes, tinyyolov2->obj_thresh, - tinyyolov2->prob_thresh, tinyyolov2->iou_thresh); + tinyyolov2->prob_thresh, tinyyolov2->iou_thresh, &probabilities); gst_inference_print_boxes (vi, gst_tinyyolov2_debug_category, detect_meta); @@ -345,6 +347,7 @@ gst_tinyyolov2_postprocess_new (GstVideoInference * vi, GstInferenceMeta *imeta = NULL; BBox *boxes = NULL; gint num_boxes, i; + gdouble *probabilities = NULL; g_return_val_if_fail (vi != NULL, FALSE); g_return_val_if_fail (meta_model != NULL, FALSE); @@ -358,7 +361,7 @@ gst_tinyyolov2_postprocess_new (GstVideoInference * vi, /* Create boxes from prediction data */ gst_create_boxes (vi, prediction, valid_prediction, &boxes, &num_boxes, tinyyolov2->obj_thresh, - tinyyolov2->prob_thresh, tinyyolov2->iou_thresh); + tinyyolov2->prob_thresh, tinyyolov2->iou_thresh, &probabilities); GST_LOG_OBJECT (tinyyolov2, "Number of predictions: %d", num_boxes); @@ -370,7 +373,8 @@ gst_tinyyolov2_postprocess_new (GstVideoInference * vi, for (i = 0; i < num_boxes; i++) { GstInferencePrediction *pred = - gst_create_prediction_from_box (vi, &boxes[i], labels_list, num_labels); + gst_create_prediction_from_box (vi, &boxes[i], labels_list, num_labels, + probabilities); gst_inference_prediction_append (imeta->prediction, pred); } diff --git a/gst-libs/gst/r2inference/gstinferenceclassification.c b/gst-libs/gst/r2inference/gstinferenceclassification.c index da9a6cd0..a6c355eb 100644 --- a/gst-libs/gst/r2inference/gstinferenceclassification.c +++ b/gst-libs/gst/r2inference/gstinferenceclassification.c @@ -187,8 +187,7 @@ gst_inference_classification_to_string (GstInferenceClassification * self, "%*s}", indent, "", self->class_id, indent, "", self->class_label, - indent, "", self->probabilities[0], indent, "", self->num_classes, indent, - ""); + indent, "", self->class_prob, indent, "", self->num_classes, indent, ""); } static void diff --git a/gst-libs/gst/r2inference/gstinferencepostprocess.c b/gst-libs/gst/r2inference/gstinferencepostprocess.c index bb870b94..747afe43 100644 --- a/gst-libs/gst/r2inference/gstinferencepostprocess.c +++ b/gst-libs/gst/r2inference/gstinferencepostprocess.c @@ -26,6 +26,8 @@ #define TOTAL_BOXES_5 845 #define TOTAL_BOXES_15 2535 +#define TOTAL_CLASSES 20 + /* Functions declaration*/ static gdouble gst_intersection_over_union (BBox box_1, BBox box_2); @@ -37,7 +39,7 @@ static void gst_box_to_pixels (BBox * normalized_box, gint row, gint col, static gdouble gst_sigmoid (gdouble x); static void gst_get_boxes_from_prediction (gfloat obj_thresh, gfloat prob_thresh, gpointer prediction, BBox * boxes, gint * elements, - gint grid_h, gint grid_w, gint boxes_size); + gint grid_h, gint grid_w, gint boxes_size, gdouble * probabilities); static void gst_get_boxes_from_prediction_float (gfloat obj_thresh, gfloat prob_thresh, gpointer prediction, BBox * boxes, gint * elements, gint total_boxes); @@ -169,7 +171,7 @@ gst_box_to_pixels (BBox * normalized_box, gint row, gint col, gint box) static void gst_get_boxes_from_prediction (gfloat obj_thresh, gfloat prob_thresh, gpointer prediction, BBox * boxes, gint * elements, gint grid_h, - gint grid_w, gint boxes_size) + gint grid_w, gint boxes_size, gdouble * probabilities) { gint i, j, c, b; gint index; @@ -178,7 +180,7 @@ gst_get_boxes_from_prediction (gfloat obj_thresh, gfloat prob_thresh, gint max_class_prob_index; gint counter = 0; gint box_dim = 5; - gint classes = 20; + gint classes = TOTAL_CLASSES; g_return_if_fail (boxes != NULL); g_return_if_fail (elements != NULL); @@ -197,6 +199,7 @@ gst_get_boxes_from_prediction (gfloat obj_thresh, gfloat prob_thresh, max_class_prob_index = 0; for (c = 0; c < classes; c++) { cur_class_prob = ((gfloat *) prediction)[index + box_dim + c]; + probabilities[c] = cur_class_prob; if (cur_class_prob > max_class_prob) { max_class_prob = cur_class_prob; max_class_prob_index = c; @@ -226,12 +229,14 @@ gst_get_boxes_from_prediction (gfloat obj_thresh, gfloat prob_thresh, gboolean gst_create_boxes (GstVideoInference * vi, const gpointer prediction, gboolean * valid_prediction, BBox ** resulting_boxes, - gint * elements, gfloat obj_thresh, gfloat prob_thresh, gfloat iou_thresh) + gint * elements, gfloat obj_thresh, gfloat prob_thresh, gfloat iou_thresh, + gdouble ** probabilities) { gint grid_h = 13; gint grid_w = 13; gint boxes_size = 5; BBox boxes[TOTAL_BOXES_5]; + gdouble classes_prob[TOTAL_CLASSES]; *elements = 0; g_return_val_if_fail (vi != NULL, FALSE); @@ -241,17 +246,19 @@ gst_create_boxes (GstVideoInference * vi, const gpointer prediction, g_return_val_if_fail (elements != NULL, FALSE); gst_get_boxes_from_prediction (obj_thresh, prob_thresh, prediction, boxes, - elements, grid_h, grid_w, boxes_size); + elements, grid_h, grid_w, boxes_size, classes_prob); gst_remove_duplicated_boxes (iou_thresh, boxes, elements); *resulting_boxes = g_malloc (*elements * sizeof (BBox)); + *probabilities = g_malloc (TOTAL_CLASSES * sizeof (gdouble)); memcpy (*resulting_boxes, boxes, *elements * sizeof (BBox)); + memcpy (*probabilities, classes_prob, TOTAL_CLASSES * sizeof (gdouble)); return TRUE; } GstInferencePrediction * gst_create_prediction_from_box (GstVideoInference * vi, BBox * box, - gchar ** labels_list, gint num_labels) + gchar ** labels_list, gint num_labels, const gdouble * probabilities) { GstInferencePrediction *predict = NULL; GstInferenceClassification *c = NULL; @@ -269,7 +276,7 @@ gst_create_prediction_from_box (GstVideoInference * vi, BBox * box, label = labels_list[box->label]; } c = gst_inference_classification_new_full (box->label, box->prob, label, - num_labels, &box->prob, labels_list); + num_labels, probabilities, labels_list); gst_inference_prediction_append_classification (predict, c); return predict; diff --git a/gst-libs/gst/r2inference/gstinferencepostprocess.h b/gst-libs/gst/r2inference/gstinferencepostprocess.h index 7d28974a..cb23285f 100644 --- a/gst-libs/gst/r2inference/gstinferencepostprocess.h +++ b/gst-libs/gst/r2inference/gstinferencepostprocess.h @@ -50,7 +50,7 @@ G_BEGIN_DECLS */ gboolean gst_create_boxes (GstVideoInference * vi, const gpointer prediction, gboolean * valid_prediction, BBox ** resulting_boxes, - gint * elements, gfloat obj_thresh, gfloat prob_thresh, gfloat iou_thresh); + gint * elements, gfloat obj_thresh, gfloat prob_thresh, gfloat iou_thresh, gdouble ** probabilities); /** * \brief Fill all the detection meta with the boxes From 79742c6eca8674d6745e4622796abb92d73da2c0 Mon Sep 17 00:00:00 2001 From: tvlenin Date: Fri, 17 Jan 2020 11:49:26 -0600 Subject: [PATCH 087/188] Remove memcopy of probabilities --- gst-libs/gst/r2inference/gstinferencepostprocess.c | 9 +++++---- gst-libs/gst/r2inference/gstinferencepostprocess.h | 4 ++-- gst-libs/gst/r2inference/gstvideoinference.c | 2 ++ 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/gst-libs/gst/r2inference/gstinferencepostprocess.c b/gst-libs/gst/r2inference/gstinferencepostprocess.c index 747afe43..4f1e88cf 100644 --- a/gst-libs/gst/r2inference/gstinferencepostprocess.c +++ b/gst-libs/gst/r2inference/gstinferencepostprocess.c @@ -236,23 +236,23 @@ gst_create_boxes (GstVideoInference * vi, const gpointer prediction, gint grid_w = 13; gint boxes_size = 5; BBox boxes[TOTAL_BOXES_5]; - gdouble classes_prob[TOTAL_CLASSES]; *elements = 0; + *probabilities = g_malloc (TOTAL_CLASSES * sizeof (gdouble)); g_return_val_if_fail (vi != NULL, FALSE); g_return_val_if_fail (prediction != NULL, FALSE); g_return_val_if_fail (valid_prediction != NULL, FALSE); g_return_val_if_fail (resulting_boxes != NULL, FALSE); g_return_val_if_fail (elements != NULL, FALSE); + g_return_val_if_fail (probabilities != NULL, FALSE); gst_get_boxes_from_prediction (obj_thresh, prob_thresh, prediction, boxes, - elements, grid_h, grid_w, boxes_size, classes_prob); + elements, grid_h, grid_w, boxes_size, *probabilities); gst_remove_duplicated_boxes (iou_thresh, boxes, elements); *resulting_boxes = g_malloc (*elements * sizeof (BBox)); - *probabilities = g_malloc (TOTAL_CLASSES * sizeof (gdouble)); memcpy (*resulting_boxes, boxes, *elements * sizeof (BBox)); - memcpy (*probabilities, classes_prob, TOTAL_CLASSES * sizeof (gdouble)); + return TRUE; } @@ -265,6 +265,7 @@ gst_create_prediction_from_box (GstVideoInference * vi, BBox * box, gchar *label = NULL; g_return_val_if_fail (vi != NULL, NULL); g_return_val_if_fail (box != NULL, NULL); + g_return_val_if_fail (probabilities != NULL, NULL); predict = gst_inference_prediction_new (); predict->bbox.x = box->x; diff --git a/gst-libs/gst/r2inference/gstinferencepostprocess.h b/gst-libs/gst/r2inference/gstinferencepostprocess.h index cb23285f..779f8286 100644 --- a/gst-libs/gst/r2inference/gstinferencepostprocess.h +++ b/gst-libs/gst/r2inference/gstinferencepostprocess.h @@ -77,8 +77,8 @@ gboolean gst_create_boxes_float (GstVideoInference * vi, * \param labels_list List with all possible lables * \param num_labels The number of posibble labels */ -GstInferencePrediction *gst_create_prediction_from_box (GstVideoInference * vi, BBox * box, - gchar **labels_list, gint num_labels, const gdouble * probabilities); +GstInferencePrediction *gst_create_prediction_from_box (GstVideoInference * vi, + BBox * box, gchar **labels_list, gint num_labels, const gdouble * probabilities); /** * \brief Create Classification from prediction data diff --git a/gst-libs/gst/r2inference/gstvideoinference.c b/gst-libs/gst/r2inference/gstvideoinference.c index 7d114cda..a36ad759 100644 --- a/gst-libs/gst/r2inference/gstvideoinference.c +++ b/gst-libs/gst/r2inference/gstvideoinference.c @@ -1226,7 +1226,9 @@ gst_video_inference_finalize (GObject * object) priv->sink_model_data = NULL; g_free (priv->model_location); priv->model_location = NULL; + g_free (priv->labels); priv->labels = NULL; + g_free (priv->labels_list); priv->labels_list = NULL; g_clear_object (&priv->backend); From eb62c8670de2399e400a3657ae57d002fd99d99b Mon Sep 17 00:00:00 2001 From: tvlenin Date: Fri, 17 Jan 2020 12:00:57 -0600 Subject: [PATCH 088/188] Fix assignation before pointer guard --- gst-libs/gst/r2inference/gstinferencepostprocess.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/gst-libs/gst/r2inference/gstinferencepostprocess.c b/gst-libs/gst/r2inference/gstinferencepostprocess.c index 4f1e88cf..054c95dc 100644 --- a/gst-libs/gst/r2inference/gstinferencepostprocess.c +++ b/gst-libs/gst/r2inference/gstinferencepostprocess.c @@ -236,8 +236,6 @@ gst_create_boxes (GstVideoInference * vi, const gpointer prediction, gint grid_w = 13; gint boxes_size = 5; BBox boxes[TOTAL_BOXES_5]; - *elements = 0; - *probabilities = g_malloc (TOTAL_CLASSES * sizeof (gdouble)); g_return_val_if_fail (vi != NULL, FALSE); g_return_val_if_fail (prediction != NULL, FALSE); @@ -246,6 +244,9 @@ gst_create_boxes (GstVideoInference * vi, const gpointer prediction, g_return_val_if_fail (elements != NULL, FALSE); g_return_val_if_fail (probabilities != NULL, FALSE); + *elements = 0; + *probabilities = g_malloc (TOTAL_CLASSES * sizeof (gdouble)); + gst_get_boxes_from_prediction (obj_thresh, prob_thresh, prediction, boxes, elements, grid_h, grid_w, boxes_size, *probabilities); gst_remove_duplicated_boxes (iou_thresh, boxes, elements); From 0e1194fab74d5e57c61303063555f43a93c1bb55 Mon Sep 17 00:00:00 2001 From: tvlenin Date: Tue, 24 Dec 2019 09:04:42 -0600 Subject: [PATCH 089/188] Add qos to FALSE by default, to avoid buffer dropping --- gst/inferencecrop/videocrop.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gst/inferencecrop/videocrop.cc b/gst/inferencecrop/videocrop.cc index db80e9ae..9b493d05 100644 --- a/gst/inferencecrop/videocrop.cc +++ b/gst/inferencecrop/videocrop.cc @@ -90,7 +90,8 @@ VideoCrop::UpdateElement (GstElement *element, "top", top, "bottom", bottom, "left", left, - "right", right, NULL); + "right", right, + "qos",FALSE, NULL); } GstPad * From c38b79ab11a99531f27abfc81020e807b4af68d7 Mon Sep 17 00:00:00 2001 From: tvlenin Date: Tue, 24 Dec 2019 09:05:36 -0600 Subject: [PATCH 090/188] Support the new inference meta --- gst/inferencecrop/gstdetectioncrop.cc | 258 +++++++++++++++++++------- 1 file changed, 187 insertions(+), 71 deletions(-) diff --git a/gst/inferencecrop/gstdetectioncrop.cc b/gst/inferencecrop/gstdetectioncrop.cc index f0daaa25..f8a871ac 100644 --- a/gst/inferencecrop/gstdetectioncrop.cc +++ b/gst/inferencecrop/gstdetectioncrop.cc @@ -72,21 +72,28 @@ static void gst_detection_crop_set_caps (GstPad *pad, GParamSpec *unused, GstDetectionCrop *self); static GstPadProbeReturn gst_detection_crop_new_buffer (GstPad *pad, GstPadProbeInfo *info, GstDetectionCrop *self); +static gint gst_detection_crop_find_predictions (GstDetectionCrop *self, + gint *crop_class, GstInferenceMeta *meta, GList **list, Prediction *pred); static gint gst_detection_crop_find_by_index (GstDetectionCrop *self, guint crop_index, GstDetectionMeta *meta); static gint gst_detection_crop_find_by_class (GstDetectionCrop *self, gint crop_class, gboolean biggest_object, GstDetectionMeta *meta); static gint gst_detection_crop_find_index (GstDetectionCrop *self, + <<< <<< < HEAD guint crop_index, gint crop_class, gboolean biggest_object, GstDetectionMeta *meta); +== == == = + guint crop_index, gint * crop_class, GstInferenceMeta * inference_meta, + GstDetectionMeta * detection_meta, GList **list, Prediction * pred); +>>> >>> > Support the new inference meta #define PROP_CROP_INDEX_DEFAULT 0 #define PROP_CROP_INDEX_MAX G_MAXUINT #define PROP_CROP_INDEX_MIN 0 -#define PROP_CROP_CLASS_DEFAULT -1 +#define PROP_CROP_CLASS_DEFAULT -2 #define PROP_CROP_CLASS_MAX G_MAXINT -#define PROP_CROP_CLASS_MIN -1 +#define PROP_CROP_CLASS_MIN -2 #define PROP_CROP_BIGGEST_DEFAULT FALSE @@ -100,6 +107,7 @@ enum { struct _GstDetectionCrop { GstBin parent; + GstPad *pad; CropElement *element; guint crop_index; gint crop_class; @@ -177,6 +185,7 @@ gst_detection_crop_init (GstDetectionCrop *self) { GstElement *element; GstPad *sinkpad, *sinkgpad, *srcpad, *srcgpad; + self->pad = NULL; self->element = new VideoCrop (); self->crop_index = PROP_CROP_INDEX_DEFAULT; self->crop_class = PROP_CROP_CLASS_DEFAULT; @@ -195,6 +204,8 @@ gst_detection_crop_init (GstDetectionCrop *self) { sinkpad = self->element->GetSinkPad (); g_return_if_fail (sinkpad); + self->pad = GST_PAD(gst_object_ref(sinkpad)); + sinkgpad = gst_ghost_pad_new ("sink", sinkpad); gst_pad_set_active (sinkgpad, TRUE); gst_element_add_pad (GST_ELEMENT (self), sinkgpad); @@ -220,6 +231,8 @@ gst_detection_crop_finalize (GObject *object) { GstDetectionCrop *self = GST_DETECTION_CROP (object); delete (self->element); + gst_object_unref (self->pad); + self->pad = NULL; G_OBJECT_CLASS (gst_detection_crop_parent_class)->finalize (object); } @@ -377,10 +390,13 @@ gst_detection_crop_find_by_class (GstDetectionCrop *self, gint crop_class, GstDetectionMeta *meta) { gint i; gint ret = -1; + <<< <<< < HEAD gint biggest = -1; gint box_size = -1; - g_return_val_if_fail (self, -1); + == == == = + >>> >>> > Support the new inference meta + g_return_val_if_fail (self, -1); g_return_val_if_fail (meta, -1); for (i = 0; i < meta->num_boxes; ++i) { @@ -404,89 +420,189 @@ gst_detection_crop_find_by_class (GstDetectionCrop *self, gint crop_class, } static gint +<<< <<< < HEAD gst_detection_crop_find_index (GstDetectionCrop *self, guint crop_index, gint crop_class, gboolean biggest_object, GstDetectionMeta *meta) { - g_return_val_if_fail (self, -1); - g_return_val_if_fail (meta, -1); + == == == = + gst_detection_crop_find_predictions (GstDetectionCrop * self, gint * crop_class, + GstInferenceMeta * meta, GList **list, Prediction * pred) { + gint i; + //gint ret = 3; + + >>> >>> > Support the new inference meta + g_return_val_if_fail (self, -1); + g_return_val_if_fail (meta, -1); + g_return_val_if_fail (list, -1); + for (i = 0; i < pred->num_predictions ; ++i) { + Prediction *predict = &pred->predictions[i]; + gst_detection_crop_find_predictions (self, crop_class, meta, list, predict ); + } + if (TRUE == pred->enabled) { + *list = g_list_append (*list, pred); + *crop_class = *crop_class + 1; + } + g_print("**************%d\n", *crop_class); - if (-1 == crop_class) { - return gst_detection_crop_find_by_index (self, crop_index, meta); - } else { - return gst_detection_crop_find_by_class (self, crop_class, biggest_object, - meta); - } -} + /*if (-1 == *crop_class) { + GST_LOG_OBJECT (self, "No valid class detected"); + }*/ -static GstPadProbeReturn -gst_detection_crop_new_buffer (GstPad *pad, GstPadProbeInfo *info, - GstDetectionCrop *self) { - GstBuffer *buffer; - GstDetectionMeta *meta; - guint crop_index; - gint crop_class; - gint crop_width_ratio; - gint crop_height_ratio; - gint requested_index; - gboolean biggest_object; - BBox box; - GstPadProbeReturn ret = GST_PAD_PROBE_DROP; + return *crop_class; + } - GST_OBJECT_LOCK (self); - crop_index = self->crop_index; - crop_class = self->crop_class; - crop_width_ratio = self->width_ratio; - crop_height_ratio = self->height_ratio; - biggest_object = self->biggest_object; - GST_OBJECT_UNLOCK (self); + static gint + gst_detection_crop_find_index (GstDetectionCrop * self, guint crop_index, + gint * crop_class, GstInferenceMeta * inference_meta, + GstDetectionMeta * detection_meta, GList **list, Prediction * pred) { + g_return_val_if_fail (self, -1); + g_return_val_if_fail (inference_meta, -1); + g_return_val_if_fail (detection_meta, -1); + g_return_val_if_fail (list, -1); + + if (-2 == *crop_class) { + *crop_class = -1; + return gst_detection_crop_find_predictions (self, crop_class, inference_meta, + list, pred); + } else if (-1 == *crop_class) { + return gst_detection_crop_find_by_index (self, crop_index, detection_meta); + } else { + <<< <<< < HEAD + return gst_detection_crop_find_by_class (self, crop_class, biggest_object, + meta); + == == == = + return gst_detection_crop_find_by_class (self, *crop_class, detection_meta); + >>> >>> > Support the new inference meta + } + } - buffer = gst_pad_probe_info_get_buffer (info); - meta = - (GstDetectionMeta *) gst_buffer_get_meta (buffer, - GST_DETECTION_META_API_TYPE); + static GstPadProbeReturn + gst_detection_crop_new_buffer (GstPad * pad, GstPadProbeInfo * info, + GstDetectionCrop * self) { + GstBuffer *buffer; + GstDetectionMeta *detection_meta; + GstInferenceMeta *inference_meta; + guint crop_index; + gint crop_class; + gint crop_class_inference; + gint crop_width_ratio; + gint crop_height_ratio; + <<< <<< < HEAD + gint requested_index; + gboolean biggest_object; + == == == = + gint requested_inference_index; + gint requested_detection_index; + >>> >>> > Support the new inference meta + BBox box; + GstPadProbeReturn ret = GST_PAD_PROBE_DROP; + GList *list = NULL; + GList *iter = NULL; + + GST_OBJECT_LOCK (self); + crop_index = self->crop_index; + crop_class = self->crop_class; + crop_class_inference = self->crop_class; + crop_width_ratio = self->width_ratio; + crop_height_ratio = self->height_ratio; + biggest_object = self->biggest_object; + GST_OBJECT_UNLOCK (self); + + buffer = gst_pad_probe_info_get_buffer (info); + + inference_meta = + (GstInferenceMeta *) gst_buffer_get_meta (buffer, + GST_INFERENCE_META_API_TYPE); + detection_meta = + (GstDetectionMeta *) gst_buffer_get_meta (buffer, + GST_DETECTION_META_API_TYPE); + + if (NULL == inference_meta && NULL == detection_meta) { + GST_LOG_OBJECT (self, "No meta found, dropping buffer"); + goto out; + } - if (NULL == meta) { - GST_LOG_OBJECT (self, "No meta found, dropping buffer"); - goto out; - } + if (inference_meta->prediction <= 0 ) { + GST_LOG_OBJECT (self, "Inference meta has no valid objects"); + goto out; + } else if (-2 == crop_class_inference) { + requested_inference_index = + gst_detection_crop_find_index (self, crop_index, &crop_class, inference_meta, + detection_meta, &list, inference_meta->prediction); + if (0 == requested_inference_index) { + goto out; + } + } - if (meta->num_boxes <= 0) { - GST_LOG_OBJECT (self, "Meta has no valid objects"); - goto out; - } + <<< <<< < HEAD + requested_index = + gst_detection_crop_find_index (self, crop_index, crop_class, biggest_object, + meta); + if (-1 == requested_index) { + == == == = + if (detection_meta->num_boxes <= 0 ) { + GST_LOG_OBJECT (self, "Detection meta has no valid objects"); + >>> >>> > Support the new inference meta + goto out; + } else { + requested_detection_index = + gst_detection_crop_find_index (self, crop_index, &crop_class, inference_meta, + detection_meta, &list, inference_meta->prediction); + if (-1 == requested_detection_index) { + goto out; + } + } - requested_index = - gst_detection_crop_find_index (self, crop_index, crop_class, biggest_object, - meta); - if (-1 == requested_index) { - goto out; - } +//Select the new inference meta + if (-2 == crop_class_inference) { + for (iter = list; iter != NULL; iter = g_list_next(iter)) { + Prediction *pred = (Prediction *)iter->data; + if ( 0 != pred->id) { + GstBuffer *croped_buffer; + box = *pred->box; + GST_LOG_OBJECT (self, "BBox: %fx%fx%fx%f", box.x, box.y, box.width, + box.height); + self->element->SetBoundingBox ((gint) box.x, (gint) box.y, (gint) box.width, + (gint) box.height, (gint) crop_width_ratio, (gint) crop_height_ratio); + croped_buffer = gst_buffer_copy (buffer); + + if (GST_FLOW_OK != gst_pad_chain(self->pad, croped_buffer)) { + GST_ELEMENT_ERROR(self, CORE, FAILED, + ("Failed to push a new buffer into crop element"), (NULL)); + } + } - box = meta->boxes[requested_index]; - GST_LOG_OBJECT (self, "BBox: %fx%fx%fx%f", box.x, box.y, box.width, - box.height); - self->element->SetBoundingBox ((gint) box.x, (gint) box.y, (gint) box.width, - (gint) box.height, (gint) crop_width_ratio, (gint) crop_height_ratio); + } + ret = GST_PAD_PROBE_DROP; + goto out; + }//Use old detection meta + else { + box = detection_meta->boxes[requested_detection_index]; + GST_LOG_OBJECT (self, "BBox: %fx%fx%fx%f", box.x, box.y, box.width, + box.height); + self->element->SetBoundingBox ((gint) box.x, (gint) box.y, (gint) box.width, + (gint) box.height, (gint) crop_width_ratio, (gint) crop_height_ratio); + ret = GST_PAD_PROBE_OK; + } - ret = GST_PAD_PROBE_OK; out: - return ret; -} + return ret; + } -static gboolean -plugin_init (GstPlugin *plugin) { - gboolean ret = TRUE; + static gboolean + plugin_init (GstPlugin * plugin) { + gboolean ret = TRUE; - ret = - gst_element_register (plugin, "detectioncrop", GST_RANK_NONE, - GST_TYPE_DETECTION_CROP); + ret = + gst_element_register (plugin, "detectioncrop", GST_RANK_NONE, + GST_TYPE_DETECTION_CROP); - return ret; -} + return ret; + } -GST_PLUGIN_DEFINE (GST_VERSION_MAJOR, - GST_VERSION_MINOR, - inferencecrop, - "Crops an incoming image based on an inference prediction bounding box", - plugin_init, VERSION, "LGPL", PACKAGE_NAME, GST_PACKAGE_ORIGIN) + GST_PLUGIN_DEFINE (GST_VERSION_MAJOR, + GST_VERSION_MINOR, + inferencecrop, + "Crops an incoming image based on an inference prediction bounding box", + plugin_init, VERSION, "LGPL", PACKAGE_NAME, GST_PACKAGE_ORIGIN) From 9bf1ef13e803e2ce3ad0a6d04b0fc63f396c9ee9 Mon Sep 17 00:00:00 2001 From: tvlenin Date: Tue, 7 Jan 2020 13:51:38 -0600 Subject: [PATCH 091/188] Add support to use new inference meta --- gst/inferencecrop/gstdetectioncrop.cc | 353 +++++++++++--------------- 1 file changed, 143 insertions(+), 210 deletions(-) diff --git a/gst/inferencecrop/gstdetectioncrop.cc b/gst/inferencecrop/gstdetectioncrop.cc index f8a871ac..dfb9bf25 100644 --- a/gst/inferencecrop/gstdetectioncrop.cc +++ b/gst/inferencecrop/gstdetectioncrop.cc @@ -40,7 +40,7 @@ #ifdef HAVE_CONFIG_H #include "config.h" #endif - +#include #include "gstdetectioncrop.h" #include "gst/r2inference/gstinferencemeta.h" @@ -72,20 +72,16 @@ static void gst_detection_crop_set_caps (GstPad *pad, GParamSpec *unused, GstDetectionCrop *self); static GstPadProbeReturn gst_detection_crop_new_buffer (GstPad *pad, GstPadProbeInfo *info, GstDetectionCrop *self); + + static gint gst_detection_crop_find_predictions (GstDetectionCrop *self, - gint *crop_class, GstInferenceMeta *meta, GList **list, Prediction *pred); + gint *crop_class, GstInferenceMeta *meta, GList **list, GstInferencePrediction *pred); static gint gst_detection_crop_find_by_index (GstDetectionCrop *self, guint crop_index, GstDetectionMeta *meta); static gint gst_detection_crop_find_by_class (GstDetectionCrop *self, - gint crop_class, gboolean biggest_object, GstDetectionMeta *meta); + gint crop_class, GstDetectionMeta *meta); static gint gst_detection_crop_find_index (GstDetectionCrop *self, - <<< <<< < HEAD - guint crop_index, gint crop_class, gboolean biggest_object, - GstDetectionMeta *meta); -== == == = - guint crop_index, gint * crop_class, GstInferenceMeta * inference_meta, - GstDetectionMeta * detection_meta, GList **list, Prediction * pred); ->>> >>> > Support the new inference meta + guint crop_index, gint *crop_class, GstInferenceMeta *inference_meta, GstDetectionMeta *detection_meta, GList **list, GstInferencePrediction *pred); #define PROP_CROP_INDEX_DEFAULT 0 #define PROP_CROP_INDEX_MAX G_MAXUINT @@ -95,14 +91,11 @@ static gint gst_detection_crop_find_index (GstDetectionCrop *self, #define PROP_CROP_CLASS_MAX G_MAXINT #define PROP_CROP_CLASS_MIN -2 -#define PROP_CROP_BIGGEST_DEFAULT FALSE - enum { PROP_0, PROP_CROP_INDEX, PROP_CROP_CLASS, PROP_CROP_ASPECT_RATIO, - PROP_CROP_BIGGEST, }; struct _GstDetectionCrop { @@ -113,7 +106,6 @@ struct _GstDetectionCrop { gint crop_class; gint width_ratio; gint height_ratio; - gboolean biggest_object; }; struct _GstDetectionCropClass { @@ -171,27 +163,19 @@ gst_detection_crop_class_init (GstDetectionCropClass *klass) { G_MAXINT, 1, PROP_CROP_RATIO_DEFAULT_WIDTH, PROP_CROP_RATIO_DEFAULT_HEIGHT, G_PARAM_READWRITE )); - g_object_class_install_property (object_class, PROP_CROP_BIGGEST, - g_param_spec_boolean ("biggest-object", "Biggest Object", - "Crop the biggest object detected of the class given in crop-class." - "The biggest object usually implies the nearest object to the camera." - "This only applies if crop-class is non-negative.", - PROP_CROP_BIGGEST_DEFAULT, - G_PARAM_READWRITE )); } static void gst_detection_crop_init (GstDetectionCrop *self) { GstElement *element; GstPad *sinkpad, *sinkgpad, *srcpad, *srcgpad; - + self->pad = NULL; self->element = new VideoCrop (); self->crop_index = PROP_CROP_INDEX_DEFAULT; self->crop_class = PROP_CROP_CLASS_DEFAULT; self->width_ratio = PROP_CROP_RATIO_DEFAULT_WIDTH; self->height_ratio = PROP_CROP_RATIO_DEFAULT_HEIGHT; - self->biggest_object = PROP_CROP_BIGGEST_DEFAULT; if (FALSE == self->element->Validate ()) { const std::string factory = self->element->GetFactory (); GST_ERROR_OBJECT (self, "Unable to find element %s", factory.c_str ()); @@ -263,11 +247,6 @@ gst_detection_crop_set_property (GObject *object, guint property_id, } GST_OBJECT_UNLOCK (self); break; - case PROP_CROP_BIGGEST: - GST_OBJECT_LOCK (self); - self->biggest_object = g_value_get_boolean (value); - GST_OBJECT_UNLOCK (self); - break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); break; @@ -297,11 +276,6 @@ gst_detection_crop_get_property (GObject *object, guint property_id, gst_value_set_fraction (value, self->width_ratio, self->height_ratio); GST_OBJECT_UNLOCK (self); break; - case PROP_CROP_BIGGEST: - GST_OBJECT_LOCK (self); - g_value_set_boolean (value, self->biggest_object); - GST_OBJECT_UNLOCK (self); - break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); break; @@ -386,29 +360,16 @@ gst_detection_crop_find_by_index (GstDetectionCrop *self, guint crop_index, static gint gst_detection_crop_find_by_class (GstDetectionCrop *self, gint crop_class, - gboolean biggest_object, GstDetectionMeta *meta) { gint i; gint ret = -1; - <<< <<< < HEAD - gint biggest = -1; - gint box_size = -1; - - == == == = - >>> >>> > Support the new inference meta - g_return_val_if_fail (self, -1); + g_return_val_if_fail (self, -1); g_return_val_if_fail (meta, -1); for (i = 0; i < meta->num_boxes; ++i) { if (meta->boxes[i].label == crop_class) { - box_size = meta->boxes[i].width * meta->boxes[i].height; - if (box_size > biggest) { - biggest = box_size; - ret = i; - if (false == biggest_object) { - break; - } - } + ret = i; + break; } } @@ -420,189 +381,161 @@ gst_detection_crop_find_by_class (GstDetectionCrop *self, gint crop_class, } static gint -<<< <<< < HEAD -gst_detection_crop_find_index (GstDetectionCrop *self, guint crop_index, - gint crop_class, gboolean biggest_object, GstDetectionMeta *meta) { - == == == = - gst_detection_crop_find_predictions (GstDetectionCrop * self, gint * crop_class, - GstInferenceMeta * meta, GList **list, Prediction * pred) { - gint i; - //gint ret = 3; - - >>> >>> > Support the new inference meta - g_return_val_if_fail (self, -1); - g_return_val_if_fail (meta, -1); - g_return_val_if_fail (list, -1); - for (i = 0; i < pred->num_predictions ; ++i) { - Prediction *predict = &pred->predictions[i]; - gst_detection_crop_find_predictions (self, crop_class, meta, list, predict ); - } - if (TRUE == pred->enabled) { - *list = g_list_append (*list, pred); - *crop_class = *crop_class + 1; - } - g_print("**************%d\n", *crop_class); +gst_detection_crop_find_predictions (GstDetectionCrop *self, gint *crop_class, + GstInferenceMeta *meta, GList **list,GstInferencePrediction *pred) { + guint i; - /*if (-1 == *crop_class) { - GST_LOG_OBJECT (self, "No valid class detected"); - }*/ + g_return_val_if_fail (self, -1); + g_return_val_if_fail (meta, -1); + g_return_val_if_fail (list, -1); + g_return_val_if_fail (crop_class, -1); + g_return_val_if_fail (pred, -1); - return *crop_class; + for (i = 0; i < g_node_n_children(pred->predictions) ; i++) { + GstInferencePrediction *predict = (GstInferencePrediction*)g_node_nth_child (pred->predictions,i)->data; + gst_detection_crop_find_predictions (self,crop_class,meta,list,predict ); + } + if(FALSE == G_NODE_IS_ROOT(pred->predictions)){ + *list = g_list_append (*list,pred); + *crop_class=*crop_class+1; } - static gint - gst_detection_crop_find_index (GstDetectionCrop * self, guint crop_index, - gint * crop_class, GstInferenceMeta * inference_meta, - GstDetectionMeta * detection_meta, GList **list, Prediction * pred) { - g_return_val_if_fail (self, -1); - g_return_val_if_fail (inference_meta, -1); - g_return_val_if_fail (detection_meta, -1); - g_return_val_if_fail (list, -1); - - if (-2 == *crop_class) { - *crop_class = -1; - return gst_detection_crop_find_predictions (self, crop_class, inference_meta, - list, pred); - } else if (-1 == *crop_class) { - return gst_detection_crop_find_by_index (self, crop_index, detection_meta); - } else { - <<< <<< < HEAD - return gst_detection_crop_find_by_class (self, crop_class, biggest_object, - meta); - == == == = - return gst_detection_crop_find_by_class (self, *crop_class, detection_meta); - >>> >>> > Support the new inference meta - } + return *crop_class; +} + +static gint +gst_detection_crop_find_index (GstDetectionCrop *self, guint crop_index, + gint *crop_class, GstInferenceMeta *inference_meta, GstDetectionMeta *detection_meta, GList **list, GstInferencePrediction *pred) { + g_return_val_if_fail (self, -1); + g_return_val_if_fail (inference_meta, -1); + g_return_val_if_fail (detection_meta, -1); + g_return_val_if_fail (list, -1); + + if (-10 == *crop_class) { + *crop_class = -1; + return gst_detection_crop_find_by_class (self, *crop_class, detection_meta); + }if (-3 == *crop_class) { + return gst_detection_crop_find_by_index (self, crop_index, detection_meta); + } else { + *crop_class = 0; + return gst_detection_crop_find_predictions (self, crop_class, inference_meta, list, pred); } +} - static GstPadProbeReturn - gst_detection_crop_new_buffer (GstPad * pad, GstPadProbeInfo * info, - GstDetectionCrop * self) { - GstBuffer *buffer; - GstDetectionMeta *detection_meta; - GstInferenceMeta *inference_meta; - guint crop_index; - gint crop_class; - gint crop_class_inference; - gint crop_width_ratio; - gint crop_height_ratio; - <<< <<< < HEAD - gint requested_index; - gboolean biggest_object; - == == == = - gint requested_inference_index; - gint requested_detection_index; - >>> >>> > Support the new inference meta - BBox box; - GstPadProbeReturn ret = GST_PAD_PROBE_DROP; - GList *list = NULL; - GList *iter = NULL; - - GST_OBJECT_LOCK (self); - crop_index = self->crop_index; - crop_class = self->crop_class; - crop_class_inference = self->crop_class; - crop_width_ratio = self->width_ratio; - crop_height_ratio = self->height_ratio; - biggest_object = self->biggest_object; - GST_OBJECT_UNLOCK (self); - - buffer = gst_pad_probe_info_get_buffer (info); - - inference_meta = - (GstInferenceMeta *) gst_buffer_get_meta (buffer, - GST_INFERENCE_META_API_TYPE); - detection_meta = - (GstDetectionMeta *) gst_buffer_get_meta (buffer, - GST_DETECTION_META_API_TYPE); - - if (NULL == inference_meta && NULL == detection_meta) { - GST_LOG_OBJECT (self, "No meta found, dropping buffer"); +static GstPadProbeReturn +gst_detection_crop_new_buffer (GstPad *pad, GstPadProbeInfo *info, + GstDetectionCrop *self) { + GstBuffer *buffer; + GstDetectionMeta *detection_meta; + GstInferenceMeta *inference_meta; + guint crop_index; + gint crop_class; + gint crop_class_inference; + gint crop_width_ratio; + gint crop_height_ratio; + gint requested_inference_index; + //gint requested_detection_index; + BoundingBox box; + GstPadProbeReturn ret = GST_PAD_PROBE_DROP; + GList *list = NULL; + GList *iter = NULL; + + GST_OBJECT_LOCK (self); + crop_index = self->crop_index; + crop_class = self->crop_class; + crop_class_inference = self->crop_class; + crop_width_ratio = self->width_ratio; + crop_height_ratio = self->height_ratio; + GST_OBJECT_UNLOCK (self); + + buffer = gst_pad_probe_info_get_buffer (info); + + inference_meta = + (GstInferenceMeta *) gst_buffer_get_meta (buffer, + GST_INFERENCE_META_API_TYPE); + detection_meta = + (GstDetectionMeta *) gst_buffer_get_meta (buffer, + GST_DETECTION_META_API_TYPE); + + if (NULL == inference_meta && NULL == detection_meta) { + GST_LOG_OBJECT (self, "No meta found, dropping buffer"); + goto out; + } + if (inference_meta->prediction <= 0 ) { + GST_LOG_OBJECT (self, "Inference meta has no valid objects"); + goto out; + }else if(-2 == crop_class_inference){ + requested_inference_index = + gst_detection_crop_find_index (self, crop_index, &crop_class, inference_meta, detection_meta, &list, inference_meta->prediction); + if (0 == requested_inference_index) { goto out; } + } - if (inference_meta->prediction <= 0 ) { - GST_LOG_OBJECT (self, "Inference meta has no valid objects"); + /*if (detection_meta->num_boxes <= 0 ) { + GST_LOG_OBJECT (self, "Detection meta has no valid objects"); + goto out; + }else{ + requested_detection_index = + gst_detection_crop_find_index (self, crop_index, &crop_class, inference_meta, detection_meta, &list, inference_meta->prediction); + if (-1 == requested_detection_index) { goto out; - } else if (-2 == crop_class_inference) { - requested_inference_index = - gst_detection_crop_find_index (self, crop_index, &crop_class, inference_meta, - detection_meta, &list, inference_meta->prediction); - if (0 == requested_inference_index) { - goto out; - } } - - <<< <<< < HEAD - requested_index = - gst_detection_crop_find_index (self, crop_index, crop_class, biggest_object, - meta); - if (-1 == requested_index) { - == == == = - if (detection_meta->num_boxes <= 0 ) { - GST_LOG_OBJECT (self, "Detection meta has no valid objects"); - >>> >>> > Support the new inference meta - goto out; - } else { - requested_detection_index = - gst_detection_crop_find_index (self, crop_index, &crop_class, inference_meta, - detection_meta, &list, inference_meta->prediction); - if (-1 == requested_detection_index) { - goto out; - } - } + }*/ //Select the new inference meta - if (-2 == crop_class_inference) { - for (iter = list; iter != NULL; iter = g_list_next(iter)) { - Prediction *pred = (Prediction *)iter->data; - if ( 0 != pred->id) { - GstBuffer *croped_buffer; - box = *pred->box; - GST_LOG_OBJECT (self, "BBox: %fx%fx%fx%f", box.x, box.y, box.width, - box.height); - self->element->SetBoundingBox ((gint) box.x, (gint) box.y, (gint) box.width, - (gint) box.height, (gint) crop_width_ratio, (gint) crop_height_ratio); - croped_buffer = gst_buffer_copy (buffer); - - if (GST_FLOW_OK != gst_pad_chain(self->pad, croped_buffer)) { - GST_ELEMENT_ERROR(self, CORE, FAILED, - ("Failed to push a new buffer into crop element"), (NULL)); - } - } - } - ret = GST_PAD_PROBE_DROP; - goto out; - }//Use old detection meta - else { - box = detection_meta->boxes[requested_detection_index]; - GST_LOG_OBJECT (self, "BBox: %fx%fx%fx%f", box.x, box.y, box.width, + + for (iter = list; iter != NULL; iter = g_list_next(iter)) { + GstInferencePrediction *pred = (GstInferencePrediction *)iter->data; + if( 0 != pred->prediction_id){ + std::cout << pred->prediction_id << std::endl; + GstBuffer *croped_buffer; + box = pred->bbox; + GST_LOG_OBJECT (self, "BBox: %dx%dx%dx%d", box.x, box.y, box.width, box.height); self->element->SetBoundingBox ((gint) box.x, (gint) box.y, (gint) box.width, - (gint) box.height, (gint) crop_width_ratio, (gint) crop_height_ratio); - ret = GST_PAD_PROBE_OK; + (gint) box.height, (gint) crop_width_ratio, (gint) crop_height_ratio); + croped_buffer = gst_buffer_copy (buffer); + + if (GST_FLOW_OK != gst_pad_chain(self->pad, croped_buffer)) { + GST_ELEMENT_ERROR(self, CORE, FAILED, + ("Failed to push a new buffer into crop element"), (NULL)); + } } + } + + ret = GST_PAD_PROBE_DROP; + goto out; + //}//Use old detection meta + /*else{ + box = detection_meta->boxes[requested_detection_index]; + GST_LOG_OBJECT (self, "BBox: %dx%dx%dx%d", box.x, box.y, box.width, + box.height); + self->element->SetBoundingBox ((gint) box.x, (gint) box.y, (gint) box.width, + (gint) box.height, (gint) crop_width_ratio, (gint) crop_height_ratio); + ret = GST_PAD_PROBE_OK; + }*/ out: - return ret; - } + return ret; +} - static gboolean - plugin_init (GstPlugin * plugin) { - gboolean ret = TRUE; +static gboolean +plugin_init (GstPlugin *plugin) { + gboolean ret = TRUE; - ret = - gst_element_register (plugin, "detectioncrop", GST_RANK_NONE, - GST_TYPE_DETECTION_CROP); + ret = + gst_element_register (plugin, "detectioncrop", GST_RANK_NONE, + GST_TYPE_DETECTION_CROP); - return ret; - } + return ret; +} - GST_PLUGIN_DEFINE (GST_VERSION_MAJOR, - GST_VERSION_MINOR, - inferencecrop, - "Crops an incoming image based on an inference prediction bounding box", - plugin_init, VERSION, "LGPL", PACKAGE_NAME, GST_PACKAGE_ORIGIN) +GST_PLUGIN_DEFINE (GST_VERSION_MAJOR, + GST_VERSION_MINOR, + inferencecrop, + "Crops an incoming image based on an inference prediction bounding box", + plugin_init, VERSION, "LGPL", PACKAGE_NAME, GST_PACKAGE_ORIGIN) From 77c10098340153ccd278bafdc66c0f1446159e5d Mon Sep 17 00:00:00 2001 From: tvlenin Date: Thu, 9 Jan 2020 13:14:13 -0600 Subject: [PATCH 092/188] Remove unnecessary pointer guard --- gst-libs/gst/r2inference/gstinferenceprediction.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/gst-libs/gst/r2inference/gstinferenceprediction.c b/gst-libs/gst/r2inference/gstinferenceprediction.c index e1334cd4..8d5767d9 100644 --- a/gst-libs/gst/r2inference/gstinferenceprediction.c +++ b/gst-libs/gst/r2inference/gstinferenceprediction.c @@ -148,7 +148,6 @@ node_copy (gconstpointer node, gpointer data) GstInferencePrediction *self = (GstInferencePrediction *) node; g_return_val_if_fail (node, NULL); - g_return_val_if_fail (data, NULL); return prediction_copy (self); } @@ -159,7 +158,6 @@ node_assign (GNode * node, gpointer data) GstInferencePrediction *pred = (GstInferencePrediction *) node->data; g_return_val_if_fail (node, FALSE); - g_return_val_if_fail (data, FALSE); pred->predictions = node; From fdd08512aec3440bd7b2949abc393f5748a3e555 Mon Sep 17 00:00:00 2001 From: tvlenin Date: Thu, 9 Jan 2020 13:45:29 -0600 Subject: [PATCH 093/188] Remove old DetectionMeta support --- gst/inferencecrop/gstdetectioncrop.cc | 101 +++----------------------- 1 file changed, 9 insertions(+), 92 deletions(-) diff --git a/gst/inferencecrop/gstdetectioncrop.cc b/gst/inferencecrop/gstdetectioncrop.cc index dfb9bf25..0b58a2de 100644 --- a/gst/inferencecrop/gstdetectioncrop.cc +++ b/gst/inferencecrop/gstdetectioncrop.cc @@ -40,7 +40,7 @@ #ifdef HAVE_CONFIG_H #include "config.h" #endif -#include + #include "gstdetectioncrop.h" #include "gst/r2inference/gstinferencemeta.h" @@ -72,16 +72,10 @@ static void gst_detection_crop_set_caps (GstPad *pad, GParamSpec *unused, GstDetectionCrop *self); static GstPadProbeReturn gst_detection_crop_new_buffer (GstPad *pad, GstPadProbeInfo *info, GstDetectionCrop *self); - - static gint gst_detection_crop_find_predictions (GstDetectionCrop *self, gint *crop_class, GstInferenceMeta *meta, GList **list, GstInferencePrediction *pred); -static gint gst_detection_crop_find_by_index (GstDetectionCrop *self, - guint crop_index, GstDetectionMeta *meta); -static gint gst_detection_crop_find_by_class (GstDetectionCrop *self, - gint crop_class, GstDetectionMeta *meta); static gint gst_detection_crop_find_index (GstDetectionCrop *self, - guint crop_index, gint *crop_class, GstInferenceMeta *inference_meta, GstDetectionMeta *detection_meta, GList **list, GstInferencePrediction *pred); + guint crop_index, gint *crop_class, GstInferenceMeta *inference_meta, GList **list, GstInferencePrediction *pred); #define PROP_CROP_INDEX_DEFAULT 0 #define PROP_CROP_INDEX_MAX G_MAXUINT @@ -340,49 +334,10 @@ gst_detection_crop_set_caps (GstPad *pad, GParamSpec *unused, gst_caps_unref(caps); } -static gint -gst_detection_crop_find_by_index (GstDetectionCrop *self, guint crop_index, - GstDetectionMeta *meta) { - gint ret = crop_index; - - g_return_val_if_fail (self, -1); - g_return_val_if_fail (meta, -1); - - if ((gint)crop_index >= meta->num_boxes) { - GST_LOG_OBJECT (self, "Configured crop index is larger than " - "the amount of objects in the prediction"); - - ret = -1; - } - - return ret; -} - -static gint -gst_detection_crop_find_by_class (GstDetectionCrop *self, gint crop_class, - GstDetectionMeta *meta) { - gint i; - gint ret = -1; - g_return_val_if_fail (self, -1); - g_return_val_if_fail (meta, -1); - - for (i = 0; i < meta->num_boxes; ++i) { - if (meta->boxes[i].label == crop_class) { - ret = i; - break; - } - } - - if (-1 == ret) { - GST_LOG_OBJECT (self, "No valid class detected"); - } - - return ret; -} static gint gst_detection_crop_find_predictions (GstDetectionCrop *self, gint *crop_class, - GstInferenceMeta *meta, GList **list,GstInferencePrediction *pred) { + GstInferenceMeta *meta, GList **list, GstInferencePrediction *pred) { guint i; g_return_val_if_fail (self, -1); @@ -405,29 +360,20 @@ gst_detection_crop_find_predictions (GstDetectionCrop *self, gint *crop_class, static gint gst_detection_crop_find_index (GstDetectionCrop *self, guint crop_index, - gint *crop_class, GstInferenceMeta *inference_meta, GstDetectionMeta *detection_meta, GList **list, GstInferencePrediction *pred) { + gint *crop_class, GstInferenceMeta *inference_meta, GList **list, GstInferencePrediction *pred) { g_return_val_if_fail (self, -1); g_return_val_if_fail (inference_meta, -1); - g_return_val_if_fail (detection_meta, -1); g_return_val_if_fail (list, -1); - if (-10 == *crop_class) { - *crop_class = -1; - return gst_detection_crop_find_by_class (self, *crop_class, detection_meta); - }if (-3 == *crop_class) { - return gst_detection_crop_find_by_index (self, crop_index, detection_meta); - } else { - *crop_class = 0; - return gst_detection_crop_find_predictions (self, crop_class, inference_meta, list, pred); - } -} + *crop_class = 0; + return gst_detection_crop_find_predictions (self, crop_class, inference_meta, list, pred); +} static GstPadProbeReturn gst_detection_crop_new_buffer (GstPad *pad, GstPadProbeInfo *info, GstDetectionCrop *self) { GstBuffer *buffer; - GstDetectionMeta *detection_meta; GstInferenceMeta *inference_meta; guint crop_index; gint crop_class; @@ -435,7 +381,6 @@ gst_detection_crop_new_buffer (GstPad *pad, GstPadProbeInfo *info, gint crop_width_ratio; gint crop_height_ratio; gint requested_inference_index; - //gint requested_detection_index; BoundingBox box; GstPadProbeReturn ret = GST_PAD_PROBE_DROP; GList *list = NULL; @@ -454,11 +399,8 @@ gst_detection_crop_new_buffer (GstPad *pad, GstPadProbeInfo *info, inference_meta = (GstInferenceMeta *) gst_buffer_get_meta (buffer, GST_INFERENCE_META_API_TYPE); - detection_meta = - (GstDetectionMeta *) gst_buffer_get_meta (buffer, - GST_DETECTION_META_API_TYPE); - if (NULL == inference_meta && NULL == detection_meta) { + if (NULL == inference_meta ) { GST_LOG_OBJECT (self, "No meta found, dropping buffer"); goto out; } @@ -467,30 +409,15 @@ gst_detection_crop_new_buffer (GstPad *pad, GstPadProbeInfo *info, goto out; }else if(-2 == crop_class_inference){ requested_inference_index = - gst_detection_crop_find_index (self, crop_index, &crop_class, inference_meta, detection_meta, &list, inference_meta->prediction); + gst_detection_crop_find_index (self, crop_index, &crop_class, inference_meta, &list, inference_meta->prediction); if (0 == requested_inference_index) { goto out; } } - /*if (detection_meta->num_boxes <= 0 ) { - GST_LOG_OBJECT (self, "Detection meta has no valid objects"); - goto out; - }else{ - requested_detection_index = - gst_detection_crop_find_index (self, crop_index, &crop_class, inference_meta, detection_meta, &list, inference_meta->prediction); - if (-1 == requested_detection_index) { - goto out; - } - }*/ - -//Select the new inference meta - - for (iter = list; iter != NULL; iter = g_list_next(iter)) { GstInferencePrediction *pred = (GstInferencePrediction *)iter->data; if( 0 != pred->prediction_id){ - std::cout << pred->prediction_id << std::endl; GstBuffer *croped_buffer; box = pred->bbox; GST_LOG_OBJECT (self, "BBox: %dx%dx%dx%d", box.x, box.y, box.width, @@ -508,16 +435,6 @@ gst_detection_crop_new_buffer (GstPad *pad, GstPadProbeInfo *info, ret = GST_PAD_PROBE_DROP; goto out; - //}//Use old detection meta - /*else{ - box = detection_meta->boxes[requested_detection_index]; - GST_LOG_OBJECT (self, "BBox: %dx%dx%dx%d", box.x, box.y, box.width, - box.height); - self->element->SetBoundingBox ((gint) box.x, (gint) box.y, (gint) box.width, - (gint) box.height, (gint) crop_width_ratio, (gint) crop_height_ratio); - ret = GST_PAD_PROBE_OK; - }*/ - out: return ret; From 5ffad4abef8d01825a2395cfe1ebcbe677ca5f05 Mon Sep 17 00:00:00 2001 From: tvlenin Date: Fri, 10 Jan 2020 08:17:18 -0600 Subject: [PATCH 094/188] Verify if prediction is enabled to be cropped --- gst/inferencecrop/gstdetectioncrop.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gst/inferencecrop/gstdetectioncrop.cc b/gst/inferencecrop/gstdetectioncrop.cc index 0b58a2de..ee5f4f37 100644 --- a/gst/inferencecrop/gstdetectioncrop.cc +++ b/gst/inferencecrop/gstdetectioncrop.cc @@ -350,7 +350,7 @@ gst_detection_crop_find_predictions (GstDetectionCrop *self, gint *crop_class, GstInferencePrediction *predict = (GstInferencePrediction*)g_node_nth_child (pred->predictions,i)->data; gst_detection_crop_find_predictions (self,crop_class,meta,list,predict ); } - if(FALSE == G_NODE_IS_ROOT(pred->predictions)){ + if(FALSE == G_NODE_IS_ROOT(pred->predictions) && TRUE == pred->enabled ){ *list = g_list_append (*list,pred); *crop_class=*crop_class+1; } From 0b4b8dc4e4d3ca64f2bbd6331b8c9dd3ba8ab7bb Mon Sep 17 00:00:00 2001 From: tvlenin Date: Fri, 10 Jan 2020 09:34:10 -0600 Subject: [PATCH 095/188] Remove unused properties --- gst/inferencecrop/gstdetectioncrop.cc | 145 +++++++------------------- 1 file changed, 40 insertions(+), 105 deletions(-) diff --git a/gst/inferencecrop/gstdetectioncrop.cc b/gst/inferencecrop/gstdetectioncrop.cc index ee5f4f37..da259828 100644 --- a/gst/inferencecrop/gstdetectioncrop.cc +++ b/gst/inferencecrop/gstdetectioncrop.cc @@ -31,7 +31,7 @@ * gst-launch-1.0 v4l2src device=$CAMERA ! videoconvert ! tee name=t t. ! videoscale ! queue ! net.sink_model t. ! queue ! net.sink_bypass tinyyolov2 name=net model-location=$MODEL_LOCATION ! \ backend=tensorflow backend::input-layer=$INPUT_LAYER backend::output-layer=OUTPUT_LAYER net.src_bypass \ - detectioncrop aspect-ratio="1:1" ! videoscale ! ximagesink sync=false + detectioncrop aspect-ratio=1/1 ! videoscale ! ximagesink sync=false * ]| * Process video frames from the camera using a detectioncrop model. * @@ -73,22 +73,11 @@ static void gst_detection_crop_set_caps (GstPad *pad, GParamSpec *unused, static GstPadProbeReturn gst_detection_crop_new_buffer (GstPad *pad, GstPadProbeInfo *info, GstDetectionCrop *self); static gint gst_detection_crop_find_predictions (GstDetectionCrop *self, - gint *crop_class, GstInferenceMeta *meta, GList **list, GstInferencePrediction *pred); -static gint gst_detection_crop_find_index (GstDetectionCrop *self, - guint crop_index, gint *crop_class, GstInferenceMeta *inference_meta, GList **list, GstInferencePrediction *pred); - -#define PROP_CROP_INDEX_DEFAULT 0 -#define PROP_CROP_INDEX_MAX G_MAXUINT -#define PROP_CROP_INDEX_MIN 0 - -#define PROP_CROP_CLASS_DEFAULT -2 -#define PROP_CROP_CLASS_MAX G_MAXINT -#define PROP_CROP_CLASS_MIN -2 + gint *num_detections, GstInferenceMeta *meta, GList **list, + GstInferencePrediction *pred); enum { PROP_0, - PROP_CROP_INDEX, - PROP_CROP_CLASS, PROP_CROP_ASPECT_RATIO, }; @@ -96,8 +85,6 @@ struct _GstDetectionCrop { GstBin parent; GstPad *pad; CropElement *element; - guint crop_index; - gint crop_class; gint width_ratio; gint height_ratio; }; @@ -136,20 +123,6 @@ gst_detection_crop_class_init (GstDetectionCropClass *klass) { object_class->set_property = gst_detection_crop_set_property; object_class->get_property = gst_detection_crop_get_property; - g_object_class_install_property (object_class, PROP_CROP_INDEX, - g_param_spec_uint ("crop-index", "Crop Index", "Index of the detected " - "object to crop in the prediction array. This will be ignored if " - "crop-class is set to a non-negative value", PROP_CROP_INDEX_MIN, - PROP_CROP_INDEX_MAX, PROP_CROP_INDEX_DEFAULT, G_PARAM_READWRITE)); - - g_object_class_install_property (object_class, PROP_CROP_CLASS, - g_param_spec_int ("crop-class", "Crop Class", - "Object class to crop look for " - "cropping. If set to -1, crop-index will be used. If set to a non-negative " - "value, the detections will be iterated until a valid class is found and then " - "used that one for cropping.", PROP_CROP_CLASS_MIN, - PROP_CROP_CLASS_MAX, PROP_CROP_CLASS_DEFAULT, G_PARAM_READWRITE)); - g_object_class_install_property (object_class, PROP_CROP_ASPECT_RATIO, gst_param_spec_fraction ("aspect-ratio", "Aspect Ratio", "Aspect ratio to crop the detections, width and height separated by '/'. " @@ -163,11 +136,9 @@ static void gst_detection_crop_init (GstDetectionCrop *self) { GstElement *element; GstPad *sinkpad, *sinkgpad, *srcpad, *srcgpad; - + self->pad = NULL; self->element = new VideoCrop (); - self->crop_index = PROP_CROP_INDEX_DEFAULT; - self->crop_class = PROP_CROP_CLASS_DEFAULT; self->width_ratio = PROP_CROP_RATIO_DEFAULT_WIDTH; self->height_ratio = PROP_CROP_RATIO_DEFAULT_HEIGHT; if (FALSE == self->element->Validate ()) { @@ -223,16 +194,6 @@ gst_detection_crop_set_property (GObject *object, guint property_id, GST_DEBUG_OBJECT (self, "set_property"); switch (property_id) { - case PROP_CROP_INDEX: - GST_OBJECT_LOCK (self); - self->crop_index = g_value_get_uint (value); - GST_OBJECT_UNLOCK (self); - break; - case PROP_CROP_CLASS: - GST_OBJECT_LOCK (self); - self->crop_class = g_value_get_int (value); - GST_OBJECT_UNLOCK (self); - break; case PROP_CROP_ASPECT_RATIO: GST_OBJECT_LOCK (self); if (GST_VALUE_HOLDS_FRACTION (value)) { @@ -255,16 +216,6 @@ gst_detection_crop_get_property (GObject *object, guint property_id, GST_DEBUG_OBJECT (self, "get_property"); switch (property_id) { - case PROP_CROP_INDEX: - GST_OBJECT_LOCK (self); - g_value_set_uint (value, self->crop_index); - GST_OBJECT_UNLOCK (self); - break; - case PROP_CROP_CLASS: - GST_OBJECT_LOCK (self); - g_value_set_int (value, self->crop_class); - GST_OBJECT_UNLOCK (self); - break; case PROP_CROP_ASPECT_RATIO: GST_OBJECT_LOCK (self); gst_value_set_fraction (value, self->width_ratio, self->height_ratio); @@ -336,38 +287,29 @@ gst_detection_crop_set_caps (GstPad *pad, GParamSpec *unused, static gint -gst_detection_crop_find_predictions (GstDetectionCrop *self, gint *crop_class, - GstInferenceMeta *meta, GList **list, GstInferencePrediction *pred) { +gst_detection_crop_find_predictions (GstDetectionCrop *self, + gint *num_detections, + GstInferenceMeta *meta, GList **list, GstInferencePrediction *pred) { guint i; g_return_val_if_fail (self, -1); + g_return_val_if_fail (num_detections, -1); g_return_val_if_fail (meta, -1); g_return_val_if_fail (list, -1); - g_return_val_if_fail (crop_class, -1); g_return_val_if_fail (pred, -1); for (i = 0; i < g_node_n_children(pred->predictions) ; i++) { - GstInferencePrediction *predict = (GstInferencePrediction*)g_node_nth_child (pred->predictions,i)->data; - gst_detection_crop_find_predictions (self,crop_class,meta,list,predict ); + GstInferencePrediction *predict = (GstInferencePrediction *)g_node_nth_child ( + pred->predictions, i)->data; + gst_detection_crop_find_predictions (self, num_detections, meta, list, + predict ); } - if(FALSE == G_NODE_IS_ROOT(pred->predictions) && TRUE == pred->enabled ){ - *list = g_list_append (*list,pred); - *crop_class=*crop_class+1; + if (FALSE == G_NODE_IS_ROOT(pred->predictions) && TRUE == pred->enabled ) { + *list = g_list_append (*list, pred); + *num_detections = *num_detections + 1; } - return *crop_class; -} - -static gint -gst_detection_crop_find_index (GstDetectionCrop *self, guint crop_index, - gint *crop_class, GstInferenceMeta *inference_meta, GList **list, GstInferencePrediction *pred) { - g_return_val_if_fail (self, -1); - g_return_val_if_fail (inference_meta, -1); - g_return_val_if_fail (list, -1); - - *crop_class = 0; - return gst_detection_crop_find_predictions (self, crop_class, inference_meta, list, pred); - + return *num_detections; } static GstPadProbeReturn @@ -375,21 +317,16 @@ gst_detection_crop_new_buffer (GstPad *pad, GstPadProbeInfo *info, GstDetectionCrop *self) { GstBuffer *buffer; GstInferenceMeta *inference_meta; - guint crop_index; - gint crop_class; - gint crop_class_inference; + gint num_detections; gint crop_width_ratio; gint crop_height_ratio; - gint requested_inference_index; + gint enabled_detections; BoundingBox box; GstPadProbeReturn ret = GST_PAD_PROBE_DROP; GList *list = NULL; GList *iter = NULL; GST_OBJECT_LOCK (self); - crop_index = self->crop_index; - crop_class = self->crop_class; - crop_class_inference = self->crop_class; crop_width_ratio = self->width_ratio; crop_height_ratio = self->height_ratio; GST_OBJECT_UNLOCK (self); @@ -404,37 +341,35 @@ gst_detection_crop_new_buffer (GstPad *pad, GstPadProbeInfo *info, GST_LOG_OBJECT (self, "No meta found, dropping buffer"); goto out; } - if (inference_meta->prediction <= 0 ) { - GST_LOG_OBJECT (self, "Inference meta has no valid objects"); + + num_detections = 0; + enabled_detections = gst_detection_crop_find_predictions (self, &num_detections, + inference_meta, &list, inference_meta->prediction); + + if (0 == enabled_detections) { goto out; - }else if(-2 == crop_class_inference){ - requested_inference_index = - gst_detection_crop_find_index (self, crop_index, &crop_class, inference_meta, &list, inference_meta->prediction); - if (0 == requested_inference_index) { - goto out; - } } for (iter = list; iter != NULL; iter = g_list_next(iter)) { - GstInferencePrediction *pred = (GstInferencePrediction *)iter->data; - if( 0 != pred->prediction_id){ - GstBuffer *croped_buffer; - box = pred->bbox; - GST_LOG_OBJECT (self, "BBox: %dx%dx%dx%d", box.x, box.y, box.width, - box.height); - self->element->SetBoundingBox ((gint) box.x, (gint) box.y, (gint) box.width, - (gint) box.height, (gint) crop_width_ratio, (gint) crop_height_ratio); - croped_buffer = gst_buffer_copy (buffer); - - if (GST_FLOW_OK != gst_pad_chain(self->pad, croped_buffer)) { - GST_ELEMENT_ERROR(self, CORE, FAILED, - ("Failed to push a new buffer into crop element"), (NULL)); - } + GstInferencePrediction *pred = (GstInferencePrediction *)iter->data; + if ( 0 != pred->prediction_id) { + GstBuffer *croped_buffer; + box = pred->bbox; + GST_LOG_OBJECT (self, "BBox: %dx%dx%dx%d", box.x, box.y, box.width, + box.height); + self->element->SetBoundingBox ((gint) box.x, (gint) box.y, (gint) box.width, + (gint) box.height, (gint) crop_width_ratio, (gint) crop_height_ratio); + croped_buffer = gst_buffer_copy (buffer); + + if (GST_FLOW_OK != gst_pad_chain(self->pad, croped_buffer)) { + GST_ELEMENT_ERROR(self, CORE, FAILED, + ("Failed to push a new buffer into crop element"), (NULL)); } } + } - ret = GST_PAD_PROBE_DROP; - goto out; + ret = GST_PAD_PROBE_DROP; + goto out; out: return ret; From 0251e16d1af357396ed3d3a2358be0c14a9dbf4f Mon Sep 17 00:00:00 2001 From: tvlenin Date: Tue, 14 Jan 2020 10:15:26 -0600 Subject: [PATCH 096/188] Copy children meta to new buffer --- gst/inferencecrop/gstdetectioncrop.cc | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/gst/inferencecrop/gstdetectioncrop.cc b/gst/inferencecrop/gstdetectioncrop.cc index da259828..b4e6b0a4 100644 --- a/gst/inferencecrop/gstdetectioncrop.cc +++ b/gst/inferencecrop/gstdetectioncrop.cc @@ -304,7 +304,7 @@ gst_detection_crop_find_predictions (GstDetectionCrop *self, gst_detection_crop_find_predictions (self, num_detections, meta, list, predict ); } - if (FALSE == G_NODE_IS_ROOT(pred->predictions) && TRUE == pred->enabled ) { + if (FALSE == G_NODE_IS_ROOT(pred->predictions) && FALSE == pred->enabled ) { *list = g_list_append (*list, pred); *num_detections = *num_detections + 1; } @@ -325,7 +325,7 @@ gst_detection_crop_new_buffer (GstPad *pad, GstPadProbeInfo *info, GstPadProbeReturn ret = GST_PAD_PROBE_DROP; GList *list = NULL; GList *iter = NULL; - + GST_OBJECT_LOCK (self); crop_width_ratio = self->width_ratio; crop_height_ratio = self->height_ratio; @@ -354,12 +354,20 @@ gst_detection_crop_new_buffer (GstPad *pad, GstPadProbeInfo *info, GstInferencePrediction *pred = (GstInferencePrediction *)iter->data; if ( 0 != pred->prediction_id) { GstBuffer *croped_buffer; + GstInferenceMeta *dmeta; box = pred->bbox; GST_LOG_OBJECT (self, "BBox: %dx%dx%dx%d", box.x, box.y, box.width, box.height); self->element->SetBoundingBox ((gint) box.x, (gint) box.y, (gint) box.width, (gint) box.height, (gint) crop_width_ratio, (gint) crop_height_ratio); - croped_buffer = gst_buffer_copy (buffer); + + croped_buffer = gst_buffer_copy_deep (buffer); + dmeta = (GstInferenceMeta *) gst_buffer_add_meta (croped_buffer, GST_INFERENCE_META_INFO, + NULL); + gst_inference_prediction_unref (dmeta->prediction); + + GST_LOG ("Copy detection metadata"); + dmeta->prediction = gst_inference_prediction_copy (pred); if (GST_FLOW_OK != gst_pad_chain(self->pad, croped_buffer)) { GST_ELEMENT_ERROR(self, CORE, FAILED, From 69d268214ec96d63363aa8b9100307733ffbf4d2 Mon Sep 17 00:00:00 2001 From: tvlenin Date: Tue, 14 Jan 2020 12:46:34 -0600 Subject: [PATCH 097/188] Refactor cropping process --- gst/inferencecrop/cropelement.cc | 50 +++++---------- gst/inferencecrop/cropelement.h | 31 +++------ gst/inferencecrop/gstdetectioncrop.cc | 90 ++++++++++++++++++++++++--- gst/inferencecrop/videocrop.cc | 56 ++--------------- gst/inferencecrop/videocrop.h | 12 ++-- 5 files changed, 112 insertions(+), 127 deletions(-) diff --git a/gst/inferencecrop/cropelement.cc b/gst/inferencecrop/cropelement.cc index 325d6a89..1c7e1ce1 100644 --- a/gst/inferencecrop/cropelement.cc +++ b/gst/inferencecrop/cropelement.cc @@ -23,14 +23,10 @@ CropElement::CropElement () { this->element = nullptr; - this->image_width = 0; - this->image_height = 0; - this->x = 0; - this->y = 0; - this->width = 0; - this->height = 0; - this->width_ratio = PROP_CROP_RATIO_DEFAULT_WIDTH; - this->height_ratio = PROP_CROP_RATIO_DEFAULT_HEIGHT; + this->top = 0; + this->bottom = 0; + this->right = 0; + this->left = 0; } bool @@ -68,40 +64,22 @@ CropElement::GetElement () { void CropElement::Reset () { - this->SetBoundingBox(0, 0, this->image_width, this->image_height, - this->width_ratio, this->height_ratio); + this->SetCroppingSize(0, 0, 0, 0); } void -CropElement::SetImageSize (gint width, gint height) { +CropElement::SetCroppingSize (gint top, gint bottom, gint right, gint left) { this->mutex.lock (); - this->image_width = width; - this->image_height = height; - this->mutex.unlock (); - - this->Reset (); -} - -void -CropElement::SetBoundingBox (gint x, gint y, gint width, gint height, - gint width_ratio, gint height_ratio) { - this->mutex.lock (); - this->x = x; - this->y = y; - this->width = width; - this->height = height; - this->width_ratio = width_ratio; - this->height_ratio = height_ratio; + this->top = top; + this->bottom = bottom; + this->right = right; + this->left = left; this->UpdateElement (this->element, - this->image_width, - this->image_height, - this->x, - this->y, - this->width, - this->height, - this->width_ratio, - this->height_ratio); + this->top, + this->bottom, + this->right, + this->left); this->mutex.unlock (); } diff --git a/gst/inferencecrop/cropelement.h b/gst/inferencecrop/cropelement.h index 7fbcf561..9f63906d 100644 --- a/gst/inferencecrop/cropelement.h +++ b/gst/inferencecrop/cropelement.h @@ -26,16 +26,13 @@ #include #include -#define PROP_CROP_RATIO_DEFAULT_WIDTH 1 -#define PROP_CROP_RATIO_DEFAULT_HEIGHT 1 + class CropElement { public: CropElement (); bool Validate (); GstElement *GetElement (); - void SetImageSize (gint width, gint height); - void SetBoundingBox (gint x, gint y, gint width, gint height, gint width_ratio, - gint height_ratio); + void SetCroppingSize (gint top, gint bottom, gint right, gint left); virtual ~ CropElement (); virtual const std::string& GetFactory () const = 0; virtual GstPad *GetSinkPad () = 0; @@ -44,26 +41,18 @@ class CropElement { protected: virtual void UpdateElement (GstElement *element, - gint image_width, - gint image_height, - gint x, - gint y, - gint width, - gint height, - gint width_ratio, - gint height_ratio) = 0; + gint top, + gint bottom, + gint right, + gint left) = 0; private: GstElement *element; - gint image_width; - gint image_height; - gint x; - gint y; - gint width; - gint height; + gint top; + gint bottom; + gint right; + gint left; std::mutex mutex; - gint width_ratio; - gint height_ratio; }; #endif //__CROP_ELEMENT_H__ diff --git a/gst/inferencecrop/gstdetectioncrop.cc b/gst/inferencecrop/gstdetectioncrop.cc index b4e6b0a4..931e5c67 100644 --- a/gst/inferencecrop/gstdetectioncrop.cc +++ b/gst/inferencecrop/gstdetectioncrop.cc @@ -41,11 +41,13 @@ #include "config.h" #endif + #include "gstdetectioncrop.h" #include "gst/r2inference/gstinferencemeta.h" #include "videocrop.h" +#include /* generic templates */ static GstStaticPadTemplate sink_template = GST_STATIC_PAD_TEMPLATE ("sink", GST_PAD_SINK, @@ -70,12 +72,19 @@ static GstStateChangeReturn gst_detection_crop_change_state (GstElement * static gboolean gst_detection_crop_start (GstDetectionCrop *self); static void gst_detection_crop_set_caps (GstPad *pad, GParamSpec *unused, GstDetectionCrop *self); +void gst_detection_crop_new_buffer_size (GstDetectionCrop *self, gint x, gint y, + gint width, gint height, gint width_ratio, gint height_ratio, gint *top, + gint *bottom, gint *right, gint *left); static GstPadProbeReturn gst_detection_crop_new_buffer (GstPad *pad, GstPadProbeInfo *info, GstDetectionCrop *self); static gint gst_detection_crop_find_predictions (GstDetectionCrop *self, gint *num_detections, GstInferenceMeta *meta, GList **list, GstInferencePrediction *pred); + +#define PROP_CROP_RATIO_DEFAULT_WIDTH 1 +#define PROP_CROP_RATIO_DEFAULT_HEIGHT 1 + enum { PROP_0, PROP_CROP_ASPECT_RATIO, @@ -87,6 +96,8 @@ struct _GstDetectionCrop { CropElement *element; gint width_ratio; gint height_ratio; + gint width; + gint height; }; struct _GstDetectionCropClass { @@ -280,11 +291,61 @@ gst_detection_crop_set_caps (GstPad *pad, GParamSpec *unused, gst_structure_get_int (st, "height", &height); GST_INFO_OBJECT (self, "Set new caps to %" GST_PTR_FORMAT, caps); - - self->element->SetImageSize (width, height); + self->width = width; + self->height = height; gst_caps_unref(caps); } +void gst_detection_crop_new_buffer_size (GstDetectionCrop *self, gint x, gint y, + gint width, gint height, gint width_ratio, gint height_ratio, gint *top, + gint *bottom, gint *right, gint *left) { + + *top = y; + *bottom = self->height - y - height; + *left = x; + *right = self->width - x - width; + + if (width_ratio > 0 && height_ratio > 0) { + gint top_bottom_modify = round(((height_ratio * width) / width_ratio - height) / + 2); + gint left_right_modify = round(((width_ratio * height) / height_ratio - width) / + 2); + if (width_ratio <= height_ratio) { + if (width > height) { + *top = *top - top_bottom_modify; + *bottom = *bottom - top_bottom_modify; + } else { + *left = *left - left_right_modify; + *right = *right - left_right_modify; + } + } else { + if (width < height) { + *left = *left - left_right_modify; + *right = *right - left_right_modify; + } else { + *top = *top - top_bottom_modify; + *bottom = *bottom - top_bottom_modify; + } + } + } + + if (*top < 0) { + *top = 0; + } + + if (*bottom < 0) { + *bottom = 0; + } + + if (*left < 0) { + *left = 0; + } + + if (*right < 0) { + *right = 0; + } + +} static gint gst_detection_crop_find_predictions (GstDetectionCrop *self, @@ -304,7 +365,7 @@ gst_detection_crop_find_predictions (GstDetectionCrop *self, gst_detection_crop_find_predictions (self, num_detections, meta, list, predict ); } - if (FALSE == G_NODE_IS_ROOT(pred->predictions) && FALSE == pred->enabled ) { + if (FALSE == G_NODE_IS_ROOT(pred->predictions) && TRUE == pred->enabled ) { *list = g_list_append (*list, pred); *num_detections = *num_detections + 1; } @@ -325,7 +386,7 @@ gst_detection_crop_new_buffer (GstPad *pad, GstPadProbeInfo *info, GstPadProbeReturn ret = GST_PAD_PROBE_DROP; GList *list = NULL; GList *iter = NULL; - + GST_OBJECT_LOCK (self); crop_width_ratio = self->width_ratio; crop_height_ratio = self->height_ratio; @@ -355,18 +416,27 @@ gst_detection_crop_new_buffer (GstPad *pad, GstPadProbeInfo *info, if ( 0 != pred->prediction_id) { GstBuffer *croped_buffer; GstInferenceMeta *dmeta; + gint top, bottom, right, left = 0; box = pred->bbox; GST_LOG_OBJECT (self, "BBox: %dx%dx%dx%d", box.x, box.y, box.width, box.height); - self->element->SetBoundingBox ((gint) box.x, (gint) box.y, (gint) box.width, - (gint) box.height, (gint) crop_width_ratio, (gint) crop_height_ratio); + + gst_detection_crop_new_buffer_size (self, box.x, box.y, box.width, box.height, + crop_width_ratio, crop_height_ratio, + &top, &bottom, &right, &left); + self->element->SetCroppingSize ((gint) top, (gint) bottom, (gint) right, + (gint) left); croped_buffer = gst_buffer_copy_deep (buffer); - dmeta = (GstInferenceMeta *) gst_buffer_add_meta (croped_buffer, GST_INFERENCE_META_INFO, - NULL); - gst_inference_prediction_unref (dmeta->prediction); + dmeta = (GstInferenceMeta *) gst_buffer_add_meta (croped_buffer, + GST_INFERENCE_META_INFO, + NULL); + + pred->bbox.x = 0; + pred->bbox.y = 0; + pred->bbox.width = self->width - right - left; + pred->bbox.height = self->height - top - bottom; - GST_LOG ("Copy detection metadata"); dmeta->prediction = gst_inference_prediction_copy (pred); if (GST_FLOW_OK != gst_pad_chain(self->pad, croped_buffer)) { diff --git a/gst/inferencecrop/videocrop.cc b/gst/inferencecrop/videocrop.cc index 9b493d05..041dfdc6 100644 --- a/gst/inferencecrop/videocrop.cc +++ b/gst/inferencecrop/videocrop.cc @@ -31,58 +31,10 @@ VideoCrop::GetFactory () const { void VideoCrop::UpdateElement (GstElement *element, - gint image_width, - gint image_height, - gint x, - gint y, - gint width, - gint height, - gint width_ratio, - gint height_ratio) { - gint top = y; - gint bottom = image_height - y - height; - gint left = x; - gint right = image_width - x - width; - - if (width_ratio > 0 && height_ratio > 0) { - gint top_bottom_modify = round(((height_ratio * width) / width_ratio - height) / - 2); - gint left_right_modify = round(((width_ratio * height) / height_ratio - width) / - 2); - if (width_ratio <= height_ratio) { - if (width > height) { - top = top - top_bottom_modify; - bottom = bottom - top_bottom_modify; - } else { - left = left - left_right_modify; - right = right - left_right_modify; - } - } else { - if (width < height) { - left = left - left_right_modify; - right = right - left_right_modify; - } else { - top = top - top_bottom_modify; - bottom = bottom - top_bottom_modify; - } - } - } - - if (top < 0) { - top = 0; - } - - if (bottom < 0) { - bottom = 0; - } - - if (left < 0) { - left = 0; - } - - if (right < 0) { - right = 0; - } + gint top, + gint bottom, + gint right, + gint left) { g_return_if_fail (element); diff --git a/gst/inferencecrop/videocrop.h b/gst/inferencecrop/videocrop.h index 7d71ed71..2510d342 100644 --- a/gst/inferencecrop/videocrop.h +++ b/gst/inferencecrop/videocrop.h @@ -34,14 +34,10 @@ class VideoCrop: public CropElement { protected: void UpdateElement (GstElement *element, - gint image_width, - gint image_height, - gint x, - gint y, - gint width, - gint height, - gint width_ratio, - gint height_ratio) override; + gint top, + gint bottom, + gint right, + gint left) override; private: GstPad *GetPad (const std::string &name); const std::string factory = "videocrop"; From ca647913f55409a9057a717bf4e52d3cf98ce0b0 Mon Sep 17 00:00:00 2001 From: tvlenin Date: Thu, 16 Jan 2020 10:53:25 -0600 Subject: [PATCH 098/188] Change function return value --- gst/inferencecrop/gstdetectioncrop.cc | 27 +++++++++++---------------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/gst/inferencecrop/gstdetectioncrop.cc b/gst/inferencecrop/gstdetectioncrop.cc index 931e5c67..a0f114de 100644 --- a/gst/inferencecrop/gstdetectioncrop.cc +++ b/gst/inferencecrop/gstdetectioncrop.cc @@ -77,7 +77,7 @@ void gst_detection_crop_new_buffer_size (GstDetectionCrop *self, gint x, gint y, gint *bottom, gint *right, gint *left); static GstPadProbeReturn gst_detection_crop_new_buffer (GstPad *pad, GstPadProbeInfo *info, GstDetectionCrop *self); -static gint gst_detection_crop_find_predictions (GstDetectionCrop *self, +static void gst_detection_crop_find_predictions (GstDetectionCrop *self, gint *num_detections, GstInferenceMeta *meta, GList **list, GstInferencePrediction *pred); @@ -190,6 +190,8 @@ static void gst_detection_crop_finalize (GObject *object) { GstDetectionCrop *self = GST_DETECTION_CROP (object); + g_return_if_fail(self); + delete (self->element); gst_object_unref (self->pad); self->pad = NULL; @@ -347,17 +349,17 @@ void gst_detection_crop_new_buffer_size (GstDetectionCrop *self, gint x, gint y, } -static gint +static void gst_detection_crop_find_predictions (GstDetectionCrop *self, gint *num_detections, GstInferenceMeta *meta, GList **list, GstInferencePrediction *pred) { guint i; - g_return_val_if_fail (self, -1); - g_return_val_if_fail (num_detections, -1); - g_return_val_if_fail (meta, -1); - g_return_val_if_fail (list, -1); - g_return_val_if_fail (pred, -1); + g_return_if_fail (self); + g_return_if_fail (num_detections); + g_return_if_fail (meta); + g_return_if_fail (list); + g_return_if_fail (pred); for (i = 0; i < g_node_n_children(pred->predictions) ; i++) { GstInferencePrediction *predict = (GstInferencePrediction *)g_node_nth_child ( @@ -365,12 +367,11 @@ gst_detection_crop_find_predictions (GstDetectionCrop *self, gst_detection_crop_find_predictions (self, num_detections, meta, list, predict ); } - if (FALSE == G_NODE_IS_ROOT(pred->predictions) && TRUE == pred->enabled ) { + if (FALSE == G_NODE_IS_ROOT(pred->predictions) && FALSE == pred->enabled ) { *list = g_list_append (*list, pred); *num_detections = *num_detections + 1; } - return *num_detections; } static GstPadProbeReturn @@ -381,7 +382,6 @@ gst_detection_crop_new_buffer (GstPad *pad, GstPadProbeInfo *info, gint num_detections; gint crop_width_ratio; gint crop_height_ratio; - gint enabled_detections; BoundingBox box; GstPadProbeReturn ret = GST_PAD_PROBE_DROP; GList *list = NULL; @@ -404,13 +404,9 @@ gst_detection_crop_new_buffer (GstPad *pad, GstPadProbeInfo *info, } num_detections = 0; - enabled_detections = gst_detection_crop_find_predictions (self, &num_detections, + gst_detection_crop_find_predictions (self, &num_detections, inference_meta, &list, inference_meta->prediction); - if (0 == enabled_detections) { - goto out; - } - for (iter = list; iter != NULL; iter = g_list_next(iter)) { GstInferencePrediction *pred = (GstInferencePrediction *)iter->data; if ( 0 != pred->prediction_id) { @@ -447,7 +443,6 @@ gst_detection_crop_new_buffer (GstPad *pad, GstPadProbeInfo *info, } ret = GST_PAD_PROBE_DROP; - goto out; out: return ret; From 75854ef5561ecb54d64447d9a0681ad736f0449f Mon Sep 17 00:00:00 2001 From: tvlenin Date: Fri, 17 Jan 2020 11:26:04 -0600 Subject: [PATCH 099/188] Optimize iteration over metadata --- gst/inferencecrop/gstdetectioncrop.cc | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/gst/inferencecrop/gstdetectioncrop.cc b/gst/inferencecrop/gstdetectioncrop.cc index a0f114de..d26d2ec2 100644 --- a/gst/inferencecrop/gstdetectioncrop.cc +++ b/gst/inferencecrop/gstdetectioncrop.cc @@ -72,7 +72,7 @@ static GstStateChangeReturn gst_detection_crop_change_state (GstElement * static gboolean gst_detection_crop_start (GstDetectionCrop *self); static void gst_detection_crop_set_caps (GstPad *pad, GParamSpec *unused, GstDetectionCrop *self); -void gst_detection_crop_new_buffer_size (GstDetectionCrop *self, gint x, gint y, +static void gst_detection_crop_new_buffer_size (GstDetectionCrop *self, gint x, gint y, gint width, gint height, gint width_ratio, gint height_ratio, gint *top, gint *bottom, gint *right, gint *left); static GstPadProbeReturn gst_detection_crop_new_buffer (GstPad *pad, @@ -298,7 +298,8 @@ gst_detection_crop_set_caps (GstPad *pad, GParamSpec *unused, gst_caps_unref(caps); } -void gst_detection_crop_new_buffer_size (GstDetectionCrop *self, gint x, gint y, +static void +gst_detection_crop_new_buffer_size (GstDetectionCrop *self, gint x, gint y, gint width, gint height, gint width_ratio, gint height_ratio, gint *top, gint *bottom, gint *right, gint *left) { @@ -353,7 +354,8 @@ static void gst_detection_crop_find_predictions (GstDetectionCrop *self, gint *num_detections, GstInferenceMeta *meta, GList **list, GstInferencePrediction *pred) { - guint i; + GSList *children_list = NULL; + GSList *iter = NULL; g_return_if_fail (self); g_return_if_fail (num_detections); @@ -361,9 +363,11 @@ gst_detection_crop_find_predictions (GstDetectionCrop *self, g_return_if_fail (list); g_return_if_fail (pred); - for (i = 0; i < g_node_n_children(pred->predictions) ; i++) { - GstInferencePrediction *predict = (GstInferencePrediction *)g_node_nth_child ( - pred->predictions, i)->data; + children_list = gst_inference_prediction_get_children(pred); + + for (iter = children_list; iter != NULL; iter = g_slist_next(iter)) { + GstInferencePrediction *predict = (GstInferencePrediction *)iter->data; + gst_detection_crop_find_predictions (self, num_detections, meta, list, predict ); } From 03e1e71429edb75ae9eb33a171b434560c7bf21e Mon Sep 17 00:00:00 2001 From: tvlenin Date: Fri, 17 Jan 2020 11:30:04 -0600 Subject: [PATCH 100/188] Modify new meta instead ol meta --- gst/inferencecrop/gstdetectioncrop.cc | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/gst/inferencecrop/gstdetectioncrop.cc b/gst/inferencecrop/gstdetectioncrop.cc index d26d2ec2..cab03eec 100644 --- a/gst/inferencecrop/gstdetectioncrop.cc +++ b/gst/inferencecrop/gstdetectioncrop.cc @@ -413,7 +413,6 @@ gst_detection_crop_new_buffer (GstPad *pad, GstPadProbeInfo *info, for (iter = list; iter != NULL; iter = g_list_next(iter)) { GstInferencePrediction *pred = (GstInferencePrediction *)iter->data; - if ( 0 != pred->prediction_id) { GstBuffer *croped_buffer; GstInferenceMeta *dmeta; gint top, bottom, right, left = 0; @@ -427,23 +426,22 @@ gst_detection_crop_new_buffer (GstPad *pad, GstPadProbeInfo *info, self->element->SetCroppingSize ((gint) top, (gint) bottom, (gint) right, (gint) left); - croped_buffer = gst_buffer_copy_deep (buffer); + croped_buffer = gst_buffer_copy (buffer); dmeta = (GstInferenceMeta *) gst_buffer_add_meta (croped_buffer, GST_INFERENCE_META_INFO, NULL); - pred->bbox.x = 0; - pred->bbox.y = 0; - pred->bbox.width = self->width - right - left; - pred->bbox.height = self->height - top - bottom; - dmeta->prediction = gst_inference_prediction_copy (pred); + dmeta->prediction->bbox.x = 0; + dmeta->prediction->bbox.y = 0; + dmeta->prediction->bbox.width = self->width - right - left; + dmeta->prediction->bbox.height = self->height - top - bottom; + if (GST_FLOW_OK != gst_pad_chain(self->pad, croped_buffer)) { GST_ELEMENT_ERROR(self, CORE, FAILED, ("Failed to push a new buffer into crop element"), (NULL)); } - } } ret = GST_PAD_PROBE_DROP; From c5b65b1f3b24b24739e84ef1cfcc8f70f40bb7fb Mon Sep 17 00:00:00 2001 From: tvlenin Date: Fri, 17 Jan 2020 13:21:32 -0600 Subject: [PATCH 101/188] Fix code indentation --- gst/inferencecrop/gstdetectioncrop.cc | 69 ++++++++++++++------------- 1 file changed, 35 insertions(+), 34 deletions(-) diff --git a/gst/inferencecrop/gstdetectioncrop.cc b/gst/inferencecrop/gstdetectioncrop.cc index cab03eec..bc267478 100644 --- a/gst/inferencecrop/gstdetectioncrop.cc +++ b/gst/inferencecrop/gstdetectioncrop.cc @@ -72,7 +72,8 @@ static GstStateChangeReturn gst_detection_crop_change_state (GstElement * static gboolean gst_detection_crop_start (GstDetectionCrop *self); static void gst_detection_crop_set_caps (GstPad *pad, GParamSpec *unused, GstDetectionCrop *self); -static void gst_detection_crop_new_buffer_size (GstDetectionCrop *self, gint x, gint y, +static void gst_detection_crop_new_buffer_size (GstDetectionCrop *self, gint x, + gint y, gint width, gint height, gint width_ratio, gint height_ratio, gint *top, gint *bottom, gint *right, gint *left); static GstPadProbeReturn gst_detection_crop_new_buffer (GstPad *pad, @@ -300,8 +301,8 @@ gst_detection_crop_set_caps (GstPad *pad, GParamSpec *unused, static void gst_detection_crop_new_buffer_size (GstDetectionCrop *self, gint x, gint y, - gint width, gint height, gint width_ratio, gint height_ratio, gint *top, - gint *bottom, gint *right, gint *left) { + gint width, gint height, gint width_ratio, gint height_ratio, gint *top, + gint *bottom, gint *right, gint *left) { *top = y; *bottom = self->height - y - height; @@ -371,7 +372,7 @@ gst_detection_crop_find_predictions (GstDetectionCrop *self, gst_detection_crop_find_predictions (self, num_detections, meta, list, predict ); } - if (FALSE == G_NODE_IS_ROOT(pred->predictions) && FALSE == pred->enabled ) { + if (FALSE == G_NODE_IS_ROOT(pred->predictions) && TRUE == pred->enabled ) { *list = g_list_append (*list, pred); *num_detections = *num_detections + 1; } @@ -409,39 +410,39 @@ gst_detection_crop_new_buffer (GstPad *pad, GstPadProbeInfo *info, num_detections = 0; gst_detection_crop_find_predictions (self, &num_detections, - inference_meta, &list, inference_meta->prediction); + inference_meta, &list, inference_meta->prediction); for (iter = list; iter != NULL; iter = g_list_next(iter)) { GstInferencePrediction *pred = (GstInferencePrediction *)iter->data; - GstBuffer *croped_buffer; - GstInferenceMeta *dmeta; - gint top, bottom, right, left = 0; - box = pred->bbox; - GST_LOG_OBJECT (self, "BBox: %dx%dx%dx%d", box.x, box.y, box.width, - box.height); - - gst_detection_crop_new_buffer_size (self, box.x, box.y, box.width, box.height, - crop_width_ratio, crop_height_ratio, - &top, &bottom, &right, &left); - self->element->SetCroppingSize ((gint) top, (gint) bottom, (gint) right, - (gint) left); - - croped_buffer = gst_buffer_copy (buffer); - dmeta = (GstInferenceMeta *) gst_buffer_add_meta (croped_buffer, - GST_INFERENCE_META_INFO, - NULL); - - dmeta->prediction = gst_inference_prediction_copy (pred); - - dmeta->prediction->bbox.x = 0; - dmeta->prediction->bbox.y = 0; - dmeta->prediction->bbox.width = self->width - right - left; - dmeta->prediction->bbox.height = self->height - top - bottom; - - if (GST_FLOW_OK != gst_pad_chain(self->pad, croped_buffer)) { - GST_ELEMENT_ERROR(self, CORE, FAILED, - ("Failed to push a new buffer into crop element"), (NULL)); - } + GstBuffer *croped_buffer; + GstInferenceMeta *dmeta; + gint top, bottom, right, left = 0; + box = pred->bbox; + GST_LOG_OBJECT (self, "BBox: %dx%dx%dx%d", box.x, box.y, box.width, + box.height); + + gst_detection_crop_new_buffer_size (self, box.x, box.y, box.width, box.height, + crop_width_ratio, crop_height_ratio, + &top, &bottom, &right, &left); + self->element->SetCroppingSize ((gint) top, (gint) bottom, (gint) right, + (gint) left); + + croped_buffer = gst_buffer_copy (buffer); + dmeta = (GstInferenceMeta *) gst_buffer_add_meta (croped_buffer, + GST_INFERENCE_META_INFO, + NULL); + + dmeta->prediction = gst_inference_prediction_copy (pred); + + dmeta->prediction->bbox.x = 0; + dmeta->prediction->bbox.y = 0; + dmeta->prediction->bbox.width = self->width - right - left; + dmeta->prediction->bbox.height = self->height - top - bottom; + + if (GST_FLOW_OK != gst_pad_chain(self->pad, croped_buffer)) { + GST_ELEMENT_ERROR(self, CORE, FAILED, + ("Failed to push a new buffer into crop element"), (NULL)); + } } ret = GST_PAD_PROBE_DROP; From b32dbdbda21f0fd15be2b62e98240efc3f34eb68 Mon Sep 17 00:00:00 2001 From: rrcarlosrodriguez Date: Fri, 17 Jan 2020 17:24:42 -0600 Subject: [PATCH 102/188] Fix failing make check Remove unnecessary inferenceoverlay dependency for the tests --- tests/check/Makefile.am | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/check/Makefile.am b/tests/check/Makefile.am index e546212c..d0df7cbc 100644 --- a/tests/check/Makefile.am +++ b/tests/check/Makefile.am @@ -45,7 +45,6 @@ AM_CFLAGS = \ LDADD = \ $(top_builddir)/gst-libs/gst/r2inference/libgstinference-1.0.la \ - $(top_builddir)/gst-libs/gst/opencv/libgstinferenceoverlay-1.0.la \ $(GST_LIBS) \ $(GST_PLUGINS_BASE_LIBS) \ -lgstvideo-1.0 \ From 6ef4979f43fa614dc09490c284dee3ee89e91fa5 Mon Sep 17 00:00:00 2001 From: tvlenin Date: Tue, 17 Dec 2019 11:08:41 -0600 Subject: [PATCH 103/188] Refactor gstinferenceoverlay name to gstinferencebaseoverlay --- ext/opencv/gstclassificationoverlay.cc | 10 ++--- ext/opencv/gstclassificationoverlay.h | 4 +- ext/opencv/gstdetectionoverlay.cc | 10 ++--- ext/opencv/gstdetectionoverlay.h | 4 +- ext/opencv/gstembeddingoverlay.cc | 10 ++--- ext/opencv/gstembeddingoverlay.h | 4 +- gst-libs/gst/opencv/Makefile.am | 4 +- ...nceoverlay.c => gstinferencebaseoverlay.c} | 43 ++++++++++--------- ...nceoverlay.h => gstinferencebaseoverlay.h} | 6 +-- 9 files changed, 48 insertions(+), 47 deletions(-) rename gst-libs/gst/opencv/{gstinferenceoverlay.c => gstinferencebaseoverlay.c} (85%) rename gst-libs/gst/opencv/{gstinferenceoverlay.h => gstinferencebaseoverlay.h} (87%) diff --git a/ext/opencv/gstclassificationoverlay.cc b/ext/opencv/gstclassificationoverlay.cc index 6c6dd7dc..d5ca1cda 100644 --- a/ext/opencv/gstclassificationoverlay.cc +++ b/ext/opencv/gstclassificationoverlay.cc @@ -40,7 +40,7 @@ GST_DEBUG_CATEGORY_STATIC (gst_classification_overlay_debug_category); /* prototypes */ static GstFlowReturn -gst_classification_overlay_process_meta (GstInferenceOverlay * +gst_classification_overlay_process_meta (GstInferenceBaseOverlay * inference_overlay, GstVideoFrame * frame, GstMeta * meta, gdouble font_scale, gint thickness, gchar ** labels_list, gint num_labels); @@ -51,12 +51,12 @@ enum struct _GstClassificationOverlay { - GstInferenceOverlay parent; + GstInferenceBaseOverlay parent; }; struct _GstClassificationOverlayClass { - GstInferenceOverlay parent; + GstInferenceBaseOverlay parent; }; /* class initialization */ @@ -70,7 +70,7 @@ G_DEFINE_TYPE_WITH_CODE (GstClassificationOverlay, gst_classification_overlay, static void gst_classification_overlay_class_init (GstClassificationOverlayClass * klass) { - GstInferenceOverlayClass *io_class = GST_INFERENCE_OVERLAY_CLASS (klass); + GstInferenceBaseOverlayClass *io_class = GST_INFERENCE_OVERLAY_CLASS (klass); gst_element_class_set_static_metadata (GST_ELEMENT_CLASS (klass), "classificationoverlay", "Filter", @@ -94,7 +94,7 @@ gst_classification_overlay_init (GstClassificationOverlay * } static GstFlowReturn -gst_classification_overlay_process_meta (GstInferenceOverlay * +gst_classification_overlay_process_meta (GstInferenceBaseOverlay * inference_overlay, GstVideoFrame * frame, GstMeta * meta, gdouble font_scale, gint thickness, gchar ** labels_list, gint num_labels) { diff --git a/ext/opencv/gstclassificationoverlay.h b/ext/opencv/gstclassificationoverlay.h index 7fd8f759..fe31be7d 100644 --- a/ext/opencv/gstclassificationoverlay.h +++ b/ext/opencv/gstclassificationoverlay.h @@ -22,12 +22,12 @@ #ifndef _GST_CLASSIFICATION_OVERLAY_H_ #define _GST_CLASSIFICATION_OVERLAY_H_ -#include +#include G_BEGIN_DECLS #define GST_TYPE_CLASSIFICATION_OVERLAY (gst_classification_overlay_get_type()) -G_DECLARE_FINAL_TYPE (GstClassificationOverlay, gst_classification_overlay, GST, CLASSIFICATION_OVERLAY, GstInferenceOverlay) +G_DECLARE_FINAL_TYPE (GstClassificationOverlay, gst_classification_overlay, GST, CLASSIFICATION_OVERLAY, GstInferenceBaseOverlay) G_END_DECLS diff --git a/ext/opencv/gstdetectionoverlay.cc b/ext/opencv/gstdetectionoverlay.cc index b68784e1..d36c093e 100644 --- a/ext/opencv/gstdetectionoverlay.cc +++ b/ext/opencv/gstdetectionoverlay.cc @@ -51,7 +51,7 @@ GST_DEBUG_CATEGORY_STATIC (gst_detection_overlay_debug_category); /* prototypes */ static GstFlowReturn -gst_detection_overlay_process_meta (GstInferenceOverlay * inference_overlay, +gst_detection_overlay_process_meta (GstInferenceBaseOverlay * inference_overlay, GstVideoFrame * frame, GstMeta * meta, gdouble font_scale, gint thickness, gchar ** labels_list, gint num_labels); @@ -62,12 +62,12 @@ enum struct _GstDetectionOverlay { - GstInferenceOverlay parent; + GstInferenceBaseOverlay parent; }; struct _GstDetectionOverlayClass { - GstInferenceOverlay parent; + GstInferenceBaseOverlay parent; }; /* class initialization */ @@ -80,7 +80,7 @@ G_DEFINE_TYPE_WITH_CODE (GstDetectionOverlay, gst_detection_overlay, static void gst_detection_overlay_class_init (GstDetectionOverlayClass * klass) { - GstInferenceOverlayClass *io_class = GST_INFERENCE_OVERLAY_CLASS (klass); + GstInferenceBaseOverlayClass *io_class = GST_INFERENCE_OVERLAY_CLASS (klass); gst_element_class_set_static_metadata (GST_ELEMENT_CLASS (klass), "detectionoverlay", "Filter", @@ -103,7 +103,7 @@ gst_detection_overlay_init (GstDetectionOverlay * detection_overlay) } static GstFlowReturn -gst_detection_overlay_process_meta (GstInferenceOverlay * inference_overlay, +gst_detection_overlay_process_meta (GstInferenceBaseOverlay * inference_overlay, GstVideoFrame * frame, GstMeta * meta, gdouble font_scale, gint thickness, gchar ** labels_list, gint num_labels) { diff --git a/ext/opencv/gstdetectionoverlay.h b/ext/opencv/gstdetectionoverlay.h index 61587274..c5878fdd 100644 --- a/ext/opencv/gstdetectionoverlay.h +++ b/ext/opencv/gstdetectionoverlay.h @@ -22,12 +22,12 @@ #ifndef _GST_DETECTION_OVERLAY_H_ #define _GST_DETECTION_OVERLAY_H_ -#include +#include G_BEGIN_DECLS #define GST_TYPE_DETECTION_OVERLAY (gst_detection_overlay_get_type()) -G_DECLARE_FINAL_TYPE (GstDetectionOverlay, gst_detection_overlay, GST, DETECTION_OVERLAY, GstInferenceOverlay) +G_DECLARE_FINAL_TYPE (GstDetectionOverlay, gst_detection_overlay, GST, DETECTION_OVERLAY, GstInferenceBaseOverlay) G_END_DECLS diff --git a/ext/opencv/gstembeddingoverlay.cc b/ext/opencv/gstembeddingoverlay.cc index 56933d74..29a370af 100644 --- a/ext/opencv/gstembeddingoverlay.cc +++ b/ext/opencv/gstembeddingoverlay.cc @@ -52,7 +52,7 @@ static void gst_embedding_overlay_get_property (GObject * object, guint property_id, GValue * value, GParamSpec * pspec); static void gst_embedding_overlay_finalize (GObject * object); static GstFlowReturn -gst_embedding_overlay_process_meta (GstInferenceOverlay * inference_overlay, +gst_embedding_overlay_process_meta (GstInferenceBaseOverlay * inference_overlay, GstVideoFrame * frame, GstMeta * meta, gdouble font_scale, gint thickness, gchar ** labels_list, gint num_labels); static gboolean @@ -68,7 +68,7 @@ enum struct _GstEmbeddingOverlay { - GstInferenceOverlay parent; + GstInferenceBaseOverlay parent; gchar *embeddings; gchar **embeddings_list; gint num_embeddings; @@ -78,7 +78,7 @@ struct _GstEmbeddingOverlay struct _GstClassificationOverlayClass { - GstInferenceOverlay parent; + GstInferenceBaseOverlay parent; }; /* class initialization */ @@ -92,7 +92,7 @@ static void gst_embedding_overlay_class_init (GstEmbeddingOverlayClass * klass) { GObjectClass *gobject_class = G_OBJECT_CLASS (klass); - GstInferenceOverlayClass *io_class = GST_INFERENCE_OVERLAY_CLASS (klass); + GstInferenceBaseOverlayClass *io_class = GST_INFERENCE_OVERLAY_CLASS (klass); gobject_class->set_property = gst_embedding_overlay_set_property; gobject_class->get_property = gst_embedding_overlay_get_property; @@ -209,7 +209,7 @@ gst_embedding_overlay_finalize (GObject * object) } static GstFlowReturn -gst_embedding_overlay_process_meta (GstInferenceOverlay * inference_overlay, +gst_embedding_overlay_process_meta (GstInferenceBaseOverlay * inference_overlay, GstVideoFrame * frame, GstMeta * meta, gdouble font_scale, gint thickness, gchar ** labels_list, gint num_labels) { diff --git a/ext/opencv/gstembeddingoverlay.h b/ext/opencv/gstembeddingoverlay.h index a1751149..3a2b503e 100644 --- a/ext/opencv/gstembeddingoverlay.h +++ b/ext/opencv/gstembeddingoverlay.h @@ -22,12 +22,12 @@ #ifndef _GST_EMBEDDING_OVERLAY_H_ #define _GST_EMBEDDING_OVERLAY_H_ -#include +#include G_BEGIN_DECLS #define GST_TYPE_EMBEDDING_OVERLAY (gst_embedding_overlay_get_type()) -G_DECLARE_FINAL_TYPE (GstEmbeddingOverlay, gst_embedding_overlay, GST, EMBEDDING_OVERLAY, GstInferenceOverlay) +G_DECLARE_FINAL_TYPE (GstEmbeddingOverlay, gst_embedding_overlay, GST, EMBEDDING_OVERLAY, GstInferenceBaseOverlay) G_END_DECLS diff --git a/gst-libs/gst/opencv/Makefile.am b/gst-libs/gst/opencv/Makefile.am index ffd1ba4c..337546c6 100644 --- a/gst-libs/gst/opencv/Makefile.am +++ b/gst-libs/gst/opencv/Makefile.am @@ -2,7 +2,7 @@ lib_LTLIBRARIES = libgstinferenceoverlay-@GST_API_VERSION@.la libgstinferenceoverlay_@GST_API_VERSION@_la_SOURCES= \ - gstinferenceoverlay.c + gstinferencebaseoverlay.c libgstinferenceoverlay_@GST_API_VERSION@_la_CFLAGS= \ $(GST_CFLAGS) \ @@ -20,4 +20,4 @@ libgstinferenceoverlay_@GST_API_VERSION@_la_LIBADD= \ gstinferenceoverlayincludedir=@includedir@/gstreamer-@GST_API_VERSION@/gst/opencv/ gstinferenceoverlayinclude_HEADERS= \ - gstinferenceoverlay.h + gstinferencebaseoverlay.h diff --git a/gst-libs/gst/opencv/gstinferenceoverlay.c b/gst-libs/gst/opencv/gstinferencebaseoverlay.c similarity index 85% rename from gst-libs/gst/opencv/gstinferenceoverlay.c rename to gst-libs/gst/opencv/gstinferencebaseoverlay.c index ca079bf4..6d065da0 100644 --- a/gst-libs/gst/opencv/gstinferenceoverlay.c +++ b/gst-libs/gst/opencv/gstinferencebaseoverlay.c @@ -19,7 +19,7 @@ * */ -#include "gstinferenceoverlay.h" +#include "gstinferencebaseoverlay.h" /* pad templates */ @@ -49,8 +49,8 @@ enum PROP_LABELS }; -typedef struct _GstInferenceOverlayPrivate GstInferenceOverlayPrivate; -struct _GstInferenceOverlayPrivate +typedef struct _GstInferenceBaseOverlayPrivate GstInferenceBaseOverlayPrivate; +struct _GstInferenceBaseOverlayPrivate { gdouble font_scale; gint thickness; @@ -74,17 +74,17 @@ gst_inference_overlay_transform_frame_ip (GstVideoFilter * trans, /* class initialization */ -G_DEFINE_TYPE_WITH_CODE (GstInferenceOverlay, gst_inference_overlay, +G_DEFINE_TYPE_WITH_CODE (GstInferenceBaseOverlay, gst_inference_overlay, GST_TYPE_VIDEO_FILTER, GST_DEBUG_CATEGORY_INIT (gst_inference_overlay_debug_category, "inferenceoverlay", 0, "debug category for inferenceoverlay class"); - G_ADD_PRIVATE (GstInferenceOverlay)); + G_ADD_PRIVATE (GstInferenceBaseOverlay)); #define GST_INFERENCE_OVERLAY_PRIVATE(self) \ - (GstInferenceOverlayPrivate *)(gst_inference_overlay_get_instance_private (self)) + (GstInferenceBaseOverlayPrivate *)(gst_inference_overlay_get_instance_private (self)) static void -gst_inference_overlay_class_init (GstInferenceOverlayClass * klass) +gst_inference_overlay_class_init (GstInferenceBaseOverlayClass * klass) { GObjectClass *gobject_class = G_OBJECT_CLASS (klass); GstBaseTransformClass *base_transform_class = @@ -125,9 +125,9 @@ gst_inference_overlay_class_init (GstInferenceOverlayClass * klass) } static void -gst_inference_overlay_init (GstInferenceOverlay * inference_overlay) +gst_inference_overlay_init (GstInferenceBaseOverlay * inference_overlay) { - GstInferenceOverlayPrivate *priv = + GstInferenceBaseOverlayPrivate *priv = GST_INFERENCE_OVERLAY_PRIVATE (inference_overlay); priv->font_scale = DEFAULT_FONT_SCALE; priv->thickness = DEFAULT_THICKNESS; @@ -140,8 +140,8 @@ void gst_inference_overlay_set_property (GObject * object, guint property_id, const GValue * value, GParamSpec * pspec) { - GstInferenceOverlay *inference_overlay = GST_INFERENCE_OVERLAY (object); - GstInferenceOverlayPrivate *priv = + GstInferenceBaseOverlay *inference_overlay = GST_INFERENCE_OVERLAY (object); + GstInferenceBaseOverlayPrivate *priv = GST_INFERENCE_OVERLAY_PRIVATE (inference_overlay); GST_DEBUG_OBJECT (inference_overlay, "set_property"); @@ -180,8 +180,8 @@ void gst_inference_overlay_get_property (GObject * object, guint property_id, GValue * value, GParamSpec * pspec) { - GstInferenceOverlay *inference_overlay = GST_INFERENCE_OVERLAY (object); - GstInferenceOverlayPrivate *priv = + GstInferenceBaseOverlay *inference_overlay = GST_INFERENCE_OVERLAY (object); + GstInferenceBaseOverlayPrivate *priv = GST_INFERENCE_OVERLAY_PRIVATE (inference_overlay); GST_DEBUG_OBJECT (inference_overlay, "get_property"); @@ -205,8 +205,8 @@ gst_inference_overlay_get_property (GObject * object, guint property_id, void gst_inference_overlay_dispose (GObject * object) { - GstInferenceOverlay *inference_overlay = GST_INFERENCE_OVERLAY (object); - GstInferenceOverlayPrivate *priv = + GstInferenceBaseOverlay *inference_overlay = GST_INFERENCE_OVERLAY (object); + GstInferenceBaseOverlayPrivate *priv = GST_INFERENCE_OVERLAY_PRIVATE (inference_overlay); GST_DEBUG_OBJECT (inference_overlay, "dispose"); @@ -225,7 +225,7 @@ gst_inference_overlay_dispose (GObject * object) void gst_inference_overlay_finalize (GObject * object) { - GstInferenceOverlay *inference_overlay = GST_INFERENCE_OVERLAY (object); + GstInferenceBaseOverlay *inference_overlay = GST_INFERENCE_OVERLAY (object); GST_DEBUG_OBJECT (inference_overlay, "finalize"); @@ -237,7 +237,7 @@ gst_inference_overlay_finalize (GObject * object) static gboolean gst_inference_overlay_start (GstBaseTransform * trans) { - GstInferenceOverlay *inference_overlay = GST_INFERENCE_OVERLAY (trans); + GstInferenceBaseOverlay *inference_overlay = GST_INFERENCE_OVERLAY (trans); GST_DEBUG_OBJECT (inference_overlay, "start"); @@ -247,7 +247,7 @@ gst_inference_overlay_start (GstBaseTransform * trans) static gboolean gst_inference_overlay_stop (GstBaseTransform * trans) { - GstInferenceOverlay *inference_overlay = GST_INFERENCE_OVERLAY (trans); + GstInferenceBaseOverlay *inference_overlay = GST_INFERENCE_OVERLAY (trans); GST_DEBUG_OBJECT (inference_overlay, "stop"); @@ -259,9 +259,10 @@ static GstFlowReturn gst_inference_overlay_transform_frame_ip (GstVideoFilter * trans, GstVideoFrame * frame) { - GstInferenceOverlayClass *io_class = GST_INFERENCE_OVERLAY_GET_CLASS (trans); - GstInferenceOverlay *inference_overlay = GST_INFERENCE_OVERLAY (trans); - GstInferenceOverlayPrivate *priv = + GstInferenceBaseOverlayClass *io_class = + GST_INFERENCE_OVERLAY_GET_CLASS (trans); + GstInferenceBaseOverlay *inference_overlay = GST_INFERENCE_OVERLAY (trans); + GstInferenceBaseOverlayPrivate *priv = GST_INFERENCE_OVERLAY_PRIVATE (inference_overlay); GstMeta *meta; GstFlowReturn ret = GST_FLOW_ERROR; diff --git a/gst-libs/gst/opencv/gstinferenceoverlay.h b/gst-libs/gst/opencv/gstinferencebaseoverlay.h similarity index 87% rename from gst-libs/gst/opencv/gstinferenceoverlay.h rename to gst-libs/gst/opencv/gstinferencebaseoverlay.h index 11362b5e..c9602139 100644 --- a/gst-libs/gst/opencv/gstinferenceoverlay.h +++ b/gst-libs/gst/opencv/gstinferencebaseoverlay.h @@ -28,14 +28,14 @@ G_BEGIN_DECLS #define GST_TYPE_INFERENCE_OVERLAY gst_inference_overlay_get_type () -G_DECLARE_DERIVABLE_TYPE (GstInferenceOverlay, gst_inference_overlay, GST, +G_DECLARE_DERIVABLE_TYPE (GstInferenceBaseOverlay, gst_inference_overlay, GST, INFERENCE_OVERLAY, GstVideoFilter); -struct _GstInferenceOverlayClass +struct _GstInferenceBaseOverlayClass { GstVideoFilterClass parent_class; - GstFlowReturn (* process_meta) (GstInferenceOverlay * inference_overlay, + GstFlowReturn (* process_meta) (GstInferenceBaseOverlay * inference_overlay, GstVideoFrame * frame, GstMeta* meta, gdouble font_scale, gint thickness, gchar **labels_list, gint num_labels); From 9a2a624075395d89c51780529b8b3c08f9738408 Mon Sep 17 00:00:00 2001 From: tvlenin Date: Tue, 17 Dec 2019 11:25:49 -0600 Subject: [PATCH 104/188] Refactor gstinferenceoverlay plugin name --- configure.ac | 2 +- ext/opencv/Makefile.am | 16 ++++++++-------- ...enceoverlay.c => gstinferenceoverlayplugin.c} | 0 3 files changed, 9 insertions(+), 9 deletions(-) rename ext/opencv/{gstinferenceoverlay.c => gstinferenceoverlayplugin.c} (100%) diff --git a/configure.ac b/configure.ac index 12e297e9..dd253aa7 100644 --- a/configure.ac +++ b/configure.ac @@ -20,7 +20,7 @@ AS_NANO(GST_GIT="no", GST_GIT="yes") dnl can autoconf find the source ? AC_CONFIG_SRCDIR([ext/r2inference/gstinference.c]) -AC_CONFIG_SRCDIR([ext/opencv/gstinferenceoverlay.c]) +AC_CONFIG_SRCDIR([ext/opencv/gstinferenceoverlayplugin.c]) dnl define the output header for config AC_CONFIG_HEADERS([config.h]) diff --git a/ext/opencv/Makefile.am b/ext/opencv/Makefile.am index 8c001671..4c4594ba 100644 --- a/ext/opencv/Makefile.am +++ b/ext/opencv/Makefile.am @@ -1,12 +1,12 @@ -plugin_LTLIBRARIES = libgstinferenceoverlay.la +plugin_LTLIBRARIES = libgstinferenceoverlayplugin.la -libgstinferenceoverlay_la_SOURCES = \ +libgstinferenceoverlayplugin_la_SOURCES = \ gstclassificationoverlay.cc \ gstdetectionoverlay.cc \ - gstinferenceoverlay.c \ + gstinferenceoverlayplugin.c \ gstembeddingoverlay.cc -libgstinferenceoverlay_la_CFLAGS = \ +libgstinferenceoverlayplugin_la_CFLAGS = \ $(GST_CFLAGS) \ $(GST_BASE_CFLAGS) \ $(GST_PLUGINS_BASE_CFLAGS) \ @@ -15,7 +15,7 @@ libgstinferenceoverlay_la_CFLAGS = \ -I$(top_srcdir)/gst-libs -libgstinferenceoverlay_la_CXXFLAGS = \ +libgstinferenceoverlayplugin_la_CXXFLAGS = \ $(GST_CXXFLAGS) \ $(GST_BASE_CFLAGS) \ $(GST_PLUGINS_BASE_CFLAGS) \ @@ -25,7 +25,7 @@ libgstinferenceoverlay_la_CXXFLAGS = \ -std=c++11 -libgstinferenceoverlay_la_LIBADD = \ +libgstinferenceoverlayplugin_la_LIBADD = \ $(GST_LIBS) \ $(GST_BASE_LIBS) \ $(GST_PLUGINS_BASE_LIBS) \ @@ -35,10 +35,10 @@ libgstinferenceoverlay_la_LIBADD = \ $(top_builddir)/gst-libs/gst/r2inference/libgstinference-@GST_API_VERSION@.la \ $(top_builddir)/gst-libs/gst/opencv/libgstinferenceoverlay-@GST_API_VERSION@.la -libgstinferenceoverlay_la_LDFLAGS = \ +libgstinferenceoverlayplugin_la_LDFLAGS = \ $(GST_PLUGIN_LDFLAGS) -libgstinferenceoverlay_la_LIBTOOLFLAGS = \ +libgstinferenceoverlayplugin_la_LIBTOOLFLAGS = \ $(GST_PLUGIN_LIBTOOLFLAGS) noinst_HEADERS = \ diff --git a/ext/opencv/gstinferenceoverlay.c b/ext/opencv/gstinferenceoverlayplugin.c similarity index 100% rename from ext/opencv/gstinferenceoverlay.c rename to ext/opencv/gstinferenceoverlayplugin.c From 4b14499ee3f208e4ddb008a880f695f77356377f Mon Sep 17 00:00:00 2001 From: tvlenin Date: Tue, 17 Dec 2019 13:30:43 -0600 Subject: [PATCH 105/188] Refactor internal names to add the word base --- gst-libs/gst/opencv/Makefile.am | 12 +-- gst-libs/gst/opencv/gstinferencebaseoverlay.c | 97 ++++++++++--------- gst-libs/gst/opencv/gstinferencebaseoverlay.h | 12 +-- 3 files changed, 65 insertions(+), 56 deletions(-) diff --git a/gst-libs/gst/opencv/Makefile.am b/gst-libs/gst/opencv/Makefile.am index 337546c6..7f94479d 100644 --- a/gst-libs/gst/opencv/Makefile.am +++ b/gst-libs/gst/opencv/Makefile.am @@ -1,23 +1,23 @@ -lib_LTLIBRARIES = libgstinferenceoverlay-@GST_API_VERSION@.la +lib_LTLIBRARIES = libgstinferencebaseoverlay-@GST_API_VERSION@.la -libgstinferenceoverlay_@GST_API_VERSION@_la_SOURCES= \ +libgstinferencebaseoverlay_@GST_API_VERSION@_la_SOURCES= \ gstinferencebaseoverlay.c -libgstinferenceoverlay_@GST_API_VERSION@_la_CFLAGS= \ +libgstinferencebaseoverlay_@GST_API_VERSION@_la_CFLAGS= \ $(GST_CFLAGS) \ $(GST_BASE_CFLAGS) \ $(GST_PLUGINS_BASE_CFLAGS) \ $(R2INFERENCE_CFLAGS) -libgstinferenceoverlay_@GST_API_VERSION@_la_LIBADD= \ +libgstinferencebaseoverlay_@GST_API_VERSION@_la_LIBADD= \ $(GST_LIBS) \ $(GST_BASE_LIBS) \ -lgstvideo-@GST_API_VERSION@ \ $(GST_PLUGINS_BASE_LIBS) \ $(R2INFERENCE_LIBS) -gstinferenceoverlayincludedir=@includedir@/gstreamer-@GST_API_VERSION@/gst/opencv/ +gstinferencebaseoverlayincludedir=@includedir@/gstreamer-@GST_API_VERSION@/gst/opencv/ -gstinferenceoverlayinclude_HEADERS= \ +gstinferencebaseoverlayinclude_HEADERS= \ gstinferencebaseoverlay.h diff --git a/gst-libs/gst/opencv/gstinferencebaseoverlay.c b/gst-libs/gst/opencv/gstinferencebaseoverlay.c index 6d065da0..a16a9b2d 100644 --- a/gst-libs/gst/opencv/gstinferencebaseoverlay.c +++ b/gst-libs/gst/opencv/gstinferencebaseoverlay.c @@ -29,8 +29,8 @@ #define VIDEO_SINK_CAPS \ GST_VIDEO_CAPS_MAKE("{RGB, RGBx, RGBA, BGR, BGRx, BGRA, xRGB, ARGB, xBGR, ABGR}") -GST_DEBUG_CATEGORY_STATIC (gst_inference_overlay_debug_category); -#define GST_CAT_DEFAULT gst_inference_overlay_debug_category +GST_DEBUG_CATEGORY_STATIC (gst_inference_base_overlay_debug_category); +#define GST_CAT_DEFAULT gst_inference_base_overlay_debug_category #define MIN_FONT_SCALE 0 #define DEFAULT_FONT_SCALE 2 @@ -59,32 +59,32 @@ struct _GstInferenceBaseOverlayPrivate gint num_labels; }; /* prototypes */ -static void gst_inference_overlay_set_property (GObject * object, +static void gst_inference_base_overlay_set_property (GObject * object, guint property_id, const GValue * value, GParamSpec * pspec); -static void gst_inference_overlay_get_property (GObject * object, +static void gst_inference_base_overlay_get_property (GObject * object, guint property_id, GValue * value, GParamSpec * pspec); -static void gst_inference_overlay_dispose (GObject * object); -static void gst_inference_overlay_finalize (GObject * object); +static void gst_inference_base_overlay_dispose (GObject * object); +static void gst_inference_base_overlay_finalize (GObject * object); -static gboolean gst_inference_overlay_start (GstBaseTransform * trans); -static gboolean gst_inference_overlay_stop (GstBaseTransform * trans); +static gboolean gst_inference_base_overlay_start (GstBaseTransform * trans); +static gboolean gst_inference_base_overlay_stop (GstBaseTransform * trans); static GstFlowReturn -gst_inference_overlay_transform_frame_ip (GstVideoFilter * trans, +gst_inference_base_overlay_transform_frame_ip (GstVideoFilter * trans, GstVideoFrame * frame); /* class initialization */ -G_DEFINE_TYPE_WITH_CODE (GstInferenceBaseOverlay, gst_inference_overlay, +G_DEFINE_TYPE_WITH_CODE (GstInferenceBaseOverlay, gst_inference_base_overlay, GST_TYPE_VIDEO_FILTER, - GST_DEBUG_CATEGORY_INIT (gst_inference_overlay_debug_category, + GST_DEBUG_CATEGORY_INIT (gst_inference_base_overlay_debug_category, "inferenceoverlay", 0, "debug category for inferenceoverlay class"); G_ADD_PRIVATE (GstInferenceBaseOverlay)); -#define GST_INFERENCE_OVERLAY_PRIVATE(self) \ - (GstInferenceBaseOverlayPrivate *)(gst_inference_overlay_get_instance_private (self)) +#define GST_INFERENCE_BASE_OVERLAY_PRIVATE(self) \ + (GstInferenceBaseOverlayPrivate *)(gst_inference_base_overlay_get_instance_private (self)) static void -gst_inference_overlay_class_init (GstInferenceBaseOverlayClass * klass) +gst_inference_base_overlay_class_init (GstInferenceBaseOverlayClass * klass) { GObjectClass *gobject_class = G_OBJECT_CLASS (klass); GstBaseTransformClass *base_transform_class = @@ -98,10 +98,10 @@ gst_inference_overlay_class_init (GstInferenceBaseOverlayClass * klass) gst_pad_template_new ("sink", GST_PAD_SINK, GST_PAD_ALWAYS, gst_caps_from_string (VIDEO_SINK_CAPS))); - gobject_class->set_property = gst_inference_overlay_set_property; - gobject_class->get_property = gst_inference_overlay_get_property; - gobject_class->dispose = gst_inference_overlay_dispose; - gobject_class->finalize = gst_inference_overlay_finalize; + gobject_class->set_property = gst_inference_base_overlay_set_property; + gobject_class->get_property = gst_inference_base_overlay_get_property; + gobject_class->dispose = gst_inference_base_overlay_dispose; + gobject_class->finalize = gst_inference_base_overlay_finalize; g_object_class_install_property (gobject_class, PROP_FONT_SCALE, g_param_spec_double ("font-scale", "font", "Font scale", MIN_FONT_SCALE, @@ -117,18 +117,20 @@ gst_inference_overlay_class_init (GstInferenceBaseOverlayClass * klass) "Semicolon separated string containing inference labels", DEFAULT_LABELS, G_PARAM_READWRITE)); - base_transform_class->start = GST_DEBUG_FUNCPTR (gst_inference_overlay_start); - base_transform_class->stop = GST_DEBUG_FUNCPTR (gst_inference_overlay_stop); + base_transform_class->start = + GST_DEBUG_FUNCPTR (gst_inference_base_overlay_start); + base_transform_class->stop = + GST_DEBUG_FUNCPTR (gst_inference_base_overlay_stop); video_filter_class->transform_frame_ip = - GST_DEBUG_FUNCPTR (gst_inference_overlay_transform_frame_ip); + GST_DEBUG_FUNCPTR (gst_inference_base_overlay_transform_frame_ip); } static void -gst_inference_overlay_init (GstInferenceBaseOverlay * inference_overlay) +gst_inference_base_overlay_init (GstInferenceBaseOverlay * inference_overlay) { GstInferenceBaseOverlayPrivate *priv = - GST_INFERENCE_OVERLAY_PRIVATE (inference_overlay); + GST_INFERENCE_BASE_OVERLAY_PRIVATE (inference_overlay); priv->font_scale = DEFAULT_FONT_SCALE; priv->thickness = DEFAULT_THICKNESS; priv->labels = DEFAULT_LABELS; @@ -137,12 +139,13 @@ gst_inference_overlay_init (GstInferenceBaseOverlay * inference_overlay) } void -gst_inference_overlay_set_property (GObject * object, guint property_id, +gst_inference_base_overlay_set_property (GObject * object, guint property_id, const GValue * value, GParamSpec * pspec) { - GstInferenceBaseOverlay *inference_overlay = GST_INFERENCE_OVERLAY (object); + GstInferenceBaseOverlay *inference_overlay = + GST_INFERENCE_BASE_OVERLAY (object); GstInferenceBaseOverlayPrivate *priv = - GST_INFERENCE_OVERLAY_PRIVATE (inference_overlay); + GST_INFERENCE_BASE_OVERLAY_PRIVATE (inference_overlay); GST_DEBUG_OBJECT (inference_overlay, "set_property"); @@ -177,12 +180,13 @@ gst_inference_overlay_set_property (GObject * object, guint property_id, } void -gst_inference_overlay_get_property (GObject * object, guint property_id, +gst_inference_base_overlay_get_property (GObject * object, guint property_id, GValue * value, GParamSpec * pspec) { - GstInferenceBaseOverlay *inference_overlay = GST_INFERENCE_OVERLAY (object); + GstInferenceBaseOverlay *inference_overlay = + GST_INFERENCE_BASE_OVERLAY (object); GstInferenceBaseOverlayPrivate *priv = - GST_INFERENCE_OVERLAY_PRIVATE (inference_overlay); + GST_INFERENCE_BASE_OVERLAY_PRIVATE (inference_overlay); GST_DEBUG_OBJECT (inference_overlay, "get_property"); @@ -203,11 +207,12 @@ gst_inference_overlay_get_property (GObject * object, guint property_id, } void -gst_inference_overlay_dispose (GObject * object) +gst_inference_base_overlay_dispose (GObject * object) { - GstInferenceBaseOverlay *inference_overlay = GST_INFERENCE_OVERLAY (object); + GstInferenceBaseOverlay *inference_overlay = + GST_INFERENCE_BASE_OVERLAY (object); GstInferenceBaseOverlayPrivate *priv = - GST_INFERENCE_OVERLAY_PRIVATE (inference_overlay); + GST_INFERENCE_BASE_OVERLAY_PRIVATE (inference_overlay); GST_DEBUG_OBJECT (inference_overlay, "dispose"); @@ -219,25 +224,27 @@ gst_inference_overlay_dispose (GObject * object) g_free (priv->labels); } - G_OBJECT_CLASS (gst_inference_overlay_parent_class)->dispose (object); + G_OBJECT_CLASS (gst_inference_base_overlay_parent_class)->dispose (object); } void -gst_inference_overlay_finalize (GObject * object) +gst_inference_base_overlay_finalize (GObject * object) { - GstInferenceBaseOverlay *inference_overlay = GST_INFERENCE_OVERLAY (object); + GstInferenceBaseOverlay *inference_overlay = + GST_INFERENCE_BASE_OVERLAY (object); GST_DEBUG_OBJECT (inference_overlay, "finalize"); /* clean up object here */ - G_OBJECT_CLASS (gst_inference_overlay_parent_class)->finalize (object); + G_OBJECT_CLASS (gst_inference_base_overlay_parent_class)->finalize (object); } static gboolean -gst_inference_overlay_start (GstBaseTransform * trans) +gst_inference_base_overlay_start (GstBaseTransform * trans) { - GstInferenceBaseOverlay *inference_overlay = GST_INFERENCE_OVERLAY (trans); + GstInferenceBaseOverlay *inference_overlay = + GST_INFERENCE_BASE_OVERLAY (trans); GST_DEBUG_OBJECT (inference_overlay, "start"); @@ -245,9 +252,10 @@ gst_inference_overlay_start (GstBaseTransform * trans) } static gboolean -gst_inference_overlay_stop (GstBaseTransform * trans) +gst_inference_base_overlay_stop (GstBaseTransform * trans) { - GstInferenceBaseOverlay *inference_overlay = GST_INFERENCE_OVERLAY (trans); + GstInferenceBaseOverlay *inference_overlay = + GST_INFERENCE_BASE_OVERLAY (trans); GST_DEBUG_OBJECT (inference_overlay, "stop"); @@ -256,14 +264,15 @@ gst_inference_overlay_stop (GstBaseTransform * trans) /* transform */ static GstFlowReturn -gst_inference_overlay_transform_frame_ip (GstVideoFilter * trans, +gst_inference_base_overlay_transform_frame_ip (GstVideoFilter * trans, GstVideoFrame * frame) { GstInferenceBaseOverlayClass *io_class = - GST_INFERENCE_OVERLAY_GET_CLASS (trans); - GstInferenceBaseOverlay *inference_overlay = GST_INFERENCE_OVERLAY (trans); + GST_INFERENCE_BASE_OVERLAY_GET_CLASS (trans); + GstInferenceBaseOverlay *inference_overlay = + GST_INFERENCE_BASE_OVERLAY (trans); GstInferenceBaseOverlayPrivate *priv = - GST_INFERENCE_OVERLAY_PRIVATE (inference_overlay); + GST_INFERENCE_BASE_OVERLAY_PRIVATE (inference_overlay); GstMeta *meta; GstFlowReturn ret = GST_FLOW_ERROR; diff --git a/gst-libs/gst/opencv/gstinferencebaseoverlay.h b/gst-libs/gst/opencv/gstinferencebaseoverlay.h index c9602139..491bb21c 100644 --- a/gst-libs/gst/opencv/gstinferencebaseoverlay.h +++ b/gst-libs/gst/opencv/gstinferencebaseoverlay.h @@ -19,23 +19,23 @@ * */ -#ifndef __GST_INFERENCE_OVERLAY_H__ -#define __GST_INFERENCE_OVERLAY_H__ +#ifndef __GST_INFERENCE_BASE_OVERLAY_H__ +#define __GST_INFERENCE_BASE_OVERLAY_H__ #include #include G_BEGIN_DECLS -#define GST_TYPE_INFERENCE_OVERLAY gst_inference_overlay_get_type () -G_DECLARE_DERIVABLE_TYPE (GstInferenceBaseOverlay, gst_inference_overlay, GST, - INFERENCE_OVERLAY, GstVideoFilter); +#define GST_TYPE_INFERENCE_BASE_OVERLAY gst_inference_base_overlay_get_type () +G_DECLARE_DERIVABLE_TYPE (GstInferenceBaseOverlay, gst_inference_base_overlay, GST, + INFERENCE_BASE_OVERLAY, GstVideoFilter); struct _GstInferenceBaseOverlayClass { GstVideoFilterClass parent_class; - GstFlowReturn (* process_meta) (GstInferenceBaseOverlay * inference_overlay, + GstFlowReturn (* process_meta) (GstInferenceBaseOverlay * inference_base_overlay, GstVideoFrame * frame, GstMeta* meta, gdouble font_scale, gint thickness, gchar **labels_list, gint num_labels); From 9070a54a424e39ec85ae9060440fc2b75cafe6d3 Mon Sep 17 00:00:00 2001 From: tvlenin Date: Tue, 17 Dec 2019 13:32:32 -0600 Subject: [PATCH 106/188] Refactor function names --- ext/opencv/Makefile.am | 4 +++- ext/opencv/gstclassificationoverlay.cc | 4 ++-- ext/opencv/gstdetectionoverlay.cc | 4 ++-- ext/opencv/gstembeddingoverlay.cc | 4 ++-- ext/opencv/gstinferenceoverlayplugin.c | 2 +- 5 files changed, 10 insertions(+), 8 deletions(-) diff --git a/ext/opencv/Makefile.am b/ext/opencv/Makefile.am index 4c4594ba..b55ade76 100644 --- a/ext/opencv/Makefile.am +++ b/ext/opencv/Makefile.am @@ -4,6 +4,7 @@ libgstinferenceoverlayplugin_la_SOURCES = \ gstclassificationoverlay.cc \ gstdetectionoverlay.cc \ gstinferenceoverlayplugin.c \ + gstinferenceoverlay.cc \ gstembeddingoverlay.cc libgstinferenceoverlayplugin_la_CFLAGS = \ @@ -33,7 +34,7 @@ libgstinferenceoverlayplugin_la_LIBADD = \ $(OPENCV_LIBS) \ $(R2INFERENCE_LIBS) \ $(top_builddir)/gst-libs/gst/r2inference/libgstinference-@GST_API_VERSION@.la \ - $(top_builddir)/gst-libs/gst/opencv/libgstinferenceoverlay-@GST_API_VERSION@.la + $(top_builddir)/gst-libs/gst/opencv/libgstinferencebaseoverlay-@GST_API_VERSION@.la libgstinferenceoverlayplugin_la_LDFLAGS = \ $(GST_PLUGIN_LDFLAGS) @@ -44,4 +45,5 @@ libgstinferenceoverlayplugin_la_LIBTOOLFLAGS = \ noinst_HEADERS = \ gstclassificationoverlay.h \ gstdetectionoverlay.h \ + gstinferenceoverlay.h \ gstembeddingoverlay.h diff --git a/ext/opencv/gstclassificationoverlay.cc b/ext/opencv/gstclassificationoverlay.cc index d5ca1cda..585aa9fd 100644 --- a/ext/opencv/gstclassificationoverlay.cc +++ b/ext/opencv/gstclassificationoverlay.cc @@ -62,7 +62,7 @@ struct _GstClassificationOverlayClass /* class initialization */ G_DEFINE_TYPE_WITH_CODE (GstClassificationOverlay, gst_classification_overlay, - GST_TYPE_INFERENCE_OVERLAY, + GST_TYPE_INFERENCE_BASE_OVERLAY, GST_DEBUG_CATEGORY_INIT (gst_classification_overlay_debug_category, "classificationoverlay", 0, "debug category for classification_overlay element")); @@ -70,7 +70,7 @@ G_DEFINE_TYPE_WITH_CODE (GstClassificationOverlay, gst_classification_overlay, static void gst_classification_overlay_class_init (GstClassificationOverlayClass * klass) { - GstInferenceBaseOverlayClass *io_class = GST_INFERENCE_OVERLAY_CLASS (klass); + GstInferenceBaseOverlayClass *io_class = GST_INFERENCE_BASE_OVERLAY_CLASS (klass); gst_element_class_set_static_metadata (GST_ELEMENT_CLASS (klass), "classificationoverlay", "Filter", diff --git a/ext/opencv/gstdetectionoverlay.cc b/ext/opencv/gstdetectionoverlay.cc index d36c093e..ba6dc3cf 100644 --- a/ext/opencv/gstdetectionoverlay.cc +++ b/ext/opencv/gstdetectionoverlay.cc @@ -73,14 +73,14 @@ struct _GstDetectionOverlayClass /* class initialization */ G_DEFINE_TYPE_WITH_CODE (GstDetectionOverlay, gst_detection_overlay, - GST_TYPE_INFERENCE_OVERLAY, + GST_TYPE_INFERENCE_BASE_OVERLAY, GST_DEBUG_CATEGORY_INIT (gst_detection_overlay_debug_category, "detectionoverlay", 0, "debug category for detection_overlay element")); static void gst_detection_overlay_class_init (GstDetectionOverlayClass * klass) { - GstInferenceBaseOverlayClass *io_class = GST_INFERENCE_OVERLAY_CLASS (klass); + GstInferenceBaseOverlayClass *io_class = GST_INFERENCE_BASE_OVERLAY_CLASS (klass); gst_element_class_set_static_metadata (GST_ELEMENT_CLASS (klass), "detectionoverlay", "Filter", diff --git a/ext/opencv/gstembeddingoverlay.cc b/ext/opencv/gstembeddingoverlay.cc index 29a370af..ba039abb 100644 --- a/ext/opencv/gstembeddingoverlay.cc +++ b/ext/opencv/gstembeddingoverlay.cc @@ -84,7 +84,7 @@ struct _GstClassificationOverlayClass /* class initialization */ G_DEFINE_TYPE_WITH_CODE (GstEmbeddingOverlay, gst_embedding_overlay, - GST_TYPE_INFERENCE_OVERLAY, + GST_TYPE_INFERENCE_BASE_OVERLAY, GST_DEBUG_CATEGORY_INIT (gst_embedding_overlay_debug_category, "embeddingoverlay", 0, "debug category for embedding_overlay element")); @@ -92,7 +92,7 @@ static void gst_embedding_overlay_class_init (GstEmbeddingOverlayClass * klass) { GObjectClass *gobject_class = G_OBJECT_CLASS (klass); - GstInferenceBaseOverlayClass *io_class = GST_INFERENCE_OVERLAY_CLASS (klass); + GstInferenceBaseOverlayClass *io_class = GST_INFERENCE_BASE_OVERLAY_CLASS (klass); gobject_class->set_property = gst_embedding_overlay_set_property; gobject_class->get_property = gst_embedding_overlay_get_property; diff --git a/ext/opencv/gstinferenceoverlayplugin.c b/ext/opencv/gstinferenceoverlayplugin.c index 47ad56cc..a03de3c7 100644 --- a/ext/opencv/gstinferenceoverlayplugin.c +++ b/ext/opencv/gstinferenceoverlayplugin.c @@ -60,6 +60,6 @@ plugin_init (GstPlugin * plugin) GST_PLUGIN_DEFINE (GST_VERSION_MAJOR, GST_VERSION_MINOR, - inferenceoverlay, + inferenceoverlayplugin, "Create overlays on incomming image frames with proper inference metadata", plugin_init, VERSION, "LGPL", PACKAGE_NAME, GST_PACKAGE_ORIGIN) From fc5971ca5cef452a578b83c1e5d6b20fb0f80c0f Mon Sep 17 00:00:00 2001 From: tvlenin Date: Tue, 17 Dec 2019 13:56:33 -0600 Subject: [PATCH 107/188] Add new gstinference overlay --- ext/opencv/gstinferenceoverlay.cc | 160 +++++++++++++++++++++++++ ext/opencv/gstinferenceoverlay.h | 34 ++++++ ext/opencv/gstinferenceoverlayplugin.c | 8 +- 3 files changed, 201 insertions(+), 1 deletion(-) create mode 100644 ext/opencv/gstinferenceoverlay.cc create mode 100644 ext/opencv/gstinferenceoverlay.h diff --git a/ext/opencv/gstinferenceoverlay.cc b/ext/opencv/gstinferenceoverlay.cc new file mode 100644 index 00000000..216dfe1b --- /dev/null +++ b/ext/opencv/gstinferenceoverlay.cc @@ -0,0 +1,160 @@ +/* + * GStreamer + * Copyright (C) 2018 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + */ +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "gstinferenceoverlay.h" +#include "gst/r2inference/gstinferencemeta.h" +#ifdef OCV_VERSION_LT_3_2 +#include "opencv2/highgui/highgui.hpp" +#else +#include "opencv2/imgproc.hpp" +#include "opencv2/highgui.hpp" +#endif + +static const +cv::Scalar +colors[] = { + cv::Scalar (254, 254, 254), cv::Scalar (239, 211, 127), + cv::Scalar (225, 169, 0), cv::Scalar (211, 127, 254), + cv::Scalar (197, 84, 127), cv::Scalar (183, 42, 0), + cv::Scalar (169, 0.0, 254), cv::Scalar (155, 42, 127), + cv::Scalar (141, 84, 0), cv::Scalar (127, 254, 254), + cv::Scalar (112, 211, 127), cv::Scalar (98, 169, 0), + cv::Scalar (84, 127, 254), cv::Scalar (70, 84, 127), + cv::Scalar (56, 42, 0), cv::Scalar (42, 0, 254), + cv::Scalar (28, 42, 127), cv::Scalar (14, 84, 0), + cv::Scalar (0, 254, 254), cv::Scalar (14, 211, 127) +}; + +#define N_C (sizeof (colors)/sizeof (cv::Scalar)) + +GST_DEBUG_CATEGORY_STATIC (gst_inference_overlay_debug_category); +#define GST_CAT_DEFAULT gst_inference_overlay_debug_category + +/* prototypes */ +static +GstFlowReturn +gst_inference_overlay_process_meta (GstInferenceBaseOverlay *inference_overlay, + GstVideoFrame *frame, GstMeta *meta, gdouble font_scale, gint thickness, + gchar **labels_list, gint num_labels); + +enum { + PROP_0 +}; + +struct _GstInferenceOverlay { + GstInferenceBaseOverlay + parent; +}; + +struct _GstInferenceOverlayClass { + GstInferenceBaseOverlay + parent; +}; + +/* class initialization */ + +G_DEFINE_TYPE_WITH_CODE (GstInferenceOverlay, gst_inference_overlay, + GST_TYPE_INFERENCE_BASE_OVERLAY, + GST_DEBUG_CATEGORY_INIT (gst_inference_overlay_debug_category, + "inferenceoverlay", 0, "debug category for inference_overlay element")); + +static void +gst_inference_overlay_class_init (GstInferenceOverlayClass *klass) { + GstInferenceBaseOverlayClass * + io_class = GST_INFERENCE_BASE_OVERLAY_CLASS (klass); + + gst_element_class_set_static_metadata (GST_ELEMENT_CLASS (klass), + "inferenceoverlay", "Filter", + "Overlays Inferece metadata on input buffer", + "Carlos Rodriguez \n\t\t\t" + " Jose Jimenez \n\t\t\t" + " Michael Gruner \n\t\t\t" + " Carlos Aguero \n\t\t\t" + " Miguel Taylor \n\t\t\t" + " Greivin Fallas "); + + io_class->process_meta = + GST_DEBUG_FUNCPTR (gst_inference_overlay_process_meta); + io_class->meta_type = GST_INFERENCE_META_API_TYPE; +} + +static void +gst_inference_overlay_init (GstInferenceOverlay *inference_overlay) { +} + +static +GstFlowReturn +gst_inference_overlay_process_meta (GstInferenceBaseOverlay *inference_overlay, + GstVideoFrame *frame, GstMeta *meta, gdouble font_scale, gint thickness, + gchar **labels_list, gint num_labels) { + GstInferenceMeta * + detect_meta; + gint + i, + width, + height, + channels; + cv::Mat cv_mat; + cv::Size size; + cv::String str; + BBox + box; + + switch (GST_VIDEO_FRAME_FORMAT (frame)) { + case GST_VIDEO_FORMAT_RGB: + case GST_VIDEO_FORMAT_BGR: + channels = 3; + break; + default: + channels = 4; + break; + } + width = GST_VIDEO_FRAME_COMP_STRIDE (frame, 0) / channels; + height = GST_VIDEO_FRAME_HEIGHT (frame); + + + detect_meta = (GstInferenceMeta *) meta; + g_print ("Id = %d \n ", detect_meta->prediction->id); + cv_mat = cv::Mat (height, width, CV_MAKETYPE (CV_8U, channels), + (char *) frame->data[0]); + for (i = 0; i < 3; ++i) { + Prediction * + predict = &detect_meta->prediction->predictions[i]; + box = *predict->box; + + if (num_labels > box.label) { + str = labels_list[box.label]; + } else { + str = cv::format ("Label #%d", box.label); + } + + cv::putText (cv_mat, str, cv::Point (box.x, box.y - 5), + cv::FONT_HERSHEY_PLAIN, font_scale, colors[box.label % N_C], thickness); + cv::rectangle (cv_mat, cv::Point (box.x, box.y), + cv::Point (box.x + box.width, box.y + box.height), + colors[box.label % N_C], thickness); + } + + return GST_FLOW_OK; +} diff --git a/ext/opencv/gstinferenceoverlay.h b/ext/opencv/gstinferenceoverlay.h new file mode 100644 index 00000000..42a51725 --- /dev/null +++ b/ext/opencv/gstinferenceoverlay.h @@ -0,0 +1,34 @@ +/* + * GStreamer + * Copyright (C) 2019 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + */ + +#ifndef _GST_INFERENCE_OVERLAY_H_ +#define _GST_INFERENCE_OVERLAY_H_ + +#include + +G_BEGIN_DECLS + +#define GST_TYPE_INFERENCE_OVERLAY (gst_inference_overlay_get_type()) +G_DECLARE_FINAL_TYPE (GstInferenceOverlay, gst_inference_overlay, GST, INFERENCE_OVERLAY, GstInferenceBaseOverlay) + +G_END_DECLS + +#endif diff --git a/ext/opencv/gstinferenceoverlayplugin.c b/ext/opencv/gstinferenceoverlayplugin.c index a03de3c7..4bd1ba1f 100644 --- a/ext/opencv/gstinferenceoverlayplugin.c +++ b/ext/opencv/gstinferenceoverlayplugin.c @@ -26,6 +26,7 @@ #include "gstclassificationoverlay.h" #include "gstdetectionoverlay.h" #include "gstembeddingoverlay.h" +#include "gstinferenceoverlay.h" static gboolean plugin_init (GstPlugin * plugin) @@ -52,7 +53,12 @@ plugin_init (GstPlugin * plugin) if (!ret) { goto out; } - + ret = + gst_element_register (plugin, "inferenceoverlay", GST_RANK_NONE, + GST_TYPE_INFERENCE_OVERLAY); + if (!ret) { + goto out; + } out: return ret; From c26f4c85455664c0e1b394fedb0233dc3d9cfea9 Mon Sep 17 00:00:00 2001 From: tvlenin Date: Tue, 17 Dec 2019 16:31:11 -0600 Subject: [PATCH 108/188] Draw inferencemeta information in the output image --- ext/opencv/gstinferenceoverlay.cc | 63 +++++++++++++++++-------------- 1 file changed, 35 insertions(+), 28 deletions(-) diff --git a/ext/opencv/gstinferenceoverlay.cc b/ext/opencv/gstinferenceoverlay.cc index 216dfe1b..2172cd1f 100644 --- a/ext/opencv/gstinferenceoverlay.cc +++ b/ext/opencv/gstinferenceoverlay.cc @@ -103,6 +103,37 @@ static void gst_inference_overlay_init (GstInferenceOverlay *inference_overlay) { } +static +void +gst_get_meta (Prediction *pred, cv::Mat cv_mat, gdouble font_scale, + gint thickness, + gchar **labels_list, gint num_labels) { + gint i; + cv::Size size; + cv::String label; + cv::String prob; + BBox box; + for (i = 0; i < pred->num_predictions ; ++i) { + Prediction *predict = &pred->predictions[i]; + gst_get_meta (predict, cv_mat, font_scale, thickness, + labels_list, num_labels); + } + if (NULL != pred->box) { + box = *pred->box; + if (num_labels > box.label) { + label = cv::format ("%s Prob: %f", labels_list[box.label], box.prob); + } else { + label = cv::format ("Label #%d Prob: %f", box.label, box.prob); + } + cv::putText (cv_mat, label, cv::Point (box.x, box.y - 5), + cv::FONT_HERSHEY_PLAIN, font_scale, colors[box.label % N_C], thickness); + cv::rectangle (cv_mat, cv::Point (box.x, box.y), + cv::Point (box.x + box.width, box.y + box.height), + colors[box.label % N_C], thickness); + + } +} + static GstFlowReturn gst_inference_overlay_process_meta (GstInferenceBaseOverlay *inference_overlay, @@ -110,16 +141,8 @@ gst_inference_overlay_process_meta (GstInferenceBaseOverlay *inference_overlay, gchar **labels_list, gint num_labels) { GstInferenceMeta * detect_meta; - gint - i, - width, - height, - channels; + gint width, height, channels; cv::Mat cv_mat; - cv::Size size; - cv::String str; - BBox - box; switch (GST_VIDEO_FRAME_FORMAT (frame)) { case GST_VIDEO_FORMAT_RGB: @@ -133,28 +156,12 @@ gst_inference_overlay_process_meta (GstInferenceBaseOverlay *inference_overlay, width = GST_VIDEO_FRAME_COMP_STRIDE (frame, 0) / channels; height = GST_VIDEO_FRAME_HEIGHT (frame); - detect_meta = (GstInferenceMeta *) meta; - g_print ("Id = %d \n ", detect_meta->prediction->id); + cv_mat = cv::Mat (height, width, CV_MAKETYPE (CV_8U, channels), (char *) frame->data[0]); - for (i = 0; i < 3; ++i) { - Prediction * - predict = &detect_meta->prediction->predictions[i]; - box = *predict->box; - - if (num_labels > box.label) { - str = labels_list[box.label]; - } else { - str = cv::format ("Label #%d", box.label); - } - - cv::putText (cv_mat, str, cv::Point (box.x, box.y - 5), - cv::FONT_HERSHEY_PLAIN, font_scale, colors[box.label % N_C], thickness); - cv::rectangle (cv_mat, cv::Point (box.x, box.y), - cv::Point (box.x + box.width, box.y + box.height), - colors[box.label % N_C], thickness); - } + gst_get_meta (detect_meta->prediction, cv_mat, font_scale, thickness, + labels_list, num_labels); return GST_FLOW_OK; } From 4d4ba27362947c7c5524d63d00e43af23221ae0d Mon Sep 17 00:00:00 2001 From: tvlenin Date: Thu, 9 Jan 2020 15:21:11 -0600 Subject: [PATCH 109/188] Add support to new InferenceMeta --- ext/opencv/gstinferenceoverlay.cc | 35 +++++++++++++++++-------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/ext/opencv/gstinferenceoverlay.cc b/ext/opencv/gstinferenceoverlay.cc index 2172cd1f..10f333f5 100644 --- a/ext/opencv/gstinferenceoverlay.cc +++ b/ext/opencv/gstinferenceoverlay.cc @@ -105,33 +105,36 @@ gst_inference_overlay_init (GstInferenceOverlay *inference_overlay) { static void -gst_get_meta (Prediction *pred, cv::Mat cv_mat, gdouble font_scale, +gst_get_meta (GstInferencePrediction *pred, cv::Mat cv_mat, gdouble font_scale, gint thickness, gchar **labels_list, gint num_labels) { - gint i; + guint i; cv::Size size; cv::String label; + GList *iter = NULL; cv::String prob; - BBox box; - for (i = 0; i < pred->num_predictions ; ++i) { - Prediction *predict = &pred->predictions[i]; + BoundingBox box; + + for (i = 0; i < g_node_n_children(pred->predictions) ; ++i) { + GstInferencePrediction *predict = (GstInferencePrediction*)g_node_nth_child (pred->predictions,i)->data; gst_get_meta (predict, cv_mat, font_scale, thickness, labels_list, num_labels); } - if (NULL != pred->box) { - box = *pred->box; - if (num_labels > box.label) { - label = cv::format ("%s Prob: %f", labels_list[box.label], box.prob); - } else { - label = cv::format ("Label #%d Prob: %f", box.label, box.prob); + box = pred->bbox; + + for (iter = pred->classifications; iter != NULL; iter = g_list_next(iter)) { + GstInferenceClassification *classification = (GstInferenceClassification *)iter->data; + if (num_labels > classification->class_id) { + label = cv::format ("%s Prob: %f", labels_list[classification->class_id], classification->class_prob); + }else{ + label = cv::format ("Label #%d Prob: %f", classification->class_id, classification->class_prob); } cv::putText (cv_mat, label, cv::Point (box.x, box.y - 5), - cv::FONT_HERSHEY_PLAIN, font_scale, colors[box.label % N_C], thickness); - cv::rectangle (cv_mat, cv::Point (box.x, box.y), - cv::Point (box.x + box.width, box.y + box.height), - colors[box.label % N_C], thickness); - + cv::FONT_HERSHEY_PLAIN, font_scale, colors[classification->class_id % N_C], thickness); } + cv::rectangle (cv_mat, cv::Point (box.x, box.y), + cv::Point (box.x + box.width, box.y + box.height), + colors[14 % N_C], thickness); } static From d76d73f428d1a2030eb9a5e05233352e4b354ec7 Mon Sep 17 00:00:00 2001 From: tvlenin Date: Mon, 13 Jan 2020 08:25:06 -0600 Subject: [PATCH 110/188] Avoid overlap of classification labels --- ext/opencv/gstinferenceoverlay.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opencv/gstinferenceoverlay.cc b/ext/opencv/gstinferenceoverlay.cc index 10f333f5..3e189d32 100644 --- a/ext/opencv/gstinferenceoverlay.cc +++ b/ext/opencv/gstinferenceoverlay.cc @@ -129,7 +129,7 @@ gst_get_meta (GstInferencePrediction *pred, cv::Mat cv_mat, gdouble font_scale, }else{ label = cv::format ("Label #%d Prob: %f", classification->class_id, classification->class_prob); } - cv::putText (cv_mat, label, cv::Point (box.x, box.y - 5), + cv::putText (cv_mat, label, cv::Point (box.x, box.y - i * 5), cv::FONT_HERSHEY_PLAIN, font_scale, colors[classification->class_id % N_C], thickness); } cv::rectangle (cv_mat, cv::Point (box.x, box.y), From fa8ca2fb6722e58dfbc8538dbe59a90eb8a16756 Mon Sep 17 00:00:00 2001 From: tvlenin Date: Mon, 13 Jan 2020 13:41:22 -0600 Subject: [PATCH 111/188] Add dotted line to bounding boxes --- ext/opencv/gstinferenceoverlay.cc | 63 ++++++++++++++++++++++++++++++- 1 file changed, 61 insertions(+), 2 deletions(-) diff --git a/ext/opencv/gstinferenceoverlay.cc b/ext/opencv/gstinferenceoverlay.cc index 3e189d32..3505cc1a 100644 --- a/ext/opencv/gstinferenceoverlay.cc +++ b/ext/opencv/gstinferenceoverlay.cc @@ -57,6 +57,7 @@ GstFlowReturn gst_inference_overlay_process_meta (GstInferenceBaseOverlay *inference_overlay, GstVideoFrame *frame, GstMeta *meta, gdouble font_scale, gint thickness, gchar **labels_list, gint num_labels); +void draw_line(cv::Mat& img, cv::Point pt1, cv::Point pt2, cv::Scalar color, int thickness, std::string style, int gap); enum { PROP_0 @@ -102,7 +103,50 @@ gst_inference_overlay_class_init (GstInferenceOverlayClass *klass) { static void gst_inference_overlay_init (GstInferenceOverlay *inference_overlay) { } +void draw_line(cv::Mat& img, cv::Point pt1, cv::Point pt2, cv::Scalar color, int thickness, std::string style, int gap){ + float dx = pt1.x - pt2.x; + float dy = pt1.y - pt2.y; + + float dist = sqrt(dx*dx + dy*dy); + + std::vector pts; + for (int i = 0; i < dist; i += gap) + { + float r = (float)i / dist; + int x = int((pt1.x * (1.0 - r) + pt2.x * r) + .5); + int y = int((pt1.y * (1.0 - r) + pt2.y * r) + .5); + cv::Point p = cv::Point(x, y); + pts.push_back(p); + } + + int pts_size = pts.size(); + if (style == "dotted") + { + for (int i = 0; i < pts_size; i++) + { + cv::circle(img, pts[i], thickness, color, -1); + } + } + else + { + cv::Point s = pts[0]; + cv::Point e = pts[0]; + + //int count = 0; + + for (int i = 0; i < pts_size; i++) + { + s = e; + e = pts[i]; + + if (i % 2 == 1) + { + cv::line(img, s, e, color, thickness); + } + } + } +} static void gst_get_meta (GstInferencePrediction *pred, cv::Mat cv_mat, gdouble font_scale, @@ -115,6 +159,8 @@ gst_get_meta (GstInferencePrediction *pred, cv::Mat cv_mat, gdouble font_scale, cv::String prob; BoundingBox box; + + for (i = 0; i < g_node_n_children(pred->predictions) ; ++i) { GstInferencePrediction *predict = (GstInferencePrediction*)g_node_nth_child (pred->predictions,i)->data; gst_get_meta (predict, cv_mat, font_scale, thickness, @@ -132,9 +178,22 @@ gst_get_meta (GstInferencePrediction *pred, cv::Mat cv_mat, gdouble font_scale, cv::putText (cv_mat, label, cv::Point (box.x, box.y - i * 5), cv::FONT_HERSHEY_PLAIN, font_scale, colors[classification->class_id % N_C], thickness); } - cv::rectangle (cv_mat, cv::Point (box.x, box.y), + + if(FALSE == G_NODE_IS_ROOT(pred->predictions)){ + draw_line (cv_mat, cv::Point (box.x, box.y), + cv::Point (box.x + box.width, box.y), + colors[50 % N_C], thickness,"dotte",20); + draw_line (cv_mat, cv::Point (box.x, box.y), + cv::Point (box.x , box.y + box.height), + colors[50 % N_C], thickness,"dotte",20); + draw_line (cv_mat, cv::Point (box.x+ box.width, box.y), + cv::Point (box.x + box.width, box.y + box.height), + colors[50 % N_C], thickness,"dotte",20); + draw_line (cv_mat, cv::Point (box.x, box.y+ box.height), cv::Point (box.x + box.width, box.y + box.height), - colors[14 % N_C], thickness); + colors[50 % N_C], thickness,"dotte",20); + } + } static From 7d9b0924a8a901f137c796c03bbe0c59fb93b9bb Mon Sep 17 00:00:00 2001 From: tvlenin Date: Mon, 13 Jan 2020 15:15:21 -0600 Subject: [PATCH 112/188] Add alpha overlay to print classes --- ext/opencv/gstinferenceoverlay.cc | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/ext/opencv/gstinferenceoverlay.cc b/ext/opencv/gstinferenceoverlay.cc index 3505cc1a..b25d61f0 100644 --- a/ext/opencv/gstinferenceoverlay.cc +++ b/ext/opencv/gstinferenceoverlay.cc @@ -158,9 +158,10 @@ gst_get_meta (GstInferencePrediction *pred, cv::Mat cv_mat, gdouble font_scale, GList *iter = NULL; cv::String prob; BoundingBox box; - - - + int classes = 0; + double alpha = 0.5; + cv::Mat alpha_overlay; + for (i = 0; i < g_node_n_children(pred->predictions) ; ++i) { GstInferencePrediction *predict = (GstInferencePrediction*)g_node_nth_child (pred->predictions,i)->data; gst_get_meta (predict, cv_mat, font_scale, thickness, @@ -170,15 +171,22 @@ gst_get_meta (GstInferencePrediction *pred, cv::Mat cv_mat, gdouble font_scale, for (iter = pred->classifications; iter != NULL; iter = g_list_next(iter)) { GstInferenceClassification *classification = (GstInferenceClassification *)iter->data; + classes++; if (num_labels > classification->class_id) { label = cv::format ("%s Prob: %f", labels_list[classification->class_id], classification->class_prob); }else{ label = cv::format ("Label #%d Prob: %f", classification->class_id, classification->class_prob); } - cv::putText (cv_mat, label, cv::Point (box.x, box.y - i * 5), - cv::FONT_HERSHEY_PLAIN, font_scale, colors[classification->class_id % N_C], thickness); + cv::putText (cv_mat, label, cv::Point (box.x + box.width , box.y + classes * 30), + cv::FONT_HERSHEY_PLAIN, font_scale,cv::Scalar::all(0), thickness); } + cv_mat.copyTo(alpha_overlay); + cv::Size text = cv::getTextSize (label, cv::FONT_HERSHEY_PLAIN, font_scale, thickness, 0); + cv::rectangle(alpha_overlay, cv::Rect(box.x + box.width, box.y, text.width, 50*classes), cv::Scalar(255, 255, 255), -1); + cv::addWeighted(alpha_overlay, alpha, cv_mat, 1 - alpha, 0, cv_mat); + + if(FALSE == G_NODE_IS_ROOT(pred->predictions)){ draw_line (cv_mat, cv::Point (box.x, box.y), cv::Point (box.x + box.width, box.y), @@ -193,7 +201,7 @@ gst_get_meta (GstInferencePrediction *pred, cv::Mat cv_mat, gdouble font_scale, cv::Point (box.x + box.width, box.y + box.height), colors[50 % N_C], thickness,"dotte",20); } - + } static From 00d87e461fd044194a6e1e4e26a675e3a88e7a8e Mon Sep 17 00:00:00 2001 From: tvlenin Date: Mon, 13 Jan 2020 16:45:15 -0600 Subject: [PATCH 113/188] Add style property to select the bounding box style --- ext/opencv/gstclassificationoverlay.cc | 4 +- ext/opencv/gstdetectionoverlay.cc | 4 +- ext/opencv/gstembeddingoverlay.cc | 4 +- ext/opencv/gstinferenceoverlay.cc | 24 ++++++------ gst-libs/gst/opencv/gstinferencebaseoverlay.c | 39 ++++++++++++++++++- gst-libs/gst/opencv/gstinferencebaseoverlay.h | 20 +++++++++- 6 files changed, 74 insertions(+), 21 deletions(-) diff --git a/ext/opencv/gstclassificationoverlay.cc b/ext/opencv/gstclassificationoverlay.cc index 585aa9fd..3d83e950 100644 --- a/ext/opencv/gstclassificationoverlay.cc +++ b/ext/opencv/gstclassificationoverlay.cc @@ -42,7 +42,7 @@ GST_DEBUG_CATEGORY_STATIC (gst_classification_overlay_debug_category); static GstFlowReturn gst_classification_overlay_process_meta (GstInferenceBaseOverlay * inference_overlay, GstVideoFrame * frame, GstMeta * meta, - gdouble font_scale, gint thickness, gchar ** labels_list, gint num_labels); + gdouble font_scale, gint thickness, gchar ** labels_list, gint num_labels, gint style); enum { @@ -96,7 +96,7 @@ gst_classification_overlay_init (GstClassificationOverlay * static GstFlowReturn gst_classification_overlay_process_meta (GstInferenceBaseOverlay * inference_overlay, GstVideoFrame * frame, GstMeta * meta, - gdouble font_scale, gint thickness, gchar ** labels_list, gint num_labels) + gdouble font_scale, gint thickness, gchar ** labels_list, gint num_labels, gint style) { GstClassificationMeta *class_meta; gint index, i, width, height, channels; diff --git a/ext/opencv/gstdetectionoverlay.cc b/ext/opencv/gstdetectionoverlay.cc index ba6dc3cf..f31eec04 100644 --- a/ext/opencv/gstdetectionoverlay.cc +++ b/ext/opencv/gstdetectionoverlay.cc @@ -53,7 +53,7 @@ GST_DEBUG_CATEGORY_STATIC (gst_detection_overlay_debug_category); static GstFlowReturn gst_detection_overlay_process_meta (GstInferenceBaseOverlay * inference_overlay, GstVideoFrame * frame, GstMeta * meta, gdouble font_scale, gint thickness, - gchar ** labels_list, gint num_labels); + gchar ** labels_list, gint num_labels, gint style); enum { @@ -105,7 +105,7 @@ gst_detection_overlay_init (GstDetectionOverlay * detection_overlay) static GstFlowReturn gst_detection_overlay_process_meta (GstInferenceBaseOverlay * inference_overlay, GstVideoFrame * frame, GstMeta * meta, gdouble font_scale, gint thickness, - gchar ** labels_list, gint num_labels) + gchar ** labels_list, gint num_labels, gint style) { GstDetectionMeta *detect_meta; gint i, width, height, channels; diff --git a/ext/opencv/gstembeddingoverlay.cc b/ext/opencv/gstembeddingoverlay.cc index ba039abb..5b320c8b 100644 --- a/ext/opencv/gstembeddingoverlay.cc +++ b/ext/opencv/gstembeddingoverlay.cc @@ -54,7 +54,7 @@ static void gst_embedding_overlay_finalize (GObject * object); static GstFlowReturn gst_embedding_overlay_process_meta (GstInferenceBaseOverlay * inference_overlay, GstVideoFrame * frame, GstMeta * meta, gdouble font_scale, gint thickness, - gchar ** labels_list, gint num_labels); + gchar ** labels_list, gint num_labels, gint style); static gboolean gst_embedding_overlay_set_embeddings (GstEmbeddingOverlay * embedding_overlay, const GValue * value); @@ -211,7 +211,7 @@ gst_embedding_overlay_finalize (GObject * object) static GstFlowReturn gst_embedding_overlay_process_meta (GstInferenceBaseOverlay * inference_overlay, GstVideoFrame * frame, GstMeta * meta, gdouble font_scale, gint thickness, - gchar ** labels_list, gint num_labels) + gchar ** labels_list, gint num_labels, gint style) { GstEmbeddingOverlay *embedding_overlay = GST_EMBEDDING_OVERLAY (inference_overlay); GstClassificationMeta *class_meta; diff --git a/ext/opencv/gstinferenceoverlay.cc b/ext/opencv/gstinferenceoverlay.cc index b25d61f0..5e9fb9b6 100644 --- a/ext/opencv/gstinferenceoverlay.cc +++ b/ext/opencv/gstinferenceoverlay.cc @@ -56,8 +56,8 @@ static GstFlowReturn gst_inference_overlay_process_meta (GstInferenceBaseOverlay *inference_overlay, GstVideoFrame *frame, GstMeta *meta, gdouble font_scale, gint thickness, - gchar **labels_list, gint num_labels); -void draw_line(cv::Mat& img, cv::Point pt1, cv::Point pt2, cv::Scalar color, int thickness, std::string style, int gap); + gchar **labels_list, gint num_labels, gint style); +void draw_line(cv::Mat& img, cv::Point pt1, cv::Point pt2, cv::Scalar color, int thickness, int style, int gap); enum { PROP_0 @@ -103,7 +103,7 @@ gst_inference_overlay_class_init (GstInferenceOverlayClass *klass) { static void gst_inference_overlay_init (GstInferenceOverlay *inference_overlay) { } -void draw_line(cv::Mat& img, cv::Point pt1, cv::Point pt2, cv::Scalar color, int thickness, std::string style, int gap){ +void draw_line(cv::Mat& img, cv::Point pt1, cv::Point pt2, cv::Scalar color, int thickness, int style, int gap){ float dx = pt1.x - pt2.x; float dy = pt1.y - pt2.y; @@ -121,7 +121,7 @@ void draw_line(cv::Mat& img, cv::Point pt1, cv::Point pt2, cv::Scalar color, int int pts_size = pts.size(); - if (style == "dotted") + if (style == 1) { for (int i = 0; i < pts_size; i++) { @@ -151,7 +151,7 @@ static void gst_get_meta (GstInferencePrediction *pred, cv::Mat cv_mat, gdouble font_scale, gint thickness, - gchar **labels_list, gint num_labels) { + gchar **labels_list, gint num_labels, gint style) { guint i; cv::Size size; cv::String label; @@ -165,7 +165,7 @@ gst_get_meta (GstInferencePrediction *pred, cv::Mat cv_mat, gdouble font_scale, for (i = 0; i < g_node_n_children(pred->predictions) ; ++i) { GstInferencePrediction *predict = (GstInferencePrediction*)g_node_nth_child (pred->predictions,i)->data; gst_get_meta (predict, cv_mat, font_scale, thickness, - labels_list, num_labels); + labels_list, num_labels, style); } box = pred->bbox; @@ -190,16 +190,16 @@ gst_get_meta (GstInferencePrediction *pred, cv::Mat cv_mat, gdouble font_scale, if(FALSE == G_NODE_IS_ROOT(pred->predictions)){ draw_line (cv_mat, cv::Point (box.x, box.y), cv::Point (box.x + box.width, box.y), - colors[50 % N_C], thickness,"dotte",20); + colors[50 % N_C], thickness,style,20); draw_line (cv_mat, cv::Point (box.x, box.y), cv::Point (box.x , box.y + box.height), - colors[50 % N_C], thickness,"dotte",20); + colors[50 % N_C], thickness,style,20); draw_line (cv_mat, cv::Point (box.x+ box.width, box.y), cv::Point (box.x + box.width, box.y + box.height), - colors[50 % N_C], thickness,"dotte",20); + colors[50 % N_C], thickness,style,20); draw_line (cv_mat, cv::Point (box.x, box.y+ box.height), cv::Point (box.x + box.width, box.y + box.height), - colors[50 % N_C], thickness,"dotte",20); + colors[50 % N_C], thickness,style,20); } } @@ -208,7 +208,7 @@ static GstFlowReturn gst_inference_overlay_process_meta (GstInferenceBaseOverlay *inference_overlay, GstVideoFrame *frame, GstMeta *meta, gdouble font_scale, gint thickness, - gchar **labels_list, gint num_labels) { + gchar **labels_list, gint num_labels, gint style) { GstInferenceMeta * detect_meta; gint width, height, channels; @@ -231,7 +231,7 @@ gst_inference_overlay_process_meta (GstInferenceBaseOverlay *inference_overlay, cv_mat = cv::Mat (height, width, CV_MAKETYPE (CV_8U, channels), (char *) frame->data[0]); gst_get_meta (detect_meta->prediction, cv_mat, font_scale, thickness, - labels_list, num_labels); + labels_list, num_labels, style); return GST_FLOW_OK; } diff --git a/gst-libs/gst/opencv/gstinferencebaseoverlay.c b/gst-libs/gst/opencv/gstinferencebaseoverlay.c index a16a9b2d..4ade6154 100644 --- a/gst-libs/gst/opencv/gstinferencebaseoverlay.c +++ b/gst-libs/gst/opencv/gstinferencebaseoverlay.c @@ -41,14 +41,35 @@ GST_DEBUG_CATEGORY_STATIC (gst_inference_base_overlay_debug_category); #define DEFAULT_LABELS NULL #define DEFAULT_NUM_LABELS 0 +#define MIN_STYLE CLASSIC +#define DEFAULT_STYLE CLASSIC +#define MAX_STYLE DASHED + enum { PROP_0, PROP_FONT_SCALE, PROP_THICKNESS, - PROP_LABELS + PROP_LABELS, + PROP_STYLE }; +GType +line_style_bounding_box_get_type (void) +{ + static GType type = G_TYPE_INVALID; + if (G_UNLIKELY (type == G_TYPE_INVALID)) { + static const GEnumValue values[] = { + {CLASSIC, "CLASSIC", "classic",}, + {DOTTED, "DOTTED", "dotted",}, + {DASHED, "DASHED", "dashed",}, + {4, NULL, NULL,}, + }; + type = g_enum_register_static ("LineStyleBoundingBox", values); + } + return type; +} + typedef struct _GstInferenceBaseOverlayPrivate GstInferenceBaseOverlayPrivate; struct _GstInferenceBaseOverlayPrivate { @@ -57,6 +78,7 @@ struct _GstInferenceBaseOverlayPrivate gchar *labels; gchar **labels_list; gint num_labels; + gint style; }; /* prototypes */ static void gst_inference_base_overlay_set_property (GObject * object, @@ -116,6 +138,10 @@ gst_inference_base_overlay_class_init (GstInferenceBaseOverlayClass * klass) g_param_spec_string ("labels", "labels", "Semicolon separated string containing inference labels", DEFAULT_LABELS, G_PARAM_READWRITE)); + g_object_class_install_property (gobject_class, PROP_STYLE, + g_param_spec_enum ("style", "style", + "Line style to draw the bounding box", LINE_STYLE_BOUNDING_BOX, + DEFAULT_STYLE, G_PARAM_READWRITE)); base_transform_class->start = GST_DEBUG_FUNCPTR (gst_inference_base_overlay_start); @@ -136,6 +162,7 @@ gst_inference_base_overlay_init (GstInferenceBaseOverlay * inference_overlay) priv->labels = DEFAULT_LABELS; priv->labels_list = DEFAULT_LABELS; priv->num_labels = DEFAULT_NUM_LABELS; + priv->style = DEFAULT_STYLE; } void @@ -173,6 +200,11 @@ gst_inference_base_overlay_set_property (GObject * object, guint property_id, GST_DEBUG_OBJECT (inference_overlay, "Changed inference labels %s", priv->labels); break; + case PROP_STYLE: + priv->style = g_value_get_enum (value); + GST_DEBUG_OBJECT (inference_overlay, "Changed box style to %d", + priv->style); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); break; @@ -200,6 +232,9 @@ gst_inference_base_overlay_get_property (GObject * object, guint property_id, case PROP_LABELS: g_value_set_string (value, priv->labels); break; + case PROP_STYLE: + g_value_set_enum (value, priv->style); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); break; @@ -286,7 +321,7 @@ gst_inference_base_overlay_transform_frame_ip (GstVideoFilter * trans, ret = io_class->process_meta (inference_overlay, frame, meta, priv->font_scale, priv->thickness, priv->labels_list, - priv->num_labels); + priv->num_labels, priv->style); } } diff --git a/gst-libs/gst/opencv/gstinferencebaseoverlay.h b/gst-libs/gst/opencv/gstinferencebaseoverlay.h index 491bb21c..668afdbe 100644 --- a/gst-libs/gst/opencv/gstinferencebaseoverlay.h +++ b/gst-libs/gst/opencv/gstinferencebaseoverlay.h @@ -37,11 +37,29 @@ struct _GstInferenceBaseOverlayClass GstFlowReturn (* process_meta) (GstInferenceBaseOverlay * inference_base_overlay, GstVideoFrame * frame, GstMeta* meta, gdouble font_scale, gint thickness, - gchar **labels_list, gint num_labels); + gchar **labels_list, gint num_labels, gint style); GType meta_type; }; +#define LINE_STYLE_BOUNDING_BOX (line_style_bounding_box_get_type ()) + +/** + * LineStyleBoundingBox: + * @CLASSIC : + * @DOTTED : + * @DASHED : + * + **/ +typedef enum +{ + CLASSIC, + DOTTED, + DASHED, +} LineStyleBoundingBox; + +GType line_style_bounding_box_get_type (void) G_GNUC_CONST; + G_END_DECLS #endif //__GST_INFERENCE_OVERLAY_H__ From ec1c16b6539a3afeaf41421cf1f9b05b5869f22d Mon Sep 17 00:00:00 2001 From: tvlenin Date: Tue, 14 Jan 2020 07:48:58 -0600 Subject: [PATCH 114/188] Add option to draw the classic bounding box without style --- ext/opencv/gstinferenceoverlay.cc | 97 +++++++++++++++++-------------- 1 file changed, 52 insertions(+), 45 deletions(-) diff --git a/ext/opencv/gstinferenceoverlay.cc b/ext/opencv/gstinferenceoverlay.cc index 5e9fb9b6..aaabd26f 100644 --- a/ext/opencv/gstinferenceoverlay.cc +++ b/ext/opencv/gstinferenceoverlay.cc @@ -57,7 +57,8 @@ GstFlowReturn gst_inference_overlay_process_meta (GstInferenceBaseOverlay *inference_overlay, GstVideoFrame *frame, GstMeta *meta, gdouble font_scale, gint thickness, gchar **labels_list, gint num_labels, gint style); -void draw_line(cv::Mat& img, cv::Point pt1, cv::Point pt2, cv::Scalar color, int thickness, int style, int gap); +void draw_line(cv::Mat &img, cv::Point pt1, cv::Point pt2, cv::Scalar color, + int thickness, int style, int gap); enum { PROP_0 @@ -93,7 +94,8 @@ gst_inference_overlay_class_init (GstInferenceOverlayClass *klass) { " Michael Gruner \n\t\t\t" " Carlos Aguero \n\t\t\t" " Miguel Taylor \n\t\t\t" - " Greivin Fallas "); + " Greivin Fallas \n\t\t\t" + " Lenin Torres "); io_class->process_meta = GST_DEBUG_FUNCPTR (gst_inference_overlay_process_meta); @@ -103,15 +105,16 @@ gst_inference_overlay_class_init (GstInferenceOverlayClass *klass) { static void gst_inference_overlay_init (GstInferenceOverlay *inference_overlay) { } -void draw_line(cv::Mat& img, cv::Point pt1, cv::Point pt2, cv::Scalar color, int thickness, int style, int gap){ + +void draw_line(cv::Mat &img, cv::Point pt1, cv::Point pt2, cv::Scalar color, + int thickness, int style, int gap) { float dx = pt1.x - pt2.x; float dy = pt1.y - pt2.y; - float dist = sqrt(dx*dx + dy*dy); + float dist = sqrt(dx * dx + dy * dy); std::vector pts; - for (int i = 0; i < dist; i += gap) - { + for (int i = 0; i < dist; i += gap) { float r = (float)i / dist; int x = int((pt1.x * (1.0 - r) + pt2.x * r) + .5); int y = int((pt1.y * (1.0 - r) + pt2.y * r) + .5); @@ -121,34 +124,26 @@ void draw_line(cv::Mat& img, cv::Point pt1, cv::Point pt2, cv::Scalar color, int int pts_size = pts.size(); - if (style == 1) - { - for (int i = 0; i < pts_size; i++) - { + if (1 == style) { + for (int i = 0; i < pts_size; i++) { cv::circle(img, pts[i], thickness, color, -1); } - } - else - { + } else { cv::Point s = pts[0]; cv::Point e = pts[0]; - //int count = 0; - - for (int i = 0; i < pts_size; i++) - { + for (int i = 0; i < pts_size; i++) { s = e; e = pts[i]; - if (i % 2 == 1) - { + if (1 == i % 2) { cv::line(img, s, e, color, thickness); } } } } -static -void + +static void gst_get_meta (GstInferencePrediction *pred, cv::Mat cv_mat, gdouble font_scale, gint thickness, gchar **labels_list, gint num_labels, gint style) { @@ -161,45 +156,57 @@ gst_get_meta (GstInferencePrediction *pred, cv::Mat cv_mat, gdouble font_scale, int classes = 0; double alpha = 0.5; cv::Mat alpha_overlay; - + for (i = 0; i < g_node_n_children(pred->predictions) ; ++i) { - GstInferencePrediction *predict = (GstInferencePrediction*)g_node_nth_child (pred->predictions,i)->data; + GstInferencePrediction *predict = (GstInferencePrediction *)g_node_nth_child ( + pred->predictions, i)->data; gst_get_meta (predict, cv_mat, font_scale, thickness, labels_list, num_labels, style); } box = pred->bbox; for (iter = pred->classifications; iter != NULL; iter = g_list_next(iter)) { - GstInferenceClassification *classification = (GstInferenceClassification *)iter->data; + GstInferenceClassification *classification = (GstInferenceClassification *) + iter->data; + classes++; if (num_labels > classification->class_id) { - label = cv::format ("%s Prob: %f", labels_list[classification->class_id], classification->class_prob); - }else{ - label = cv::format ("Label #%d Prob: %f", classification->class_id, classification->class_prob); + label = cv::format ("%s Prob: %f", labels_list[classification->class_id], + classification->class_prob); + } else { + label = cv::format ("Label #%d Prob: %f", classification->class_id, + classification->class_prob); } - cv::putText (cv_mat, label, cv::Point (box.x + box.width , box.y + classes * 30), - cv::FONT_HERSHEY_PLAIN, font_scale,cv::Scalar::all(0), thickness); + cv::putText (cv_mat, label, cv::Point (box.x + box.width, box.y + classes * 30), + cv::FONT_HERSHEY_PLAIN, font_scale, cv::Scalar::all(0), thickness); } cv_mat.copyTo(alpha_overlay); - cv::Size text = cv::getTextSize (label, cv::FONT_HERSHEY_PLAIN, font_scale, thickness, 0); - cv::rectangle(alpha_overlay, cv::Rect(box.x + box.width, box.y, text.width, 50*classes), cv::Scalar(255, 255, 255), -1); + cv::Size text = cv::getTextSize (label, cv::FONT_HERSHEY_PLAIN, font_scale, + thickness, 0); + cv::rectangle(alpha_overlay, cv::Rect(box.x + box.width, box.y, text.width, + 50 * classes), cv::Scalar(255, 255, 255), -1); cv::addWeighted(alpha_overlay, alpha, cv_mat, 1 - alpha, 0, cv_mat); - - if(FALSE == G_NODE_IS_ROOT(pred->predictions)){ - draw_line (cv_mat, cv::Point (box.x, box.y), - cv::Point (box.x + box.width, box.y), - colors[50 % N_C], thickness,style,20); - draw_line (cv_mat, cv::Point (box.x, box.y), - cv::Point (box.x , box.y + box.height), - colors[50 % N_C], thickness,style,20); - draw_line (cv_mat, cv::Point (box.x+ box.width, box.y), - cv::Point (box.x + box.width, box.y + box.height), - colors[50 % N_C], thickness,style,20); - draw_line (cv_mat, cv::Point (box.x, box.y+ box.height), - cv::Point (box.x + box.width, box.y + box.height), - colors[50 % N_C], thickness,style,20); + if (FALSE == G_NODE_IS_ROOT(pred->predictions)) { + if (0 == style) { + cv::rectangle (cv_mat, cv::Point (box.x, box.y), + cv::Point (box.x + box.width, box.y + box.height), + colors[50 % N_C], thickness); + }else { + draw_line (cv_mat, cv::Point (box.x, box.y), + cv::Point (box.x + box.width, box.y), + colors[50 % N_C], thickness, style, 20); + draw_line (cv_mat, cv::Point (box.x, box.y), + cv::Point (box.x, box.y + box.height), + colors[50 % N_C], thickness, style, 20); + draw_line (cv_mat, cv::Point (box.x + box.width, box.y), + cv::Point (box.x + box.width, box.y + box.height), + colors[50 % N_C], thickness, style, 20); + draw_line (cv_mat, cv::Point (box.x, box.y + box.height), + cv::Point (box.x + box.width, box.y + box.height), + colors[50 % N_C], thickness, style, 20); + } } } From 8c7b28a937726d4d1877a2fbc752e28e3cfc9181 Mon Sep 17 00:00:00 2001 From: tvlenin Date: Tue, 14 Jan 2020 14:00:55 -0600 Subject: [PATCH 115/188] Add some pointer guards --- ext/opencv/gstinferenceoverlay.cc | 39 ++++++++++++++++--------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/ext/opencv/gstinferenceoverlay.cc b/ext/opencv/gstinferenceoverlay.cc index aaabd26f..c1039ec9 100644 --- a/ext/opencv/gstinferenceoverlay.cc +++ b/ext/opencv/gstinferenceoverlay.cc @@ -57,13 +57,9 @@ GstFlowReturn gst_inference_overlay_process_meta (GstInferenceBaseOverlay *inference_overlay, GstVideoFrame *frame, GstMeta *meta, gdouble font_scale, gint thickness, gchar **labels_list, gint num_labels, gint style); -void draw_line(cv::Mat &img, cv::Point pt1, cv::Point pt2, cv::Scalar color, +void draw_line(cv::Mat *img, cv::Point pt1, cv::Point pt2, cv::Scalar color, int thickness, int style, int gap); -enum { - PROP_0 -}; - struct _GstInferenceOverlay { GstInferenceBaseOverlay parent; @@ -89,13 +85,7 @@ gst_inference_overlay_class_init (GstInferenceOverlayClass *klass) { gst_element_class_set_static_metadata (GST_ELEMENT_CLASS (klass), "inferenceoverlay", "Filter", "Overlays Inferece metadata on input buffer", - "Carlos Rodriguez \n\t\t\t" - " Jose Jimenez \n\t\t\t" - " Michael Gruner \n\t\t\t" - " Carlos Aguero \n\t\t\t" - " Miguel Taylor \n\t\t\t" - " Greivin Fallas \n\t\t\t" - " Lenin Torres "); + "Lenin Torres "); io_class->process_meta = GST_DEBUG_FUNCPTR (gst_inference_overlay_process_meta); @@ -106,8 +96,11 @@ static void gst_inference_overlay_init (GstInferenceOverlay *inference_overlay) { } -void draw_line(cv::Mat &img, cv::Point pt1, cv::Point pt2, cv::Scalar color, +void draw_line(cv::Mat *img, cv::Point pt1, cv::Point pt2, cv::Scalar color, int thickness, int style, int gap) { + + g_return_if_fail (img != NULL); + float dx = pt1.x - pt2.x; float dy = pt1.y - pt2.y; @@ -126,7 +119,7 @@ void draw_line(cv::Mat &img, cv::Point pt1, cv::Point pt2, cv::Scalar color, if (1 == style) { for (int i = 0; i < pts_size; i++) { - cv::circle(img, pts[i], thickness, color, -1); + cv::circle(*img, pts[i], thickness, color, -1); } } else { cv::Point s = pts[0]; @@ -137,7 +130,7 @@ void draw_line(cv::Mat &img, cv::Point pt1, cv::Point pt2, cv::Scalar color, e = pts[i]; if (1 == i % 2) { - cv::line(img, s, e, color, thickness); + cv::line(*img, s, e, color, thickness); } } } @@ -157,6 +150,9 @@ gst_get_meta (GstInferencePrediction *pred, cv::Mat cv_mat, gdouble font_scale, double alpha = 0.5; cv::Mat alpha_overlay; + g_return_if_fail (pred != NULL); + g_return_if_fail (labels_list != NULL); + for (i = 0; i < g_node_n_children(pred->predictions) ; ++i) { GstInferencePrediction *predict = (GstInferencePrediction *)g_node_nth_child ( pred->predictions, i)->data; @@ -194,16 +190,16 @@ gst_get_meta (GstInferencePrediction *pred, cv::Mat cv_mat, gdouble font_scale, cv::Point (box.x + box.width, box.y + box.height), colors[50 % N_C], thickness); }else { - draw_line (cv_mat, cv::Point (box.x, box.y), + draw_line (&cv_mat, cv::Point (box.x, box.y), cv::Point (box.x + box.width, box.y), colors[50 % N_C], thickness, style, 20); - draw_line (cv_mat, cv::Point (box.x, box.y), + draw_line (&cv_mat, cv::Point (box.x, box.y), cv::Point (box.x, box.y + box.height), colors[50 % N_C], thickness, style, 20); - draw_line (cv_mat, cv::Point (box.x + box.width, box.y), + draw_line (&cv_mat, cv::Point (box.x + box.width, box.y), cv::Point (box.x + box.width, box.y + box.height), colors[50 % N_C], thickness, style, 20); - draw_line (cv_mat, cv::Point (box.x, box.y + box.height), + draw_line (&cv_mat, cv::Point (box.x, box.y + box.height), cv::Point (box.x + box.width, box.y + box.height), colors[50 % N_C], thickness, style, 20); } @@ -221,6 +217,11 @@ gst_inference_overlay_process_meta (GstInferenceBaseOverlay *inference_overlay, gint width, height, channels; cv::Mat cv_mat; + g_return_val_if_fail (inference_overlay != NULL ,GST_FLOW_ERROR); + g_return_val_if_fail (frame != NULL ,GST_FLOW_ERROR); + g_return_val_if_fail (meta != NULL ,GST_FLOW_ERROR); + g_return_val_if_fail (labels_list != NULL ,GST_FLOW_ERROR); + switch (GST_VIDEO_FRAME_FORMAT (frame)) { case GST_VIDEO_FORMAT_RGB: case GST_VIDEO_FORMAT_BGR: From 6168f55eec2539eabe8eefd83fc8eddbc0dbf7f3 Mon Sep 17 00:00:00 2001 From: tvlenin Date: Tue, 14 Jan 2020 17:04:32 -0600 Subject: [PATCH 116/188] Add support to read labels from meta --- ext/opencv/gstinferenceoverlay.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ext/opencv/gstinferenceoverlay.cc b/ext/opencv/gstinferenceoverlay.cc index c1039ec9..59a1d0e4 100644 --- a/ext/opencv/gstinferenceoverlay.cc +++ b/ext/opencv/gstinferenceoverlay.cc @@ -166,8 +166,8 @@ gst_get_meta (GstInferencePrediction *pred, cv::Mat cv_mat, gdouble font_scale, iter->data; classes++; - if (num_labels > classification->class_id) { - label = cv::format ("%s Prob: %f", labels_list[classification->class_id], + if (classification->num_classes > classification->class_id) { + label = cv::format ("%s Prob: %f", classification->labels[classification->class_id], classification->class_prob); } else { label = cv::format ("Label #%d Prob: %f", classification->class_id, From be43495a64fe329201464d10968dc65d32daf7a3 Mon Sep 17 00:00:00 2001 From: tvlenin Date: Tue, 14 Jan 2020 17:15:42 -0600 Subject: [PATCH 117/188] Change license date --- ext/opencv/gstinferenceoverlay.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/opencv/gstinferenceoverlay.cc b/ext/opencv/gstinferenceoverlay.cc index 59a1d0e4..9b57f689 100644 --- a/ext/opencv/gstinferenceoverlay.cc +++ b/ext/opencv/gstinferenceoverlay.cc @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2018 RidgeRun + * Copyright (C) 2019 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public From 6468179b89054b1f56ecd5bdd9900f4023f2e71a Mon Sep 17 00:00:00 2001 From: tvlenin Date: Fri, 17 Jan 2020 10:23:21 -0600 Subject: [PATCH 118/188] Modify enum usage on overlay elements --- ext/opencv/gstclassificationoverlay.cc | 4 +-- ext/opencv/gstdetectionoverlay.cc | 4 +-- ext/opencv/gstembeddingoverlay.cc | 4 +-- ext/opencv/gstinferenceoverlay.cc | 7 ++-- gst-libs/gst/opencv/gstinferencebaseoverlay.c | 2 +- gst-libs/gst/opencv/gstinferencebaseoverlay.h | 36 +++++++++---------- 6 files changed, 27 insertions(+), 30 deletions(-) diff --git a/ext/opencv/gstclassificationoverlay.cc b/ext/opencv/gstclassificationoverlay.cc index 3d83e950..3528f3fe 100644 --- a/ext/opencv/gstclassificationoverlay.cc +++ b/ext/opencv/gstclassificationoverlay.cc @@ -42,7 +42,7 @@ GST_DEBUG_CATEGORY_STATIC (gst_classification_overlay_debug_category); static GstFlowReturn gst_classification_overlay_process_meta (GstInferenceBaseOverlay * inference_overlay, GstVideoFrame * frame, GstMeta * meta, - gdouble font_scale, gint thickness, gchar ** labels_list, gint num_labels, gint style); + gdouble font_scale, gint thickness, gchar ** labels_list, gint num_labels, LineStyleBoundingBox style); enum { @@ -96,7 +96,7 @@ gst_classification_overlay_init (GstClassificationOverlay * static GstFlowReturn gst_classification_overlay_process_meta (GstInferenceBaseOverlay * inference_overlay, GstVideoFrame * frame, GstMeta * meta, - gdouble font_scale, gint thickness, gchar ** labels_list, gint num_labels, gint style) + gdouble font_scale, gint thickness, gchar ** labels_list, gint num_labels, LineStyleBoundingBox style) { GstClassificationMeta *class_meta; gint index, i, width, height, channels; diff --git a/ext/opencv/gstdetectionoverlay.cc b/ext/opencv/gstdetectionoverlay.cc index f31eec04..a0409e61 100644 --- a/ext/opencv/gstdetectionoverlay.cc +++ b/ext/opencv/gstdetectionoverlay.cc @@ -53,7 +53,7 @@ GST_DEBUG_CATEGORY_STATIC (gst_detection_overlay_debug_category); static GstFlowReturn gst_detection_overlay_process_meta (GstInferenceBaseOverlay * inference_overlay, GstVideoFrame * frame, GstMeta * meta, gdouble font_scale, gint thickness, - gchar ** labels_list, gint num_labels, gint style); + gchar ** labels_list, gint num_labels, LineStyleBoundingBox style); enum { @@ -105,7 +105,7 @@ gst_detection_overlay_init (GstDetectionOverlay * detection_overlay) static GstFlowReturn gst_detection_overlay_process_meta (GstInferenceBaseOverlay * inference_overlay, GstVideoFrame * frame, GstMeta * meta, gdouble font_scale, gint thickness, - gchar ** labels_list, gint num_labels, gint style) + gchar ** labels_list, gint num_labels, LineStyleBoundingBox style) { GstDetectionMeta *detect_meta; gint i, width, height, channels; diff --git a/ext/opencv/gstembeddingoverlay.cc b/ext/opencv/gstembeddingoverlay.cc index 5b320c8b..47314111 100644 --- a/ext/opencv/gstembeddingoverlay.cc +++ b/ext/opencv/gstembeddingoverlay.cc @@ -54,7 +54,7 @@ static void gst_embedding_overlay_finalize (GObject * object); static GstFlowReturn gst_embedding_overlay_process_meta (GstInferenceBaseOverlay * inference_overlay, GstVideoFrame * frame, GstMeta * meta, gdouble font_scale, gint thickness, - gchar ** labels_list, gint num_labels, gint style); + gchar ** labels_list, gint num_labels, LineStyleBoundingBox style); static gboolean gst_embedding_overlay_set_embeddings (GstEmbeddingOverlay * embedding_overlay, const GValue * value); @@ -211,7 +211,7 @@ gst_embedding_overlay_finalize (GObject * object) static GstFlowReturn gst_embedding_overlay_process_meta (GstInferenceBaseOverlay * inference_overlay, GstVideoFrame * frame, GstMeta * meta, gdouble font_scale, gint thickness, - gchar ** labels_list, gint num_labels, gint style) + gchar ** labels_list, gint num_labels, LineStyleBoundingBox style) { GstEmbeddingOverlay *embedding_overlay = GST_EMBEDDING_OVERLAY (inference_overlay); GstClassificationMeta *class_meta; diff --git a/ext/opencv/gstinferenceoverlay.cc b/ext/opencv/gstinferenceoverlay.cc index 9b57f689..8b5c6e1c 100644 --- a/ext/opencv/gstinferenceoverlay.cc +++ b/ext/opencv/gstinferenceoverlay.cc @@ -52,11 +52,10 @@ GST_DEBUG_CATEGORY_STATIC (gst_inference_overlay_debug_category); #define GST_CAT_DEFAULT gst_inference_overlay_debug_category /* prototypes */ -static -GstFlowReturn +static GstFlowReturn gst_inference_overlay_process_meta (GstInferenceBaseOverlay *inference_overlay, GstVideoFrame *frame, GstMeta *meta, gdouble font_scale, gint thickness, - gchar **labels_list, gint num_labels, gint style); + gchar **labels_list, gint num_labels, LineStyleBoundingBox style); void draw_line(cv::Mat *img, cv::Point pt1, cv::Point pt2, cv::Scalar color, int thickness, int style, int gap); @@ -211,7 +210,7 @@ static GstFlowReturn gst_inference_overlay_process_meta (GstInferenceBaseOverlay *inference_overlay, GstVideoFrame *frame, GstMeta *meta, gdouble font_scale, gint thickness, - gchar **labels_list, gint num_labels, gint style) { + gchar **labels_list, gint num_labels, LineStyleBoundingBox style) { GstInferenceMeta * detect_meta; gint width, height, channels; diff --git a/gst-libs/gst/opencv/gstinferencebaseoverlay.c b/gst-libs/gst/opencv/gstinferencebaseoverlay.c index 4ade6154..f790e40f 100644 --- a/gst-libs/gst/opencv/gstinferencebaseoverlay.c +++ b/gst-libs/gst/opencv/gstinferencebaseoverlay.c @@ -78,7 +78,7 @@ struct _GstInferenceBaseOverlayPrivate gchar *labels; gchar **labels_list; gint num_labels; - gint style; + LineStyleBoundingBox style; }; /* prototypes */ static void gst_inference_base_overlay_set_property (GObject * object, diff --git a/gst-libs/gst/opencv/gstinferencebaseoverlay.h b/gst-libs/gst/opencv/gstinferencebaseoverlay.h index 668afdbe..a1713279 100644 --- a/gst-libs/gst/opencv/gstinferencebaseoverlay.h +++ b/gst-libs/gst/opencv/gstinferencebaseoverlay.h @@ -27,28 +27,12 @@ G_BEGIN_DECLS -#define GST_TYPE_INFERENCE_BASE_OVERLAY gst_inference_base_overlay_get_type () -G_DECLARE_DERIVABLE_TYPE (GstInferenceBaseOverlay, gst_inference_base_overlay, GST, - INFERENCE_BASE_OVERLAY, GstVideoFilter); - -struct _GstInferenceBaseOverlayClass -{ - GstVideoFilterClass parent_class; - - GstFlowReturn (* process_meta) (GstInferenceBaseOverlay * inference_base_overlay, - GstVideoFrame * frame, GstMeta* meta, gdouble font_scale, gint thickness, - gchar **labels_list, gint num_labels, gint style); - - GType meta_type; -}; - #define LINE_STYLE_BOUNDING_BOX (line_style_bounding_box_get_type ()) - /** * LineStyleBoundingBox: - * @CLASSIC : - * @DOTTED : - * @DASHED : + * @CLASSIC : Draw lines without any style + * @DOTTED : Draw lines with dots + * @DASHED : Draw lines with dashed lines * **/ typedef enum @@ -59,6 +43,20 @@ typedef enum } LineStyleBoundingBox; GType line_style_bounding_box_get_type (void) G_GNUC_CONST; +#define GST_TYPE_INFERENCE_BASE_OVERLAY gst_inference_base_overlay_get_type () +G_DECLARE_DERIVABLE_TYPE (GstInferenceBaseOverlay, gst_inference_base_overlay, GST, + INFERENCE_BASE_OVERLAY, GstVideoFilter); + +struct _GstInferenceBaseOverlayClass +{ + GstVideoFilterClass parent_class; + + GstFlowReturn (* process_meta) (GstInferenceBaseOverlay * inference_base_overlay, + GstVideoFrame * frame, GstMeta* meta, gdouble font_scale, gint thickness, + gchar **labels_list, gint num_labels, LineStyleBoundingBox style); + + GType meta_type; +}; G_END_DECLS From 2f9378438949c80b5954fe78c598136bea5e14eb Mon Sep 17 00:00:00 2001 From: tvlenin Date: Fri, 17 Jan 2020 10:41:41 -0600 Subject: [PATCH 119/188] Remove unnecessary check on configure --- configure.ac | 1 - 1 file changed, 1 deletion(-) diff --git a/configure.ac b/configure.ac index dd253aa7..99cd9fde 100644 --- a/configure.ac +++ b/configure.ac @@ -20,7 +20,6 @@ AS_NANO(GST_GIT="no", GST_GIT="yes") dnl can autoconf find the source ? AC_CONFIG_SRCDIR([ext/r2inference/gstinference.c]) -AC_CONFIG_SRCDIR([ext/opencv/gstinferenceoverlayplugin.c]) dnl define the output header for config AC_CONFIG_HEADERS([config.h]) From 72569de2afb28bf1345d3f2abfa849ecea2b5369 Mon Sep 17 00:00:00 2001 From: tvlenin Date: Fri, 17 Jan 2020 16:53:27 -0600 Subject: [PATCH 120/188] Remove magic numbers --- ext/opencv/gstinferenceoverlay.cc | 32 +++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/ext/opencv/gstinferenceoverlay.cc b/ext/opencv/gstinferenceoverlay.cc index 8b5c6e1c..6f5442ed 100644 --- a/ext/opencv/gstinferenceoverlay.cc +++ b/ext/opencv/gstinferenceoverlay.cc @@ -48,6 +48,10 @@ colors[] = { #define N_C (sizeof (colors)/sizeof (cv::Scalar)) +#define CHOSEN_COLOR 14 +#define OVERLAY_HEIGHT 50 +#define OVERLAY_WIDTH 30 + GST_DEBUG_CATEGORY_STATIC (gst_inference_overlay_debug_category); #define GST_CAT_DEFAULT gst_inference_overlay_debug_category @@ -57,7 +61,7 @@ gst_inference_overlay_process_meta (GstInferenceBaseOverlay *inference_overlay, GstVideoFrame *frame, GstMeta *meta, gdouble font_scale, gint thickness, gchar **labels_list, gint num_labels, LineStyleBoundingBox style); void draw_line(cv::Mat *img, cv::Point pt1, cv::Point pt2, cv::Scalar color, - int thickness, int style, int gap); + int thickness, LineStyleBoundingBox style, int gap); struct _GstInferenceOverlay { GstInferenceBaseOverlay @@ -96,7 +100,7 @@ gst_inference_overlay_init (GstInferenceOverlay *inference_overlay) { } void draw_line(cv::Mat *img, cv::Point pt1, cv::Point pt2, cv::Scalar color, - int thickness, int style, int gap) { + int thickness, LineStyleBoundingBox style, int gap) { g_return_if_fail (img != NULL); @@ -116,7 +120,7 @@ void draw_line(cv::Mat *img, cv::Point pt1, cv::Point pt2, cv::Scalar color, int pts_size = pts.size(); - if (1 == style) { + if (DOTTED == style) { for (int i = 0; i < pts_size; i++) { cv::circle(*img, pts[i], thickness, color, -1); } @@ -138,15 +142,15 @@ void draw_line(cv::Mat *img, cv::Point pt1, cv::Point pt2, cv::Scalar color, static void gst_get_meta (GstInferencePrediction *pred, cv::Mat cv_mat, gdouble font_scale, gint thickness, - gchar **labels_list, gint num_labels, gint style) { + gchar **labels_list, gint num_labels, LineStyleBoundingBox style) { guint i; cv::Size size; cv::String label; GList *iter = NULL; cv::String prob; BoundingBox box; - int classes = 0; - double alpha = 0.5; + gint classes = 0; + gdouble alpha = 0.5; cv::Mat alpha_overlay; g_return_if_fail (pred != NULL); @@ -165,14 +169,14 @@ gst_get_meta (GstInferencePrediction *pred, cv::Mat cv_mat, gdouble font_scale, iter->data; classes++; - if (classification->num_classes > classification->class_id) { + if (classification->num_classes > classification->class_id && classification->labels != NULL) { label = cv::format ("%s Prob: %f", classification->labels[classification->class_id], classification->class_prob); } else { label = cv::format ("Label #%d Prob: %f", classification->class_id, classification->class_prob); } - cv::putText (cv_mat, label, cv::Point (box.x + box.width, box.y + classes * 30), + cv::putText (cv_mat, label, cv::Point (box.x + box.width, box.y + classes * OVERLAY_WIDTH), cv::FONT_HERSHEY_PLAIN, font_scale, cv::Scalar::all(0), thickness); } @@ -180,27 +184,27 @@ gst_get_meta (GstInferencePrediction *pred, cv::Mat cv_mat, gdouble font_scale, cv::Size text = cv::getTextSize (label, cv::FONT_HERSHEY_PLAIN, font_scale, thickness, 0); cv::rectangle(alpha_overlay, cv::Rect(box.x + box.width, box.y, text.width, - 50 * classes), cv::Scalar(255, 255, 255), -1); + OVERLAY_HEIGHT * classes), cv::Scalar(255, 255, 255), -1); cv::addWeighted(alpha_overlay, alpha, cv_mat, 1 - alpha, 0, cv_mat); if (FALSE == G_NODE_IS_ROOT(pred->predictions)) { if (0 == style) { cv::rectangle (cv_mat, cv::Point (box.x, box.y), cv::Point (box.x + box.width, box.y + box.height), - colors[50 % N_C], thickness); + colors[CHOSEN_COLOR % N_C], thickness); }else { draw_line (&cv_mat, cv::Point (box.x, box.y), cv::Point (box.x + box.width, box.y), - colors[50 % N_C], thickness, style, 20); + colors[CHOSEN_COLOR % N_C], thickness, style, 20); draw_line (&cv_mat, cv::Point (box.x, box.y), cv::Point (box.x, box.y + box.height), - colors[50 % N_C], thickness, style, 20); + colors[CHOSEN_COLOR % N_C], thickness, style, 20); draw_line (&cv_mat, cv::Point (box.x + box.width, box.y), cv::Point (box.x + box.width, box.y + box.height), - colors[50 % N_C], thickness, style, 20); + colors[CHOSEN_COLOR % N_C], thickness, style, 20); draw_line (&cv_mat, cv::Point (box.x, box.y + box.height), cv::Point (box.x + box.width, box.y + box.height), - colors[50 % N_C], thickness, style, 20); + colors[CHOSEN_COLOR % N_C], thickness, style, 20); } } From cc45bd9b8d36e4012c8add90092c0f7aded245a9 Mon Sep 17 00:00:00 2001 From: tvlenin Date: Fri, 17 Jan 2020 17:02:49 -0600 Subject: [PATCH 121/188] Optimize the iteration over the tree --- ext/opencv/gstinferenceoverlay.cc | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/ext/opencv/gstinferenceoverlay.cc b/ext/opencv/gstinferenceoverlay.cc index 6f5442ed..492bf5d4 100644 --- a/ext/opencv/gstinferenceoverlay.cc +++ b/ext/opencv/gstinferenceoverlay.cc @@ -143,10 +143,11 @@ static void gst_get_meta (GstInferencePrediction *pred, cv::Mat cv_mat, gdouble font_scale, gint thickness, gchar **labels_list, gint num_labels, LineStyleBoundingBox style) { - guint i; cv::Size size; cv::String label; GList *iter = NULL; + GSList *list = NULL; + GSList *tree_iter = NULL; cv::String prob; BoundingBox box; gint classes = 0; @@ -156,9 +157,11 @@ gst_get_meta (GstInferencePrediction *pred, cv::Mat cv_mat, gdouble font_scale, g_return_if_fail (pred != NULL); g_return_if_fail (labels_list != NULL); - for (i = 0; i < g_node_n_children(pred->predictions) ; ++i) { - GstInferencePrediction *predict = (GstInferencePrediction *)g_node_nth_child ( - pred->predictions, i)->data; + list = gst_inference_prediction_get_children(pred); + + for (tree_iter = list; tree_iter != NULL; tree_iter = g_slist_next(tree_iter)) { + GstInferencePrediction *predict = (GstInferencePrediction *)tree_iter->data; + gst_get_meta (predict, cv_mat, font_scale, thickness, labels_list, num_labels, style); } From eea2ceb224a941893cf6af7afcf939e9af0be829 Mon Sep 17 00:00:00 2001 From: tvlenin Date: Fri, 17 Jan 2020 17:09:27 -0600 Subject: [PATCH 122/188] Pass argument as reference to avoid a copy --- ext/opencv/gstinferenceoverlay.cc | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/ext/opencv/gstinferenceoverlay.cc b/ext/opencv/gstinferenceoverlay.cc index 492bf5d4..61ad1e26 100644 --- a/ext/opencv/gstinferenceoverlay.cc +++ b/ext/opencv/gstinferenceoverlay.cc @@ -140,7 +140,7 @@ void draw_line(cv::Mat *img, cv::Point pt1, cv::Point pt2, cv::Scalar color, } static void -gst_get_meta (GstInferencePrediction *pred, cv::Mat cv_mat, gdouble font_scale, +gst_get_meta (GstInferencePrediction *pred, cv::Mat *cv_mat, gdouble font_scale, gint thickness, gchar **labels_list, gint num_labels, LineStyleBoundingBox style) { cv::Size size; @@ -156,6 +156,7 @@ gst_get_meta (GstInferencePrediction *pred, cv::Mat cv_mat, gdouble font_scale, g_return_if_fail (pred != NULL); g_return_if_fail (labels_list != NULL); + g_return_if_fail (cv_mat != NULL); list = gst_inference_prediction_get_children(pred); @@ -179,33 +180,33 @@ gst_get_meta (GstInferencePrediction *pred, cv::Mat cv_mat, gdouble font_scale, label = cv::format ("Label #%d Prob: %f", classification->class_id, classification->class_prob); } - cv::putText (cv_mat, label, cv::Point (box.x + box.width, box.y + classes * OVERLAY_WIDTH), + cv::putText (*cv_mat, label, cv::Point (box.x + box.width, box.y + classes * OVERLAY_WIDTH), cv::FONT_HERSHEY_PLAIN, font_scale, cv::Scalar::all(0), thickness); } - cv_mat.copyTo(alpha_overlay); + cv_mat->copyTo(alpha_overlay); cv::Size text = cv::getTextSize (label, cv::FONT_HERSHEY_PLAIN, font_scale, thickness, 0); cv::rectangle(alpha_overlay, cv::Rect(box.x + box.width, box.y, text.width, OVERLAY_HEIGHT * classes), cv::Scalar(255, 255, 255), -1); - cv::addWeighted(alpha_overlay, alpha, cv_mat, 1 - alpha, 0, cv_mat); + cv::addWeighted(alpha_overlay, alpha, *cv_mat, 1 - alpha, 0, *cv_mat); if (FALSE == G_NODE_IS_ROOT(pred->predictions)) { if (0 == style) { - cv::rectangle (cv_mat, cv::Point (box.x, box.y), + cv::rectangle (*cv_mat, cv::Point (box.x, box.y), cv::Point (box.x + box.width, box.y + box.height), colors[CHOSEN_COLOR % N_C], thickness); }else { - draw_line (&cv_mat, cv::Point (box.x, box.y), + draw_line (cv_mat, cv::Point (box.x, box.y), cv::Point (box.x + box.width, box.y), colors[CHOSEN_COLOR % N_C], thickness, style, 20); - draw_line (&cv_mat, cv::Point (box.x, box.y), + draw_line (cv_mat, cv::Point (box.x, box.y), cv::Point (box.x, box.y + box.height), colors[CHOSEN_COLOR % N_C], thickness, style, 20); - draw_line (&cv_mat, cv::Point (box.x + box.width, box.y), + draw_line (cv_mat, cv::Point (box.x + box.width, box.y), cv::Point (box.x + box.width, box.y + box.height), colors[CHOSEN_COLOR % N_C], thickness, style, 20); - draw_line (&cv_mat, cv::Point (box.x, box.y + box.height), + draw_line (cv_mat, cv::Point (box.x, box.y + box.height), cv::Point (box.x + box.width, box.y + box.height), colors[CHOSEN_COLOR % N_C], thickness, style, 20); } @@ -244,7 +245,7 @@ gst_inference_overlay_process_meta (GstInferenceBaseOverlay *inference_overlay, cv_mat = cv::Mat (height, width, CV_MAKETYPE (CV_8U, channels), (char *) frame->data[0]); - gst_get_meta (detect_meta->prediction, cv_mat, font_scale, thickness, + gst_get_meta (detect_meta->prediction, &cv_mat, font_scale, thickness, labels_list, num_labels, style); return GST_FLOW_OK; From 38bc6e76698c8636824d7b7d530421a9f4dfb9f3 Mon Sep 17 00:00:00 2001 From: tvlenin Date: Mon, 20 Jan 2020 08:38:18 -0600 Subject: [PATCH 123/188] Update license to some files --- ext/opencv/gstinferenceoverlay.cc | 2 +- ext/opencv/gstinferenceoverlay.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ext/opencv/gstinferenceoverlay.cc b/ext/opencv/gstinferenceoverlay.cc index 61ad1e26..26ac2e44 100644 --- a/ext/opencv/gstinferenceoverlay.cc +++ b/ext/opencv/gstinferenceoverlay.cc @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2019 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public diff --git a/ext/opencv/gstinferenceoverlay.h b/ext/opencv/gstinferenceoverlay.h index 42a51725..d89f2009 100644 --- a/ext/opencv/gstinferenceoverlay.h +++ b/ext/opencv/gstinferenceoverlay.h @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2019 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public From fd2a380cc7c5b080ae59b794802210734d308a54 Mon Sep 17 00:00:00 2001 From: tvlenin Date: Mon, 20 Jan 2020 08:50:45 -0600 Subject: [PATCH 124/188] Pass image as reference --- ext/opencv/gstinferenceoverlay.cc | 48 +++++++++++++++---------------- 1 file changed, 23 insertions(+), 25 deletions(-) diff --git a/ext/opencv/gstinferenceoverlay.cc b/ext/opencv/gstinferenceoverlay.cc index 26ac2e44..71c29b27 100644 --- a/ext/opencv/gstinferenceoverlay.cc +++ b/ext/opencv/gstinferenceoverlay.cc @@ -18,6 +18,7 @@ * Boston, MA 02111-1307, USA. * */ + #ifdef HAVE_CONFIG_H #include "config.h" #endif @@ -60,8 +61,8 @@ static GstFlowReturn gst_inference_overlay_process_meta (GstInferenceBaseOverlay *inference_overlay, GstVideoFrame *frame, GstMeta *meta, gdouble font_scale, gint thickness, gchar **labels_list, gint num_labels, LineStyleBoundingBox style); -void draw_line(cv::Mat *img, cv::Point pt1, cv::Point pt2, cv::Scalar color, - int thickness, LineStyleBoundingBox style, int gap); +void draw_line(cv::Mat &img, cv::Point pt1, cv::Point pt2, cv::Scalar color, + gint thickness, LineStyleBoundingBox style, gint gap); struct _GstInferenceOverlay { GstInferenceBaseOverlay @@ -99,48 +100,46 @@ static void gst_inference_overlay_init (GstInferenceOverlay *inference_overlay) { } -void draw_line(cv::Mat *img, cv::Point pt1, cv::Point pt2, cv::Scalar color, - int thickness, LineStyleBoundingBox style, int gap) { - - g_return_if_fail (img != NULL); +void draw_line(cv::Mat &img, cv::Point pt1, cv::Point pt2, cv::Scalar color, + gint thickness, LineStyleBoundingBox style, gint gap) { - float dx = pt1.x - pt2.x; - float dy = pt1.y - pt2.y; + gfloat dx = pt1.x - pt2.x; + gfloat dy = pt1.y - pt2.y; - float dist = sqrt(dx * dx + dy * dy); + gfloat dist = sqrt(dx * dx + dy * dy); std::vector pts; - for (int i = 0; i < dist; i += gap) { - float r = (float)i / dist; - int x = int((pt1.x * (1.0 - r) + pt2.x * r) + .5); - int y = int((pt1.y * (1.0 - r) + pt2.y * r) + .5); + for (gint i = 0; i < dist; i += gap) { + gfloat r = (gfloat)i / dist; + gint x = gint((pt1.x * (1.0 - r) + pt2.x * r) + .5); + gint y = gint((pt1.y * (1.0 - r) + pt2.y * r) + .5); cv::Point p = cv::Point(x, y); pts.push_back(p); } - int pts_size = pts.size(); + gint pts_size = pts.size(); if (DOTTED == style) { - for (int i = 0; i < pts_size; i++) { - cv::circle(*img, pts[i], thickness, color, -1); + for (gint i = 0; i < pts_size; i++) { + cv::circle(img, pts[i], thickness, color, -1); } } else { cv::Point s = pts[0]; cv::Point e = pts[0]; - for (int i = 0; i < pts_size; i++) { + for (gint i = 0; i < pts_size; i++) { s = e; e = pts[i]; if (1 == i % 2) { - cv::line(*img, s, e, color, thickness); + cv::line(img, s, e, color, thickness); } } } } static void -gst_get_meta (GstInferencePrediction *pred, cv::Mat *cv_mat, gdouble font_scale, +gst_get_meta (GstInferencePrediction *pred, cv::Mat &cv_mat, gdouble font_scale, gint thickness, gchar **labels_list, gint num_labels, LineStyleBoundingBox style) { cv::Size size; @@ -156,7 +155,6 @@ gst_get_meta (GstInferencePrediction *pred, cv::Mat *cv_mat, gdouble font_scale, g_return_if_fail (pred != NULL); g_return_if_fail (labels_list != NULL); - g_return_if_fail (cv_mat != NULL); list = gst_inference_prediction_get_children(pred); @@ -180,20 +178,20 @@ gst_get_meta (GstInferencePrediction *pred, cv::Mat *cv_mat, gdouble font_scale, label = cv::format ("Label #%d Prob: %f", classification->class_id, classification->class_prob); } - cv::putText (*cv_mat, label, cv::Point (box.x + box.width, box.y + classes * OVERLAY_WIDTH), + cv::putText (cv_mat, label, cv::Point (box.x + box.width, box.y + classes * OVERLAY_WIDTH), cv::FONT_HERSHEY_PLAIN, font_scale, cv::Scalar::all(0), thickness); } - cv_mat->copyTo(alpha_overlay); + cv_mat.copyTo(alpha_overlay); cv::Size text = cv::getTextSize (label, cv::FONT_HERSHEY_PLAIN, font_scale, thickness, 0); cv::rectangle(alpha_overlay, cv::Rect(box.x + box.width, box.y, text.width, OVERLAY_HEIGHT * classes), cv::Scalar(255, 255, 255), -1); - cv::addWeighted(alpha_overlay, alpha, *cv_mat, 1 - alpha, 0, *cv_mat); + cv::addWeighted(alpha_overlay, alpha, cv_mat, 1 - alpha, 0, cv_mat); if (FALSE == G_NODE_IS_ROOT(pred->predictions)) { if (0 == style) { - cv::rectangle (*cv_mat, cv::Point (box.x, box.y), + cv::rectangle (cv_mat, cv::Point (box.x, box.y), cv::Point (box.x + box.width, box.y + box.height), colors[CHOSEN_COLOR % N_C], thickness); }else { @@ -245,7 +243,7 @@ gst_inference_overlay_process_meta (GstInferenceBaseOverlay *inference_overlay, cv_mat = cv::Mat (height, width, CV_MAKETYPE (CV_8U, channels), (char *) frame->data[0]); - gst_get_meta (detect_meta->prediction, &cv_mat, font_scale, thickness, + gst_get_meta (detect_meta->prediction, cv_mat, font_scale, thickness, labels_list, num_labels, style); return GST_FLOW_OK; From 5063f7a5d146d830a0f523543fd744838ba86082 Mon Sep 17 00:00:00 2001 From: tvlenin Date: Mon, 20 Jan 2020 13:07:18 -0600 Subject: [PATCH 125/188] Use ROI instead of copying the entire image --- ext/opencv/gstinferenceoverlay.cc | 40 +++++++++++++++++++++++++++---- 1 file changed, 35 insertions(+), 5 deletions(-) diff --git a/ext/opencv/gstinferenceoverlay.cc b/ext/opencv/gstinferenceoverlay.cc index 71c29b27..9ddd969d 100644 --- a/ext/opencv/gstinferenceoverlay.cc +++ b/ext/opencv/gstinferenceoverlay.cc @@ -152,7 +152,7 @@ gst_get_meta (GstInferencePrediction *pred, cv::Mat &cv_mat, gdouble font_scale, gint classes = 0; gdouble alpha = 0.5; cv::Mat alpha_overlay; - + gint width, height,x, y = 0; g_return_if_fail (pred != NULL); g_return_if_fail (labels_list != NULL); @@ -182,12 +182,42 @@ gst_get_meta (GstInferencePrediction *pred, cv::Mat &cv_mat, gdouble font_scale, cv::FONT_HERSHEY_PLAIN, font_scale, cv::Scalar::all(0), thickness); } - cv_mat.copyTo(alpha_overlay); cv::Size text = cv::getTextSize (label, cv::FONT_HERSHEY_PLAIN, font_scale, thickness, 0); - cv::rectangle(alpha_overlay, cv::Rect(box.x + box.width, box.y, text.width, - OVERLAY_HEIGHT * classes), cv::Scalar(255, 255, 255), -1); - cv::addWeighted(alpha_overlay, alpha, cv_mat, 1 - alpha, 0, cv_mat); + + if ((box.x + box.width) < 0) { + x = 0; + } else if ((int)(box.x + box.width) >= cv_mat.cols) { + x = cv_mat.cols - 1; + } else { + x = box.x + box.width; + } + + if ((int)(x + box.width + text.width) >= cv_mat.cols) { + width = cv_mat.cols - x - 1; + } else { + width = text.width; + } + + if ((int)(box.y + OVERLAY_HEIGHT * classes ) >= cv_mat.rows) { + y = cv_mat.rows - 1; + }else if ((int)box.y < 0) { + y = 1; + }else { + y = box.y; + } + if ((int)(y + (OVERLAY_HEIGHT * classes)) >= cv_mat.rows) { + height = cv_mat.rows - y - 1; + } else { + height = OVERLAY_HEIGHT * classes; + } + + if (width && height) { + cv::Mat rectangle (height, width, cv_mat.type(), cv::Scalar (255, 255, 255) ); + cv::Rect roi (x, y, width, height); + cv::addWeighted (cv_mat (roi), alpha, rectangle, 1.0 - alpha, 0.0, + cv_mat (roi) ); + } if (FALSE == G_NODE_IS_ROOT(pred->predictions)) { if (0 == style) { From b333a2796627efa8118dc54a3dd1e16edc19e006 Mon Sep 17 00:00:00 2001 From: tvlenin Date: Mon, 20 Jan 2020 16:17:14 -0600 Subject: [PATCH 126/188] Add constant to represent GAP between lines --- ext/opencv/gstinferenceoverlay.cc | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/ext/opencv/gstinferenceoverlay.cc b/ext/opencv/gstinferenceoverlay.cc index 9ddd969d..eb1a33a6 100644 --- a/ext/opencv/gstinferenceoverlay.cc +++ b/ext/opencv/gstinferenceoverlay.cc @@ -52,6 +52,7 @@ colors[] = { #define CHOSEN_COLOR 14 #define OVERLAY_HEIGHT 50 #define OVERLAY_WIDTH 30 +#define LINES_GAP 20 GST_DEBUG_CATEGORY_STATIC (gst_inference_overlay_debug_category); #define GST_CAT_DEFAULT gst_inference_overlay_debug_category @@ -227,16 +228,16 @@ gst_get_meta (GstInferencePrediction *pred, cv::Mat &cv_mat, gdouble font_scale, }else { draw_line (cv_mat, cv::Point (box.x, box.y), cv::Point (box.x + box.width, box.y), - colors[CHOSEN_COLOR % N_C], thickness, style, 20); + colors[CHOSEN_COLOR % N_C], thickness, style, LINES_GAP); draw_line (cv_mat, cv::Point (box.x, box.y), cv::Point (box.x, box.y + box.height), - colors[CHOSEN_COLOR % N_C], thickness, style, 20); + colors[CHOSEN_COLOR % N_C], thickness, style, LINES_GAP); draw_line (cv_mat, cv::Point (box.x + box.width, box.y), cv::Point (box.x + box.width, box.y + box.height), - colors[CHOSEN_COLOR % N_C], thickness, style, 20); + colors[CHOSEN_COLOR % N_C], thickness, style, LINES_GAP); draw_line (cv_mat, cv::Point (box.x, box.y + box.height), cv::Point (box.x + box.width, box.y + box.height), - colors[CHOSEN_COLOR % N_C], thickness, style, 20); + colors[CHOSEN_COLOR % N_C], thickness, style, LINES_GAP); } } From 80975ae9d2c36913bebdecb5e96c6fe5fa6a92d4 Mon Sep 17 00:00:00 2001 From: Michael Gruner Date: Mon, 13 Jan 2020 19:18:24 -0600 Subject: [PATCH 127/188] Make predictions thread safe --- ext/r2inference/gsttinyyolov2.c | 6 +- .../r2inference/gstinferenceclassification.c | 20 +++++- .../r2inference/gstinferenceclassification.h | 18 ++++++ gst-libs/gst/r2inference/gstinferencedebug.c | 3 +- .../gst/r2inference/gstinferencepostprocess.c | 13 ++-- .../gst/r2inference/gstinferenceprediction.c | 63 +++++++++++++++++-- .../gst/r2inference/gstinferenceprediction.h | 41 ++++++++++++ 7 files changed, 149 insertions(+), 15 deletions(-) diff --git a/ext/r2inference/gsttinyyolov2.c b/ext/r2inference/gsttinyyolov2.c index b2c27f84..33aace84 100644 --- a/ext/r2inference/gsttinyyolov2.c +++ b/ext/r2inference/gsttinyyolov2.c @@ -366,15 +366,15 @@ gst_tinyyolov2_postprocess_new (GstVideoInference * vi, GST_LOG_OBJECT (tinyyolov2, "Number of predictions: %d", num_boxes); if (NULL == imeta->prediction) { - imeta->prediction = gst_inference_prediction_new (); - imeta->prediction->bbox.width = info_model->width; - imeta->prediction->bbox.height = info_model->height; + BoundingBox bbox = { 0, 0, info_model->width, info_model->height }; + imeta->prediction = gst_inference_prediction_new_full (&bbox); } for (i = 0; i < num_boxes; i++) { GstInferencePrediction *pred = gst_create_prediction_from_box (vi, &boxes[i], labels_list, num_labels, probabilities); + gst_inference_prediction_append (imeta->prediction, pred); } diff --git a/gst-libs/gst/r2inference/gstinferenceclassification.c b/gst-libs/gst/r2inference/gstinferenceclassification.c index a6c355eb..b2d24bd8 100644 --- a/gst-libs/gst/r2inference/gstinferenceclassification.c +++ b/gst-libs/gst/r2inference/gstinferenceclassification.c @@ -74,6 +74,8 @@ gst_inference_classification_new (void) (GstMiniObjectCopyFunction) gst_inference_classification_copy, NULL, (GstMiniObjectFreeFunction) classification_free); + g_mutex_init (&self->mutex); + self->class_label = NULL; self->probabilities = NULL; self->labels = NULL; @@ -107,6 +109,8 @@ gst_inference_classification_new_full (gint class_id, gdouble class_prob, { GstInferenceClassification *self = gst_inference_classification_new (); + GST_INFERENCE_CLASSIFICATION_LOCK (self); + self->class_id = class_id; self->class_prob = class_prob; self->num_classes = num_classes; @@ -123,6 +127,8 @@ gst_inference_classification_new_full (gint class_id, gdouble class_prob, self->labels = g_strdupv (labels); } + GST_INFERENCE_CLASSIFICATION_UNLOCK (self); + return self; } @@ -152,6 +158,8 @@ gst_inference_classification_copy (const GstInferenceClassification * self) other = gst_inference_classification_new (); + GST_INFERENCE_CLASSIFICATION_LOCK ((GstInferenceClassification *) self); + other->class_id = self->class_id; other->class_prob = self->class_prob; other->num_classes = self->num_classes; @@ -169,6 +177,8 @@ gst_inference_classification_copy (const GstInferenceClassification * self) other->labels = g_strdupv (self->labels); } + GST_INFERENCE_CLASSIFICATION_UNLOCK ((GstInferenceClassification *) self); + return other; } @@ -177,9 +187,13 @@ gst_inference_classification_to_string (GstInferenceClassification * self, gint level) { gint indent = level * 2; + gchar *serial = NULL; + g_return_val_if_fail (self, NULL); - return g_strdup_printf ("{\n" + GST_INFERENCE_CLASSIFICATION_LOCK (self); + + serial = g_strdup_printf ("{\n" "%*s Class : %d\n" "%*s Label : %s\n" "%*s Probability : %f\n" @@ -188,6 +202,10 @@ gst_inference_classification_to_string (GstInferenceClassification * self, indent, "", self->class_id, indent, "", self->class_label, indent, "", self->class_prob, indent, "", self->num_classes, indent, ""); + + GST_INFERENCE_CLASSIFICATION_UNLOCK (self); + + return serial; } static void diff --git a/gst-libs/gst/r2inference/gstinferenceclassification.h b/gst-libs/gst/r2inference/gstinferenceclassification.h index 19f8ad6b..71d477fc 100644 --- a/gst-libs/gst/r2inference/gstinferenceclassification.h +++ b/gst-libs/gst/r2inference/gstinferenceclassification.h @@ -43,6 +43,7 @@ struct _GstInferenceClassification { /**/ GstMiniObject base; + GMutex mutex; /**/ gint class_id; @@ -134,6 +135,23 @@ void gst_inference_classification_unref (GstInferenceClassification * self); */ gchar * gst_inference_classification_to_string (GstInferenceClassification * self, gint level); +/** + * GST_INFERENCE_CLASSIFICATION_LOCK: + * @c: The GstInferenceClassification to lock + * + * Locks the classification to avoid concurrent access from different + * threads. + */ +#define GST_INFERENCE_CLASSIFICATION_LOCK(c) g_mutex_lock (&((c)->mutex)) + +/** + * GST_INFERENCE_CLASSIFICATION_UNLOCK: + * @c: The GstInferenceClassification to unlock + * + * Unlocks the prediction to yield the access to other threads. + */ +#define GST_INFERENCE_CLASSIFICATION_UNLOCK(c) g_mutex_unlock (&((c)->mutex)) + G_END_DECLS #endif // __GST_INFERENCE_CLASSIFICATION__ diff --git a/gst-libs/gst/r2inference/gstinferencedebug.c b/gst-libs/gst/r2inference/gstinferencedebug.c index 35970970..facf0c86 100644 --- a/gst-libs/gst/r2inference/gstinferencedebug.c +++ b/gst-libs/gst/r2inference/gstinferencedebug.c @@ -106,8 +106,9 @@ gst_inference_print_predictions (GstVideoInference * vi, g_return_if_fail (inference_meta != NULL); pred = inference_meta->prediction; - spred = gst_inference_prediction_to_string (pred); + GST_CAT_LOG (category, "\n%s", spred); + g_free (spred); } diff --git a/gst-libs/gst/r2inference/gstinferencepostprocess.c b/gst-libs/gst/r2inference/gstinferencepostprocess.c index 054c95dc..328456cf 100644 --- a/gst-libs/gst/r2inference/gstinferencepostprocess.c +++ b/gst-libs/gst/r2inference/gstinferencepostprocess.c @@ -264,15 +264,18 @@ gst_create_prediction_from_box (GstVideoInference * vi, BBox * box, GstInferencePrediction *predict = NULL; GstInferenceClassification *c = NULL; gchar *label = NULL; + BoundingBox bbox; + g_return_val_if_fail (vi != NULL, NULL); g_return_val_if_fail (box != NULL, NULL); g_return_val_if_fail (probabilities != NULL, NULL); - predict = gst_inference_prediction_new (); - predict->bbox.x = box->x; - predict->bbox.y = box->y; - predict->bbox.width = box->width; - predict->bbox.height = box->height; + bbox.x = box->x; + bbox.y = box->y; + bbox.width = box->width; + bbox.height = box->height; + + predict = gst_inference_prediction_new_full (&bbox); if (num_labels > box->label) { label = labels_list[box->label]; diff --git a/gst-libs/gst/r2inference/gstinferenceprediction.c b/gst-libs/gst/r2inference/gstinferenceprediction.c index 8d5767d9..122dd059 100644 --- a/gst-libs/gst/r2inference/gstinferenceprediction.c +++ b/gst-libs/gst/r2inference/gstinferenceprediction.c @@ -35,6 +35,7 @@ static gchar *prediction_classes_to_string (GstInferencePrediction * self, gint level); static GstInferencePrediction *prediction_scale (const GstInferencePrediction * self, GstVideoInfo * to, GstVideoInfo * from); +static GSList *prediction_get_children_unlocked (GstInferencePrediction * self); static GstInferenceClassification * classification_copy (GstInferenceClassification * from, gpointer data); @@ -81,6 +82,8 @@ gst_inference_prediction_new (void) (GstMiniObjectCopyFunction) gst_inference_prediction_copy, NULL, (GstMiniObjectFreeFunction) prediction_free); + g_mutex_init (&self->mutex); + self->predictions = NULL; self->classifications = NULL; @@ -89,6 +92,22 @@ gst_inference_prediction_new (void) return self; } +GstInferencePrediction * +gst_inference_prediction_new_full (BoundingBox * bbox) +{ + GstInferencePrediction *self = NULL; + + g_return_val_if_fail (bbox, NULL); + + self = gst_inference_prediction_new (); + + GST_INFERENCE_PREDICTION_LOCK (self); + self->bbox = *bbox; + GST_INFERENCE_PREDICTION_UNLOCK (self); + + return self; +} + GstInferencePrediction * gst_inference_prediction_ref (GstInferencePrediction * self) { @@ -113,7 +132,11 @@ gst_inference_prediction_append (GstInferencePrediction * self, g_return_if_fail (self); g_return_if_fail (child); + GST_INFERENCE_PREDICTION_LOCK (self); + GST_INFERENCE_PREDICTION_LOCK (child); g_node_append (self->predictions, child->predictions); + GST_INFERENCE_PREDICTION_UNLOCK (child); + GST_INFERENCE_PREDICTION_UNLOCK (self); } static GstInferenceClassification * @@ -171,12 +194,16 @@ gst_inference_prediction_copy (const GstInferencePrediction * self) g_return_val_if_fail (self, NULL); + GST_INFERENCE_PREDICTION_LOCK ((GstInferencePrediction *) self); + /* Copy the binary tree */ other = g_node_copy_deep (self->predictions, node_copy, NULL); /* Now finish assigning the nodes to the predictions */ g_node_traverse (other, G_IN_ORDER, G_TRAVERSE_ALL, -1, node_assign, NULL); + GST_INFERENCE_PREDICTION_UNLOCK ((GstInferencePrediction *) self); + return (GstInferencePrediction *) other->data; } @@ -210,7 +237,7 @@ prediction_children_to_string (GstInferencePrediction * self, gint level) /* Build the child predictions using a GString */ string = g_string_new (NULL); - subpreds = gst_inference_prediction_get_children (self); + subpreds = prediction_get_children_unlocked (self); for (iter = subpreds; iter != NULL; iter = g_slist_next (iter)) { GstInferencePrediction *pred = (GstInferencePrediction *) iter->data; @@ -287,7 +314,15 @@ prediction_to_string (GstInferencePrediction * self, gint level) gchar * gst_inference_prediction_to_string (GstInferencePrediction * self) { - return prediction_to_string (self, 0); + gchar *serial = NULL; + + g_return_val_if_fail (self, NULL); + + GST_INFERENCE_PREDICTION_LOCK (self); + serial = prediction_to_string (self, 0); + GST_INFERENCE_PREDICTION_UNLOCK (self); + + return serial; } static void @@ -304,8 +339,8 @@ node_get_children (GNode * node, gpointer data) *children = g_slist_append (*children, prediction); } -GSList * -gst_inference_prediction_get_children (GstInferencePrediction * self) +static GSList * +prediction_get_children_unlocked (GstInferencePrediction * self) { GSList *children = NULL; @@ -319,6 +354,18 @@ gst_inference_prediction_get_children (GstInferencePrediction * self) return children; } +GSList * +gst_inference_prediction_get_children (GstInferencePrediction * self) +{ + GSList *children = NULL; + + GST_INFERENCE_PREDICTION_LOCK (self); + children = prediction_get_children_unlocked (self); + GST_INFERENCE_PREDICTION_UNLOCK (self); + + return children; +} + static void bounding_box_reset (BoundingBox * bbox) { @@ -349,7 +396,7 @@ prediction_reset (GstInferencePrediction * self) static void prediction_free (GstInferencePrediction * self) { - GSList *children = gst_inference_prediction_get_children (self); + GSList *children = prediction_get_children_unlocked (self); GSList *iter = NULL; for (iter = children; iter != NULL; iter = g_slist_next (iter)) { @@ -375,7 +422,9 @@ gst_inference_prediction_append_classification (GstInferencePrediction * self, g_return_if_fail (self); g_return_if_fail (c); + GST_INFERENCE_PREDICTION_LOCK (self); self->classifications = g_list_append (self->classifications, c); + GST_INFERENCE_PREDICTION_UNLOCK (self); } static GstInferencePrediction * @@ -437,10 +486,14 @@ gst_inference_prediction_scale (GstInferencePrediction * self, g_return_val_if_fail (to, NULL); g_return_val_if_fail (from, NULL); + GST_INFERENCE_PREDICTION_LOCK (self); + other = g_node_copy_deep (self->predictions, node_scale, &data); /* Now finish assigning the nodes to the predictions */ g_node_traverse (other, G_IN_ORDER, G_TRAVERSE_ALL, -1, node_assign, NULL); + GST_INFERENCE_PREDICTION_UNLOCK (self); + return (GstInferencePrediction *) other->data; } diff --git a/gst-libs/gst/r2inference/gstinferenceprediction.h b/gst-libs/gst/r2inference/gstinferenceprediction.h index c0262a60..734345b6 100644 --- a/gst-libs/gst/r2inference/gstinferenceprediction.h +++ b/gst-libs/gst/r2inference/gstinferenceprediction.h @@ -68,6 +68,7 @@ struct _GstInferencePrediction { /**/ GstMiniObject base; + GMutex mutex; /**/ guint64 prediction_id; @@ -80,6 +81,29 @@ struct _GstInferencePrediction /** * gst_inference_prediction_new: * + * Creates a new GstInferencePrediction. Values can be later assigned + * manually, however these assignments should be done with the + * GST_INFERENCE_PREDICTION_LOCK held. See + * gst_inference_prediction_new_full for a thread safe version. + * + * Returns: A newly allocated and initialized GstInferencePrediction. + */ +GstInferencePrediction * gst_inference_prediction_new (void); + +/** + * gst_inference_prediction_new_full: + * @bbox: The bounding box of this prediction. + * + * Creates a new GstInferencePrediction and initializes its internal + * values. + * + * Returns: A newly allocated and initialized GstInferencePrediction. + */ +GstInferencePrediction * gst_inference_prediction_new_full (BoundingBox *bbox); + +/** + * gst_inference_prediction_new_full: + * * Creates a new GstInferencePrediction. * * Returns: A newly allocated and initialized GstInferencePrediction. @@ -188,6 +212,23 @@ void gst_inference_prediction_append_classification (GstInferencePrediction * se GstInferencePrediction * gst_inference_prediction_scale (GstInferencePrediction * self, GstVideoInfo * to, GstVideoInfo * from); +/** + * GST_INFERENCE_PREDICTION_LOCK: + * @p: The GstInferencePrediction to lock + * + * Locks the prediction to avoid concurrent access from different + * threads. + */ +#define GST_INFERENCE_PREDICTION_LOCK(p) g_mutex_lock (&((p)->mutex)) + +/** + * GST_INFERENCE_PREDICTION_UNLOCK: + * @p: The GstInferencePrediction to unlock + * + * Unlocks the prediction to yield the access to other threads. + */ +#define GST_INFERENCE_PREDICTION_UNLOCK(p) g_mutex_unlock (&((p)->mutex)) + G_END_DECLS #endif // __GST_INFERENCE_PREDICTION__ From 37addd53cfde68eb8493b599aba64a7c2eeb10ea Mon Sep 17 00:00:00 2001 From: Michael Gruner Date: Thu, 16 Jan 2020 19:35:51 -0600 Subject: [PATCH 128/188] Implement a detection find method --- .../gst/r2inference/gstinferenceprediction.c | 68 +++++++++++++++++-- .../gst/r2inference/gstinferenceprediction.h | 14 ++++ 2 files changed, 75 insertions(+), 7 deletions(-) diff --git a/gst-libs/gst/r2inference/gstinferenceprediction.c b/gst-libs/gst/r2inference/gstinferenceprediction.c index 122dd059..1fa49718 100644 --- a/gst-libs/gst/r2inference/gstinferenceprediction.c +++ b/gst-libs/gst/r2inference/gstinferenceprediction.c @@ -24,9 +24,25 @@ static GType gst_inference_prediction_get_type (void); GST_DEFINE_MINI_OBJECT_TYPE (GstInferencePrediction, gst_inference_prediction); +typedef struct _PredictionScaleData PredictionScaleData; +struct _PredictionScaleData +{ + GstVideoInfo *from; + GstVideoInfo *to; +}; + +typedef struct _PredictionFindData PredictionFindData; +struct _PredictionFindData +{ + GstInferencePrediction *prediction; + guint64 prediction_id; +}; + static GstInferencePrediction *prediction_copy (const GstInferencePrediction * self); static void prediction_free (GstInferencePrediction * obj); +static gboolean prediction_find (GstInferencePrediction * obj, + PredictionFindData * found); static void prediction_reset (GstInferencePrediction * self); static gchar *prediction_to_string (GstInferencePrediction * self, gint level); static gchar *prediction_children_to_string (GstInferencePrediction * self, @@ -47,16 +63,10 @@ static void node_get_children (GNode * node, gpointer data); static gpointer node_copy (gconstpointer node, gpointer data); static gpointer node_scale (gconstpointer node, gpointer data); static gboolean node_assign (GNode * node, gpointer data); +static gboolean node_find (GNode * node, gpointer data); static guint64 get_new_id (void); -typedef struct _PredictionScaleData PredictionScaleData; -struct _PredictionScaleData -{ - GstVideoInfo *from; - GstVideoInfo *to; -}; - static guint64 get_new_id (void) { @@ -497,3 +507,47 @@ gst_inference_prediction_scale (GstInferencePrediction * self, return (GstInferencePrediction *) other->data; } + +static gboolean +prediction_find (GstInferencePrediction * obj, PredictionFindData * found) +{ + gboolean ret = FALSE; + + g_return_val_if_fail (obj, TRUE); + g_return_val_if_fail (found, TRUE); + + if (obj->prediction_id == found->prediction_id) { + found->prediction = gst_inference_prediction_ref (obj); + ret = TRUE; + } + + return ret; +} + +static gboolean +node_find (GNode * node, gpointer data) +{ + PredictionFindData *found = (PredictionFindData *) data; + GstInferencePrediction *current = (GstInferencePrediction *) node->data; + + g_return_val_if_fail (found, TRUE); + + return prediction_find (current, found); +} + +GstInferencePrediction * +gst_inference_prediction_find (GstInferencePrediction * self, guint64 id) +{ + PredictionFindData found = { 0 }; + + g_return_val_if_fail (self, NULL); + + GST_INFERENCE_PREDICTION_LOCK (self); + + g_node_traverse (self->predictions, G_IN_ORDER, G_TRAVERSE_ALL, -1, node_find, + &found); + + GST_INFERENCE_PREDICTION_UNLOCK (self); + + return found.prediction; +} diff --git a/gst-libs/gst/r2inference/gstinferenceprediction.h b/gst-libs/gst/r2inference/gstinferenceprediction.h index 734345b6..9feb4d2c 100644 --- a/gst-libs/gst/r2inference/gstinferenceprediction.h +++ b/gst-libs/gst/r2inference/gstinferenceprediction.h @@ -212,6 +212,20 @@ void gst_inference_prediction_append_classification (GstInferencePrediction * se GstInferencePrediction * gst_inference_prediction_scale (GstInferencePrediction * self, GstVideoInfo * to, GstVideoInfo * from); +/** + * gst_inference_prediction_find: + * @self: the root prediction + * @id: the prediction_id of the prediction to return + * + * Traverses the prediction tree looking for a child with the given + * id. + * + * Returns: a reference to the prediction with id or NULL if not + * found. Unref after usage. + */ +GstInferencePrediction * gst_inference_prediction_find (GstInferencePrediction * self, + guint64 id); + /** * GST_INFERENCE_PREDICTION_LOCK: * @p: The GstInferencePrediction to lock From 3fce1fd19c66f98fc90015fbc5b0fa61e372bdfa Mon Sep 17 00:00:00 2001 From: Michael Gruner Date: Thu, 16 Jan 2020 19:57:47 -0600 Subject: [PATCH 129/188] Install an ID to the classification as well --- .../r2inference/gstinferenceclassification.c | 20 +++++++++++++++++++ .../r2inference/gstinferenceclassification.h | 2 ++ .../gst/r2inference/gstinferenceprediction.c | 2 +- 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/gst-libs/gst/r2inference/gstinferenceclassification.c b/gst-libs/gst/r2inference/gstinferenceclassification.c index b2d24bd8..b6d3d022 100644 --- a/gst-libs/gst/r2inference/gstinferenceclassification.c +++ b/gst-libs/gst/r2inference/gstinferenceclassification.c @@ -39,11 +39,28 @@ static void classification_reset (GstInferenceClassification * self); static gdouble *probabilities_copy (const gdouble * from, gint num_classes); +static guint64 get_new_id (void); + +static guint64 +get_new_id (void) +{ + static guint64 _id = G_GUINT64_CONSTANT (0); + static GMutex _id_mutex; + static guint64 ret = 0; + + g_mutex_lock (&_id_mutex); + ret = _id++; + g_mutex_unlock (&_id_mutex); + + return ret; +} + static void classification_reset (GstInferenceClassification * self) { g_return_if_fail (self); + self->classification_id = get_new_id (); self->class_id = DEFAULT_CLASS_ID; self->class_prob = DEFAULT_CLASS_PROB; self->num_classes = DEFAULT_NUM_CLASSES; @@ -160,6 +177,7 @@ gst_inference_classification_copy (const GstInferenceClassification * self) GST_INFERENCE_CLASSIFICATION_LOCK ((GstInferenceClassification *) self); + other->classification_id = self->classification_id; other->class_id = self->class_id; other->class_prob = self->class_prob; other->num_classes = self->num_classes; @@ -194,11 +212,13 @@ gst_inference_classification_to_string (GstInferenceClassification * self, GST_INFERENCE_CLASSIFICATION_LOCK (self); serial = g_strdup_printf ("{\n" + "%*s Id : %" G_GUINT64_FORMAT "\n" "%*s Class : %d\n" "%*s Label : %s\n" "%*s Probability : %f\n" "%*s Classes : %d\n" "%*s}", + indent, "", self->classification_id, indent, "", self->class_id, indent, "", self->class_label, indent, "", self->class_prob, indent, "", self->num_classes, indent, ""); diff --git a/gst-libs/gst/r2inference/gstinferenceclassification.h b/gst-libs/gst/r2inference/gstinferenceclassification.h index 71d477fc..3b3cb95a 100644 --- a/gst-libs/gst/r2inference/gstinferenceclassification.h +++ b/gst-libs/gst/r2inference/gstinferenceclassification.h @@ -28,6 +28,7 @@ G_BEGIN_DECLS /** * GstInferenceClassification: + * @classification_id: a unique id associated to this classification * @class_id: the numerical id associated to the assigned class * @class_prob: the resulting probability of the assigned * class. Typically between 0 and 1 @@ -46,6 +47,7 @@ struct _GstInferenceClassification GMutex mutex; /**/ + guint64 classification_id; gint class_id; gdouble class_prob; gchar *class_label; diff --git a/gst-libs/gst/r2inference/gstinferenceprediction.c b/gst-libs/gst/r2inference/gstinferenceprediction.c index 1fa49718..dd61a46e 100644 --- a/gst-libs/gst/r2inference/gstinferenceprediction.c +++ b/gst-libs/gst/r2inference/gstinferenceprediction.c @@ -70,7 +70,7 @@ static guint64 get_new_id (void); static guint64 get_new_id (void) { - static guint64 _id = 0; + static guint64 _id = G_GUINT64_CONSTANT (0); static GMutex _id_mutex; static guint64 ret = 0; From e483cc510f4829a9a0496aa02e3a6cee2965ede4 Mon Sep 17 00:00:00 2001 From: Michael Gruner Date: Thu, 16 Jan 2020 20:18:17 -0600 Subject: [PATCH 130/188] Implement a prediction merge method --- .../gst/r2inference/gstinferenceprediction.c | 103 +++++++++++++++++- .../gst/r2inference/gstinferenceprediction.h | 11 +- 2 files changed, 111 insertions(+), 3 deletions(-) diff --git a/gst-libs/gst/r2inference/gstinferenceprediction.c b/gst-libs/gst/r2inference/gstinferenceprediction.c index dd61a46e..bfdc67ff 100644 --- a/gst-libs/gst/r2inference/gstinferenceprediction.c +++ b/gst-libs/gst/r2inference/gstinferenceprediction.c @@ -52,9 +52,13 @@ static gchar *prediction_classes_to_string (GstInferencePrediction * self, static GstInferencePrediction *prediction_scale (const GstInferencePrediction * self, GstVideoInfo * to, GstVideoInfo * from); static GSList *prediction_get_children_unlocked (GstInferencePrediction * self); +static void prediction_merge (GstInferencePrediction * src, + GstInferencePrediction * dst); static GstInferenceClassification * classification_copy (GstInferenceClassification * from, gpointer data); +static void classification_merge (GList * src, GList ** dst); +static gint classification_compare (gconstpointer a, gconstpointer b); static void bounding_box_reset (BoundingBox * bbox); static gchar *bounding_box_to_string (BoundingBox * bbox, gint level); @@ -81,7 +85,6 @@ get_new_id (void) return ret; } - GstInferencePrediction * gst_inference_prediction_new (void) { @@ -538,7 +541,7 @@ node_find (GNode * node, gpointer data) GstInferencePrediction * gst_inference_prediction_find (GstInferencePrediction * self, guint64 id) { - PredictionFindData found = { 0 }; + PredictionFindData found = { 0, id }; g_return_val_if_fail (self, NULL); @@ -551,3 +554,99 @@ gst_inference_prediction_find (GstInferencePrediction * self, guint64 id) return found.prediction; } + +static gint +classification_compare (gconstpointer a, gconstpointer b) +{ + GstInferenceClassification *ca = (GstInferenceClassification *) a; + GstInferenceClassification *cb = (GstInferenceClassification *) b; + + return ca->classification_id == cb->classification_id ? 0 : 1; +} + +static void +classification_merge (GList * src, GList ** dst) +{ + GList *iter = NULL; + + g_return_if_fail (dst); + + /* For each classification in the src, see if it exists in the dst */ + for (iter = src; iter; iter = g_list_next (iter)) { + GList *exists = + g_list_find_custom (*dst, iter->data, classification_compare); + + /* Copy and append it to the dst if it doesn't exist */ + if (!exists) { + GstInferenceClassification *child = + gst_inference_classification_copy (iter->data); + *dst = g_list_append (*dst, child); + } + } +} + +static void +prediction_merge (GstInferencePrediction * src, GstInferencePrediction * dst) +{ + GSList *src_children = prediction_get_children_unlocked (src); + GSList *iter = NULL; + GSList *new_children = NULL; + + g_return_if_fail (src); + g_return_if_fail (dst); + g_return_if_fail (src->prediction_id == dst->prediction_id); + + /* Two things might've happened: + * 1) A new class was added + * 2) A new subprediction was added + */ + + /* Handle 1) here */ + classification_merge (src->classifications, &dst->classifications); + + /* Handle 2) here */ + for (iter = src_children; iter; iter = g_slist_next (iter)) { + GstInferencePrediction *current = (GstInferencePrediction *) iter->data; + + /* TODO: Optimize this by only searching the immediate children */ + GstInferencePrediction *found = + gst_inference_prediction_find (dst, current->prediction_id); + + /* No matching prediction, save it to append it later */ + if (!found) { + new_children = g_slist_append (new_children, current); + continue; + } + + /* Recurse into the children */ + prediction_merge (current, found); + + gst_inference_prediction_unref (found); + } + + /* Finally append all the new children to dst. Do it after all + children have been processed */ + for (iter = new_children; iter; iter = g_slist_next (iter)) { + GstInferencePrediction *prediction = + gst_inference_prediction_copy ((GstInferencePrediction *) iter->data); + gst_inference_prediction_append (dst, prediction); + } +} + +void +gst_inference_prediction_merge (GstInferencePrediction * src, + GstInferencePrediction * dst) +{ + g_return_if_fail (src); + g_return_if_fail (dst); + + GST_INFERENCE_PREDICTION_LOCK (src); + if (src != dst) + GST_INFERENCE_PREDICTION_LOCK (dst); + + prediction_merge (src, dst); + + if (src != dst) + GST_INFERENCE_PREDICTION_UNLOCK (dst); + GST_INFERENCE_PREDICTION_UNLOCK (src); +} diff --git a/gst-libs/gst/r2inference/gstinferenceprediction.h b/gst-libs/gst/r2inference/gstinferenceprediction.h index 9feb4d2c..9ec45811 100644 --- a/gst-libs/gst/r2inference/gstinferenceprediction.h +++ b/gst-libs/gst/r2inference/gstinferenceprediction.h @@ -198,7 +198,7 @@ void gst_inference_prediction_append_classification (GstInferencePrediction * se /** * gst_inference_prediction_scale: - * @self: the original prediction + * @self: the prediction to scale * @to: the resulting image size * @from: the original image size * @@ -226,6 +226,15 @@ GstInferencePrediction * gst_inference_prediction_scale (GstInferencePrediction GstInferencePrediction * gst_inference_prediction_find (GstInferencePrediction * self, guint64 id); +/** + * gst_inference_prediction_merge: + * @src: the source prediction + * @dst: the destination prediction + * + * Copies the extra information from src to dst. + */ +void gst_inference_prediction_merge (GstInferencePrediction * src, GstInferencePrediction * dst); + /** * GST_INFERENCE_PREDICTION_LOCK: * @p: The GstInferencePrediction to lock From a9a0756474a4ea796afa8ca3a69d509fca1cb4e0 Mon Sep 17 00:00:00 2001 From: Michael Gruner Date: Tue, 21 Jan 2020 10:26:41 -0600 Subject: [PATCH 131/188] Support buffers with existing metas --- gst-libs/gst/r2inference/gstinferencemeta.c | 78 ++++++++++- .../gst/r2inference/gstinferenceprediction.c | 121 +++++++++++++++--- .../gst/r2inference/gstinferenceprediction.h | 14 ++ gst-libs/gst/r2inference/gstvideoinference.c | 39 +++--- 4 files changed, 210 insertions(+), 42 deletions(-) diff --git a/gst-libs/gst/r2inference/gstinferencemeta.c b/gst-libs/gst/r2inference/gstinferencemeta.c index 8811a275..0aee191d 100644 --- a/gst-libs/gst/r2inference/gstinferencemeta.c +++ b/gst-libs/gst/r2inference/gstinferencemeta.c @@ -199,16 +199,68 @@ gst_classification_meta_free (GstMeta * meta, GstBuffer * buffer) } static gboolean -gst_inference_meta_transform (GstBuffer * dest, GstMeta * meta, +gst_inference_meta_transform_existing_meta (GstBuffer * dest, GstMeta * meta, GstBuffer * buffer, GQuark type, gpointer data) { GstInferenceMeta *dmeta, *smeta; + GstInferencePrediction *pred = NULL; + gboolean ret = TRUE; g_return_val_if_fail (dest, FALSE); g_return_val_if_fail (meta, FALSE); g_return_val_if_fail (buffer, FALSE); - GST_LOG ("Transforming inference metadata"); + smeta = (GstInferenceMeta *) meta; + dmeta = + (GstInferenceMeta *) gst_buffer_get_meta (dest, + gst_inference_meta_api_get_type ()); + + g_return_val_if_fail (dmeta, FALSE); + + pred = + gst_inference_prediction_find (dmeta->prediction, + smeta->prediction->prediction_id); + + if (!pred) { + GST_ERROR + ("Predictions between metas do not match. Something really wrong happened"); + g_return_val_if_reached (FALSE); + } + + gst_inference_prediction_merge (smeta->prediction, pred); + + if (GST_META_TRANSFORM_IS_COPY (type)) { + GST_LOG ("Copy detection metadata"); + + /* The merge already handled the copy */ + goto out; + } + + if (GST_VIDEO_META_TRANSFORM_IS_SCALE (type)) { + GstVideoMetaTransform *trans = (GstVideoMetaTransform *) data; + gst_inference_prediction_scale_ip (pred, trans->out_info, trans->in_info); + + goto out; + } + + /* Invalid transformation */ + gst_buffer_remove_meta (dest, (GstMeta *) dmeta); + ret = FALSE; + +out: + gst_inference_prediction_unref (pred); + return ret; +} + +static gboolean +gst_inference_meta_transform_new_meta (GstBuffer * dest, GstMeta * meta, + GstBuffer * buffer, GQuark type, gpointer data) +{ + GstInferenceMeta *dmeta, *smeta; + + g_return_val_if_fail (dest, FALSE); + g_return_val_if_fail (meta, FALSE); + g_return_val_if_fail (buffer, FALSE); smeta = (GstInferenceMeta *) meta; dmeta = @@ -241,6 +293,28 @@ gst_inference_meta_transform (GstBuffer * dest, GstMeta * meta, return FALSE; } +static gboolean +gst_inference_meta_transform (GstBuffer * dest, GstMeta * meta, + GstBuffer * buffer, GQuark type, gpointer data) +{ + GstMeta *dmeta; + + g_return_val_if_fail (dest, FALSE); + g_return_val_if_fail (meta, FALSE); + g_return_val_if_fail (buffer, FALSE); + + GST_LOG ("Transforming inference metadata"); + + dmeta = gst_buffer_get_meta (dest, gst_inference_meta_api_get_type ()); + if (!dmeta) { + return gst_inference_meta_transform_new_meta (dest, meta, buffer, type, + data); + } else { + return gst_inference_meta_transform_existing_meta (dest, meta, buffer, type, + data); + } +} + static gboolean gst_detection_meta_init (GstMeta * meta, gpointer params, GstBuffer * buffer) { diff --git a/gst-libs/gst/r2inference/gstinferenceprediction.c b/gst-libs/gst/r2inference/gstinferenceprediction.c index bfdc67ff..cf50e218 100644 --- a/gst-libs/gst/r2inference/gstinferenceprediction.c +++ b/gst-libs/gst/r2inference/gstinferenceprediction.c @@ -43,6 +43,8 @@ static GstInferencePrediction *prediction_copy (const GstInferencePrediction * static void prediction_free (GstInferencePrediction * obj); static gboolean prediction_find (GstInferencePrediction * obj, PredictionFindData * found); +static GstInferencePrediction *prediction_find_unlocked (GstInferencePrediction + * self, guint64 id); static void prediction_reset (GstInferencePrediction * self); static gchar *prediction_to_string (GstInferencePrediction * self, gint level); static gchar *prediction_children_to_string (GstInferencePrediction * self, @@ -51,6 +53,8 @@ static gchar *prediction_classes_to_string (GstInferencePrediction * self, gint level); static GstInferencePrediction *prediction_scale (const GstInferencePrediction * self, GstVideoInfo * to, GstVideoInfo * from); +static void prediction_scale_ip (GstInferencePrediction * self, + GstVideoInfo * to, GstVideoInfo * from); static GSList *prediction_get_children_unlocked (GstInferencePrediction * self); static void prediction_merge (GstInferencePrediction * src, GstInferencePrediction * dst); @@ -65,10 +69,13 @@ static gchar *bounding_box_to_string (BoundingBox * bbox, gint level); static void node_get_children (GNode * node, gpointer data); static gpointer node_copy (gconstpointer node, gpointer data); -static gpointer node_scale (gconstpointer node, gpointer data); +static gboolean node_scale_ip (GNode * node, gpointer data); +static gpointer node_scale (gconstpointer, gpointer data); static gboolean node_assign (GNode * node, gpointer data); static gboolean node_find (GNode * node, gpointer data); +static void compute_factors (GstVideoInfo * from, GstVideoInfo * to, + gdouble * hfactor, gdouble * vfactor); static guint64 get_new_id (void); static guint64 @@ -440,12 +447,35 @@ gst_inference_prediction_append_classification (GstInferencePrediction * self, GST_INFERENCE_PREDICTION_UNLOCK (self); } +static void +compute_factors (GstVideoInfo * from, GstVideoInfo * to, gdouble * hfactor, + gdouble * vfactor) +{ + gint fw, fh, tw, th; + + g_return_if_fail (from); + g_return_if_fail (to); + g_return_if_fail (hfactor); + g_return_if_fail (vfactor); + + fw = GST_VIDEO_INFO_WIDTH (from); + tw = GST_VIDEO_INFO_WIDTH (to); + fh = GST_VIDEO_INFO_HEIGHT (from); + th = GST_VIDEO_INFO_HEIGHT (to); + + g_return_if_fail (fw); + g_return_if_fail (fh); + + *hfactor = tw * 1.0 / fw; + *vfactor = th * 1.0 / fh; +} + static GstInferencePrediction * prediction_scale (const GstInferencePrediction * self, GstVideoInfo * to, GstVideoInfo * from) { GstInferencePrediction *dest = NULL; - gint fw, fh, tw, th; + gdouble hfactor, vfactor; g_return_val_if_fail (self, NULL); @@ -454,16 +484,7 @@ prediction_scale (const GstInferencePrediction * self, GstVideoInfo * to, dest = gst_inference_prediction_new (); - fw = GST_VIDEO_INFO_WIDTH (from); - tw = GST_VIDEO_INFO_WIDTH (to); - fh = GST_VIDEO_INFO_HEIGHT (from); - th = GST_VIDEO_INFO_HEIGHT (to); - - g_return_val_if_fail (fw, FALSE); - g_return_val_if_fail (fh, FALSE); - - hfactor = tw * 1.0 / fw; - vfactor = th * 1.0 / fh; + compute_factors (from, to, &hfactor, &vfactor); dest->bbox.x = self->bbox.x * hfactor; dest->bbox.y = self->bbox.y * vfactor; @@ -479,6 +500,36 @@ prediction_scale (const GstInferencePrediction * self, GstVideoInfo * to, return dest; } +static void +prediction_scale_ip (GstInferencePrediction * self, GstVideoInfo * to, + GstVideoInfo * from) +{ + gdouble hfactor, vfactor; + + g_return_if_fail (self); + g_return_if_fail (to); + g_return_if_fail (from); + + compute_factors (from, to, &hfactor, &vfactor); + + self->bbox.x = self->bbox.x * hfactor; + self->bbox.y = self->bbox.y * vfactor; + + self->bbox.width = self->bbox.width * hfactor; + self->bbox.height = self->bbox.height * vfactor; +} + +static gboolean +node_scale_ip (GNode * node, gpointer data) +{ + GstInferencePrediction *self = (GstInferencePrediction *) node->data; + PredictionScaleData *sdata = (PredictionScaleData *) data; + + prediction_scale_ip (self, sdata->to, sdata->from); + + return FALSE; +} + static gpointer node_scale (gconstpointer node, gpointer data) { @@ -488,6 +539,24 @@ node_scale (gconstpointer node, gpointer data) return prediction_scale (self, sdata->to, sdata->from); } +void +gst_inference_prediction_scale_ip (GstInferencePrediction * self, + GstVideoInfo * to, GstVideoInfo * from) +{ + PredictionScaleData data = {.from = from,.to = to }; + + g_return_if_fail (self); + g_return_if_fail (to); + g_return_if_fail (from); + + GST_INFERENCE_PREDICTION_LOCK (self); + + g_node_traverse (self->predictions, G_IN_ORDER, G_TRAVERSE_ALL, -1, + node_scale_ip, &data); + + GST_INFERENCE_PREDICTION_UNLOCK (self); +} + GstInferencePrediction * gst_inference_prediction_scale (GstInferencePrediction * self, GstVideoInfo * to, GstVideoInfo * from) @@ -538,21 +607,33 @@ node_find (GNode * node, gpointer data) return prediction_find (current, found); } +static GstInferencePrediction * +prediction_find_unlocked (GstInferencePrediction * self, guint64 id) +{ + PredictionFindData found = { 0, id }; + + g_return_val_if_fail (self, NULL); + + g_node_traverse (self->predictions, G_IN_ORDER, G_TRAVERSE_ALL, -1, node_find, + &found); + + return found.prediction; +} + GstInferencePrediction * gst_inference_prediction_find (GstInferencePrediction * self, guint64 id) { - PredictionFindData found = { 0, id }; + GstInferencePrediction *found = NULL; g_return_val_if_fail (self, NULL); GST_INFERENCE_PREDICTION_LOCK (self); - g_node_traverse (self->predictions, G_IN_ORDER, G_TRAVERSE_ALL, -1, node_find, - &found); + found = prediction_find_unlocked (self, id); GST_INFERENCE_PREDICTION_UNLOCK (self); - return found.prediction; + return found; } static gint @@ -610,7 +691,7 @@ prediction_merge (GstInferencePrediction * src, GstInferencePrediction * dst) /* TODO: Optimize this by only searching the immediate children */ GstInferencePrediction *found = - gst_inference_prediction_find (dst, current->prediction_id); + prediction_find_unlocked (dst, current->prediction_id); /* No matching prediction, save it to append it later */ if (!found) { @@ -641,12 +722,10 @@ gst_inference_prediction_merge (GstInferencePrediction * src, g_return_if_fail (dst); GST_INFERENCE_PREDICTION_LOCK (src); - if (src != dst) - GST_INFERENCE_PREDICTION_LOCK (dst); + GST_INFERENCE_PREDICTION_LOCK (dst); prediction_merge (src, dst); - if (src != dst) - GST_INFERENCE_PREDICTION_UNLOCK (dst); + GST_INFERENCE_PREDICTION_UNLOCK (dst); GST_INFERENCE_PREDICTION_UNLOCK (src); } diff --git a/gst-libs/gst/r2inference/gstinferenceprediction.h b/gst-libs/gst/r2inference/gstinferenceprediction.h index 9ec45811..6b8f1b69 100644 --- a/gst-libs/gst/r2inference/gstinferenceprediction.h +++ b/gst-libs/gst/r2inference/gstinferenceprediction.h @@ -212,6 +212,20 @@ void gst_inference_prediction_append_classification (GstInferencePrediction * se GstInferencePrediction * gst_inference_prediction_scale (GstInferencePrediction * self, GstVideoInfo * to, GstVideoInfo * from); +/** + * gst_inference_prediction_scale_ip: + * @self: the prediction to scale in place + * @to: the resulting image size + * @from: the original image size + * + * Modifies the BoundingBox associated with this prediction (and all + * its children) to scale to the new image size. This is typically + * used by the GstMeta subsystem automatically and not for public + * usage. + */ +void gst_inference_prediction_scale_ip (GstInferencePrediction * self, + GstVideoInfo * to, GstVideoInfo * from); + /** * gst_inference_prediction_find: * @self: the root prediction diff --git a/gst-libs/gst/r2inference/gstvideoinference.c b/gst-libs/gst/r2inference/gstvideoinference.c index a36ad759..e0ec33a3 100644 --- a/gst-libs/gst/r2inference/gstvideoinference.c +++ b/gst-libs/gst/r2inference/gstvideoinference.c @@ -169,7 +169,7 @@ static void video_inference_map_buffers (GstVideoInferencePad * data, GstBuffer * inbuf, GstVideoFrame * inframe, GstVideoFrame * outframe); static gboolean video_inference_prepare_postprocess (const GstMetaInfo * meta_info, GstBuffer * buffer, GstVideoInfo * video_info, - GstVideoFrame * out_frame, GstMeta ** out_meta); + GstVideoFrame * out_frame); static void video_inference_buffer_unref (GstBuffer * buffer); static void video_inference_frame_unmap (GstBuffer * buffer, GstVideoFrame * frame); @@ -794,8 +794,7 @@ gst_video_inference_model_buffer_process (GstVideoInference * self, static gboolean video_inference_prepare_postprocess (const GstMetaInfo * meta_info, - GstBuffer * buffer, GstVideoInfo * video_info, GstVideoFrame * out_frame, - GstMeta ** out_meta) + GstBuffer * buffer, GstVideoInfo * video_info, GstVideoFrame * out_frame) { GstMapFlags flags; @@ -807,19 +806,6 @@ video_inference_prepare_postprocess (const GstMetaInfo * meta_info, return TRUE; } - if (out_meta) { - GstInferenceMeta *imeta = NULL; - - g_return_val_if_fail (gst_buffer_is_writable (buffer), FALSE); - out_meta[0] = gst_buffer_add_meta (buffer, meta_info, NULL); - out_meta[1] = - gst_buffer_add_meta (buffer, gst_inference_meta_get_info (), NULL); - - imeta = (GstInferenceMeta *) out_meta[1]; - imeta->prediction->bbox.width = video_info->width; - imeta->prediction->bbox.height = video_info->height; - } - flags = (GstMapFlags) (GST_MAP_READ | GST_VIDEO_FRAME_MAP_FLAG_NO_REF); gst_video_frame_map (out_frame, video_info, buffer, flags); @@ -909,13 +895,29 @@ gst_video_inference_postprocess (GstVideoInference * self, return TRUE; } + meta_model[1] = + gst_buffer_get_meta (buffer_model, gst_inference_meta_api_get_type ()); + if (!meta_model[1]) { + GstInferenceMeta *imeta; + + meta_model[0] = + gst_buffer_add_meta (buffer_model, klass->inference_meta_info, NULL); + meta_model[1] = + gst_buffer_add_meta (buffer_model, gst_inference_meta_get_info (), + NULL); + + imeta = (GstInferenceMeta *) meta_model[1]; + imeta->prediction->bbox.width = info_model->width; + imeta->prediction->bbox.height = info_model->height; + } + if (!video_inference_prepare_postprocess (klass->inference_meta_info, - buffer_model, info_model, &frame_model, meta_model)) { + buffer_model, info_model, &frame_model)) { return FALSE; } if (!video_inference_prepare_postprocess (klass->inference_meta_info, - buffer_bypass, info_bypass, &frame_bypass, NULL)) { + buffer_bypass, info_bypass, &frame_bypass)) { return FALSE; } @@ -943,7 +945,6 @@ gst_video_inference_postprocess (GstVideoInference * self, } else { video_inference_remove_meta (buffer_model, meta_model[0]); - video_inference_remove_meta (buffer_model, meta_model[1]); } video_inference_frame_unmap (buffer_model, &frame_model); From 0fd98109f483f1ce4b883d1d0ced2f487f8f44ee Mon Sep 17 00:00:00 2001 From: Michael Gruner Date: Tue, 21 Jan 2020 17:25:23 -0600 Subject: [PATCH 132/188] Remove meta tags to avoid meta drop Elements based on basetransform or videofilter will drop metas if they contain tags other than the video one. As such, videoscale (which our pipelines likely are going to need) was dropping the metas. We temporarily remove the tags and treat all transformations as scale transformations. --- gst-libs/gst/r2inference/gstinferencemeta.c | 18 ++++-------- gst-libs/gst/r2inference/gstvideoinference.c | 29 ++++++++++---------- 2 files changed, 20 insertions(+), 27 deletions(-) diff --git a/gst-libs/gst/r2inference/gstinferencemeta.c b/gst-libs/gst/r2inference/gstinferencemeta.c index 0aee191d..de113193 100644 --- a/gst-libs/gst/r2inference/gstinferencemeta.c +++ b/gst-libs/gst/r2inference/gstinferencemeta.c @@ -51,9 +51,7 @@ GType gst_inference_meta_api_get_type (void) { static volatile GType type = 0; - static const gchar *tags[] = - { GST_META_TAG_VIDEO_STR, GST_META_TAG_VIDEO_ORIENTATION_STR, - GST_META_TAG_VIDEO_SIZE_STR, NULL + static const gchar *tags[] = { GST_META_TAG_VIDEO_STR, NULL }; if (g_once_init_enter (&type)) { @@ -144,9 +142,7 @@ GType gst_detection_meta_api_get_type (void) { static volatile GType type = 0; - static const gchar *tags[] = - { GST_META_TAG_VIDEO_STR, GST_META_TAG_VIDEO_ORIENTATION_STR, - GST_META_TAG_VIDEO_SIZE_STR, NULL + static const gchar *tags[] = { GST_META_TAG_VIDEO_STR, NULL }; if (g_once_init_enter (&type)) { @@ -282,6 +278,7 @@ gst_inference_meta_transform_new_meta (GstBuffer * dest, GstMeta * meta, if (GST_VIDEO_META_TRANSFORM_IS_SCALE (type)) { GstVideoMetaTransform *trans = (GstVideoMetaTransform *) data; + dmeta->prediction = gst_inference_prediction_scale (smeta->prediction, trans->out_info, trans->in_info); @@ -476,12 +473,9 @@ gst_classification_meta_transform (GstBuffer * dest, GstMeta * meta, { GST_LOG ("Transforming detection metadata"); - if (GST_META_TRANSFORM_IS_COPY (type)) { - return gst_classification_meta_copy (dest, meta, buffer); - } - - /* No transform supported */ - return FALSE; + /* TODO: Eventually check for the specific transformation here. + Fail if its an unsupported transformation */ + return gst_classification_meta_copy (dest, meta, buffer); } /* inference metadata functions */ diff --git a/gst-libs/gst/r2inference/gstvideoinference.c b/gst-libs/gst/r2inference/gstvideoinference.c index e0ec33a3..6bb12bdf 100644 --- a/gst-libs/gst/r2inference/gstvideoinference.c +++ b/gst-libs/gst/r2inference/gstvideoinference.c @@ -838,6 +838,7 @@ video_inference_transform_meta (GstBuffer * buffer_model, { GstMeta *meta_bypass = NULL; const GstMetaInfo *info; + GstVideoMetaTransform trans = { info_model, info_bypass }; g_return_val_if_fail (buffer_model, NULL); g_return_val_if_fail (info_model, NULL); @@ -849,16 +850,12 @@ video_inference_transform_meta (GstBuffer * buffer_model, info = meta_model->info; - if (gst_meta_api_type_has_tag (info->api, _size_quark) || - gst_meta_api_type_has_tag (info->api, _orientation_quark)) { - GstVideoMetaTransform trans = { info_model, info_bypass }; - - info->transform_func (buffer_bypass, meta_model, buffer_model, - _scale_quark, &trans); - } else { - info->transform_func (buffer_bypass, meta_model, buffer_model, - _copy_quark, NULL); - } + /* TODO: elements such as videoscale will drop metas that have tags + such as video-size. Until this is fixed, tread all + transformations as scaling transformations + */ + info->transform_func (buffer_bypass, meta_model, buffer_model, + _scale_quark, &trans); meta_bypass = gst_buffer_get_meta (buffer_bypass, info->api); return meta_bypass; @@ -897,6 +894,9 @@ gst_video_inference_postprocess (GstVideoInference * self, meta_model[1] = gst_buffer_get_meta (buffer_model, gst_inference_meta_api_get_type ()); + meta_model[0] = + gst_buffer_get_meta (buffer_model, gst_detection_meta_api_get_type ()); + if (!meta_model[1]) { GstInferenceMeta *imeta; @@ -931,15 +931,10 @@ gst_video_inference_postprocess (GstVideoInference * self, if (pred_valid) { GstVideoFrame *pbpass = buffer_bypass ? &frame_bypass : NULL; - meta_bypass[0] = video_inference_transform_meta (buffer_model, info_model, meta_model[0], buffer_bypass, info_bypass); - meta_bypass[1] = - video_inference_transform_meta (buffer_model, info_model, meta_model[1], - buffer_bypass, info_bypass); - g_signal_emit (self, gst_video_inference_signals[NEW_PREDICTION_SIGNAL], 0, meta_model[0], &frame_model, meta_bypass[0], pbpass); @@ -947,6 +942,10 @@ gst_video_inference_postprocess (GstVideoInference * self, video_inference_remove_meta (buffer_model, meta_model[0]); } + meta_bypass[1] = + video_inference_transform_meta (buffer_model, info_model, meta_model[1], + buffer_bypass, info_bypass); + video_inference_frame_unmap (buffer_model, &frame_model); video_inference_frame_unmap (buffer_bypass, &frame_bypass); From 4fdaf3c37849ea31a798daae1a3f70d5136fcb74 Mon Sep 17 00:00:00 2001 From: Michael Gruner Date: Tue, 21 Jan 2020 17:57:15 -0600 Subject: [PATCH 133/188] Install old-style metas independently --- gst-libs/gst/r2inference/gstvideoinference.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/gst-libs/gst/r2inference/gstvideoinference.c b/gst-libs/gst/r2inference/gstvideoinference.c index 6bb12bdf..df38c360 100644 --- a/gst-libs/gst/r2inference/gstvideoinference.c +++ b/gst-libs/gst/r2inference/gstvideoinference.c @@ -892,16 +892,18 @@ gst_video_inference_postprocess (GstVideoInference * self, return TRUE; } - meta_model[1] = - gst_buffer_get_meta (buffer_model, gst_inference_meta_api_get_type ()); meta_model[0] = gst_buffer_get_meta (buffer_model, gst_detection_meta_api_get_type ()); + if (!meta_model[0]) { + meta_model[0] = + gst_buffer_add_meta (buffer_model, klass->inference_meta_info, NULL); + } + meta_model[1] = + gst_buffer_get_meta (buffer_model, gst_inference_meta_api_get_type ()); if (!meta_model[1]) { GstInferenceMeta *imeta; - meta_model[0] = - gst_buffer_add_meta (buffer_model, klass->inference_meta_info, NULL); meta_model[1] = gst_buffer_add_meta (buffer_model, gst_inference_meta_get_info (), NULL); From f96f58ae9f0c6508bdd077e0e459eed764559bc4 Mon Sep 17 00:00:00 2001 From: Michael Gruner Date: Wed, 22 Jan 2020 14:59:01 -0600 Subject: [PATCH 134/188] Avoid potential deadlocks on prediction merges --- gst-libs/gst/r2inference/gstinferenceprediction.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/gst-libs/gst/r2inference/gstinferenceprediction.c b/gst-libs/gst/r2inference/gstinferenceprediction.c index cf50e218..c4ec5b0b 100644 --- a/gst-libs/gst/r2inference/gstinferenceprediction.c +++ b/gst-libs/gst/r2inference/gstinferenceprediction.c @@ -721,6 +721,10 @@ gst_inference_prediction_merge (GstInferencePrediction * src, g_return_if_fail (src); g_return_if_fail (dst); + if (src == dst) { + return; + } + GST_INFERENCE_PREDICTION_LOCK (src); GST_INFERENCE_PREDICTION_LOCK (dst); From eccebbfdcba93be4fa1ca1a8b9d1111c12d96bf4 Mon Sep 17 00:00:00 2001 From: Michael Gruner Date: Wed, 22 Jan 2020 15:00:06 -0600 Subject: [PATCH 135/188] Preserve prediction ID when scaling off-place --- gst-libs/gst/r2inference/gstinferenceprediction.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gst-libs/gst/r2inference/gstinferenceprediction.c b/gst-libs/gst/r2inference/gstinferenceprediction.c index c4ec5b0b..7ff87f57 100644 --- a/gst-libs/gst/r2inference/gstinferenceprediction.c +++ b/gst-libs/gst/r2inference/gstinferenceprediction.c @@ -482,7 +482,7 @@ prediction_scale (const GstInferencePrediction * self, GstVideoInfo * to, g_return_val_if_fail (to, NULL); g_return_val_if_fail (from, NULL); - dest = gst_inference_prediction_new (); + dest = prediction_copy (self); compute_factors (from, to, &hfactor, &vfactor); From 4171b9b371c636ab86c5dfef90b8726c95f16bd8 Mon Sep 17 00:00:00 2001 From: Carlos Rodriguez Date: Wed, 22 Jan 2020 17:42:44 -0600 Subject: [PATCH 136/188] Remove double function definition --- gst-libs/gst/r2inference/gstinferenceprediction.h | 9 --------- 1 file changed, 9 deletions(-) diff --git a/gst-libs/gst/r2inference/gstinferenceprediction.h b/gst-libs/gst/r2inference/gstinferenceprediction.h index 6b8f1b69..bc1853ab 100644 --- a/gst-libs/gst/r2inference/gstinferenceprediction.h +++ b/gst-libs/gst/r2inference/gstinferenceprediction.h @@ -101,15 +101,6 @@ GstInferencePrediction * gst_inference_prediction_new (void); */ GstInferencePrediction * gst_inference_prediction_new_full (BoundingBox *bbox); -/** - * gst_inference_prediction_new_full: - * - * Creates a new GstInferencePrediction. - * - * Returns: A newly allocated and initialized GstInferencePrediction. - */ -GstInferencePrediction * gst_inference_prediction_new (void); - /** * gst_inference_prediction_reset: * @self: the prediction to reset From db9d841f7fd05fde802cdde36586578c01dba012 Mon Sep 17 00:00:00 2001 From: Michael Gruner Date: Fri, 24 Jan 2020 08:39:31 -0600 Subject: [PATCH 137/188] Free classification and prediction mutex --- gst-libs/gst/r2inference/gstinferenceclassification.c | 2 ++ gst-libs/gst/r2inference/gstinferenceprediction.c | 2 ++ 2 files changed, 4 insertions(+) diff --git a/gst-libs/gst/r2inference/gstinferenceclassification.c b/gst-libs/gst/r2inference/gstinferenceclassification.c index b6d3d022..0b26e853 100644 --- a/gst-libs/gst/r2inference/gstinferenceclassification.c +++ b/gst-libs/gst/r2inference/gstinferenceclassification.c @@ -232,4 +232,6 @@ static void classification_free (GstInferenceClassification * self) { classification_reset (self); + + g_mutex_clear (&self->mutex); } diff --git a/gst-libs/gst/r2inference/gstinferenceprediction.c b/gst-libs/gst/r2inference/gstinferenceprediction.c index 7ff87f57..18249fb9 100644 --- a/gst-libs/gst/r2inference/gstinferenceprediction.c +++ b/gst-libs/gst/r2inference/gstinferenceprediction.c @@ -433,6 +433,8 @@ prediction_free (GstInferencePrediction * self) g_node_destroy (self->predictions); self->predictions = NULL; } + + g_mutex_clear (&self->mutex); } void From a3d66c8507e7aac90e7cc823a1102ae2be1fa1c5 Mon Sep 17 00:00:00 2001 From: rrcarlosrodriguez Date: Fri, 24 Jan 2020 10:31:30 -0600 Subject: [PATCH 138/188] Add new Inferencedebug element --- configure.ac | 1 + gst/Makefile.am | 5 +- gst/inferencedebug/Makefile.am | 27 ++++ gst/inferencedebug/gstinferencedebug.c | 166 +++++++++++++++++++++++++ gst/inferencedebug/gstinferencedebug.h | 33 +++++ 5 files changed, 231 insertions(+), 1 deletion(-) create mode 100644 gst/inferencedebug/Makefile.am create mode 100644 gst/inferencedebug/gstinferencedebug.c create mode 100644 gst/inferencedebug/gstinferencedebug.h diff --git a/configure.ac b/configure.ac index 99cd9fde..f763d3c5 100644 --- a/configure.ac +++ b/configure.ac @@ -383,6 +383,7 @@ ext/r2inference/Makefile gst/Makefile gst/inferencecrop/Makefile gst/inferencefilter/Makefile +gst/inferencedebug/Makefile gst-libs/Makefile gst-libs/gst/Makefile gst-libs/gst/opencv/Makefile diff --git a/gst/Makefile.am b/gst/Makefile.am index c765c2e8..1c7fd945 100644 --- a/gst/Makefile.am +++ b/gst/Makefile.am @@ -1 +1,4 @@ -SUBDIRS = inferencecrop inferencefilter +SUBDIRS = \ + inferencecrop \ + inferencefilter \ + inferencedebug diff --git a/gst/inferencedebug/Makefile.am b/gst/inferencedebug/Makefile.am new file mode 100644 index 00000000..3f9c28a3 --- /dev/null +++ b/gst/inferencedebug/Makefile.am @@ -0,0 +1,27 @@ +plugin_LTLIBRARIES = libgstinferencedebug.la + +libgstinferencedebug_la_SOURCES = \ + gstinferencedebug.c + +libgstinferencedebug_la_CFLAGS = \ + $(GST_CFLAGS) \ + $(GST_BASE_CFLAGS) \ + $(GST_PLUGINS_BASE_CFLAGS) \ + -I$(top_srcdir)/gst-libs \ + $(R2INFERENCE_CFLAGS) + +libgstinferencedebug_la_LIBADD = \ + $(GST_LIBS) \ + $(GST_BASE_LIBS) \ + $(GST_PLUGINS_BASE_LIBS) \ + $(R2INFERENCE_LIBS) \ + $(top_builddir)/gst-libs/gst/r2inference/libgstinference-@GST_API_VERSION@.la + +libgstinferencedebug_la_LDFLAGS = \ +$(GST_PLUGIN_LDFLAGS) + +libgstinferencedebug_la_LIBTOOLFLAGS = \ + $(GST_PLUGIN_LIBTOOLFLAGS) + +noinst_HEADERS = \ + gstinferencedebug.h diff --git a/gst/inferencedebug/gstinferencedebug.c b/gst/inferencedebug/gstinferencedebug.c new file mode 100644 index 00000000..34e01b91 --- /dev/null +++ b/gst/inferencedebug/gstinferencedebug.c @@ -0,0 +1,166 @@ +/* + * GStreamer + * Copyright (C) 2018-2020 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + */ + +/** + * SECTION:element-gstinferencedebug + * + * The inferencedebug element prints the inferencemeta predictions tree + * + * + * Example launch line + * |[ + * gst-launch-1.0 v4l2src device=$CAMERA ! "video/x-raw, width=1280, height=720" ! videoconvert ! tee name=t \ + t. ! videoscale ! queue ! net.sink_model t. ! queue ! net.sink_bypass \ + tinyyolov2 name=net model-location=$MODEL_LOCATION backend=tensorflow backend::input-layer=$INPUT_LAYER \ + backend::output-layer=$OUTPUT_LAYER net.src_model ! inferencedebug ! fakesink + * ]| + * + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "gstinferencedebug.h" + +#include +#include + + +GST_DEBUG_CATEGORY_STATIC (gst_inferencedebug_debug_category); +#define GST_CAT_DEFAULT gst_inferencedebug_debug_category + +/* prototypes */ + +static void gst_inferencedebug_print_predictions (GstInferencedebug * + inferencedebug, GstInferencePrediction * root); +static GstFlowReturn gst_inferencedebug_transform_ip (GstBaseTransform * trans, + GstBuffer * buf); + +struct _GstInferencedebug +{ + GstBaseTransform base_inferencedebug; +}; + + +/* pad templates */ + +static GstStaticPadTemplate gst_inferencedebug_src_template = +GST_STATIC_PAD_TEMPLATE ("src", + GST_PAD_SRC, + GST_PAD_ALWAYS, + GST_STATIC_CAPS_ANY); + +static GstStaticPadTemplate gst_inferencedebug_sink_template = +GST_STATIC_PAD_TEMPLATE ("sink", + GST_PAD_SINK, + GST_PAD_ALWAYS, + GST_STATIC_CAPS_ANY); + + +/* class initialization */ + +G_DEFINE_TYPE_WITH_CODE (GstInferencedebug, gst_inferencedebug, + GST_TYPE_BASE_TRANSFORM, + GST_DEBUG_CATEGORY_INIT (gst_inferencedebug_debug_category, + "inferencedebug", 0, "debug category for inferencedebug element")); + +static void +gst_inferencedebug_class_init (GstInferencedebugClass * klass) +{ + GstBaseTransformClass *base_transform_class = + GST_BASE_TRANSFORM_CLASS (klass); + + /* Setting up pads and setting metadata should be moved to + base_class_init if you intend to subclass this class. */ + gst_element_class_add_static_pad_template (GST_ELEMENT_CLASS (klass), + &gst_inferencedebug_src_template); + gst_element_class_add_static_pad_template (GST_ELEMENT_CLASS (klass), + &gst_inferencedebug_sink_template); + + gst_element_class_set_static_metadata (GST_ELEMENT_CLASS (klass), + "Inference Debug", "Generic", + "Prints InferenceMeta Predictions Tree", + ""); + + base_transform_class->transform_ip = + GST_DEBUG_FUNCPTR (gst_inferencedebug_transform_ip); +} + +static void +gst_inferencedebug_init (GstInferencedebug * inferencedebug) +{ +} + + +static void +gst_inferencedebug_print_predictions (GstInferencedebug * inferencedebug, + GstInferencePrediction * root) +{ + gchar *prediction_tree = NULL; + + g_return_if_fail (inferencedebug); + g_return_if_fail (root); + + prediction_tree = gst_inference_prediction_to_string (root); + + GST_DEBUG_OBJECT (inferencedebug, "Prediction Tree: \n %s", prediction_tree); + + g_free (prediction_tree); +} + +static GstFlowReturn +gst_inferencedebug_transform_ip (GstBaseTransform * trans, GstBuffer * buf) +{ + GstInferencedebug *inferencedebug = GST_INFERENCEDEBUG (trans); + GstInferenceMeta *meta; + + GST_DEBUG_OBJECT (inferencedebug, "transform_ip"); + + meta = (GstInferenceMeta *) gst_buffer_get_meta (buf, + GST_INFERENCE_META_API_TYPE); + + if (NULL == meta) { + GST_LOG_OBJECT (inferencedebug, + "No inference meta found. Buffer passthrough."); + return GST_FLOW_OK; + } + + g_return_val_if_fail (meta->prediction, GST_FLOW_ERROR); + + gst_inferencedebug_print_predictions (inferencedebug, meta->prediction); + + return GST_FLOW_OK; +} + +static gboolean +plugin_init (GstPlugin * plugin) +{ + return gst_element_register (plugin, "inferencedebug", GST_RANK_NONE, + GST_TYPE_INFERENCEDEBUG); +} + + +GST_PLUGIN_DEFINE (GST_VERSION_MAJOR, + GST_VERSION_MINOR, + inferencedebug, + "Print InferenceMeta Predictions", + plugin_init, VERSION, "LGPL", PACKAGE_NAME, GST_PACKAGE_ORIGIN) diff --git a/gst/inferencedebug/gstinferencedebug.h b/gst/inferencedebug/gstinferencedebug.h new file mode 100644 index 00000000..055250fc --- /dev/null +++ b/gst/inferencedebug/gstinferencedebug.h @@ -0,0 +1,33 @@ +/* + * GStreamer + * Copyright (C) 2018-2020 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + */ + +#ifndef _GST_INFERENCEDEBUG_H_ +#define _GST_INFERENCEDEBUG_H_ + +#include + +G_BEGIN_DECLS +#define GST_TYPE_INFERENCEDEBUG (gst_inferencedebug_get_type()) +G_DECLARE_FINAL_TYPE (GstInferencedebug, gst_inferencedebug, GST, + INFERENCEDEBUG, GstBaseTransform) + +G_END_DECLS +#endif From dbb5f907a460635e280554ac30f59e934f1ae8f6 Mon Sep 17 00:00:00 2001 From: rrcarlosrodriguez Date: Fri, 24 Jan 2020 10:33:01 -0600 Subject: [PATCH 139/188] Fix inferencefilter header --- gst/inferencefilter/gstinferencefilter.c | 2 +- gst/inferencefilter/gstinferencefilter.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/gst/inferencefilter/gstinferencefilter.c b/gst/inferencefilter/gstinferencefilter.c index e8b7ad5b..03607875 100644 --- a/gst/inferencefilter/gstinferencefilter.c +++ b/gst/inferencefilter/gstinferencefilter.c @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2020 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public diff --git a/gst/inferencefilter/gstinferencefilter.h b/gst/inferencefilter/gstinferencefilter.h index 597e9dc5..85005487 100644 --- a/gst/inferencefilter/gstinferencefilter.h +++ b/gst/inferencefilter/gstinferencefilter.h @@ -1,6 +1,6 @@ /* * GStreamer - * Copyright (C) 2020 RidgeRun + * Copyright (C) 2018-2020 RidgeRun * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public From 20b4f6fcc034ca166c18d848516d2518f7550ee1 Mon Sep 17 00:00:00 2001 From: rrcarlosrodriguez Date: Fri, 24 Jan 2020 10:52:36 -0600 Subject: [PATCH 140/188] Change inferencedebug naming format --- gst/inferencedebug/gstinferencedebug.c | 46 +++++++++++--------------- gst/inferencedebug/gstinferencedebug.h | 10 +++--- 2 files changed, 25 insertions(+), 31 deletions(-) diff --git a/gst/inferencedebug/gstinferencedebug.c b/gst/inferencedebug/gstinferencedebug.c index 34e01b91..60559d5a 100644 --- a/gst/inferencedebug/gstinferencedebug.c +++ b/gst/inferencedebug/gstinferencedebug.c @@ -42,49 +42,45 @@ #include "gstinferencedebug.h" #include -#include - -GST_DEBUG_CATEGORY_STATIC (gst_inferencedebug_debug_category); -#define GST_CAT_DEFAULT gst_inferencedebug_debug_category +GST_DEBUG_CATEGORY_STATIC (gst_inference_debug_debug_category); +#define GST_CAT_DEFAULT gst_inference_debug_debug_category /* prototypes */ -static void gst_inferencedebug_print_predictions (GstInferencedebug * +static void gst_inference_debug_print_predictions (GstInferenceDebug * inferencedebug, GstInferencePrediction * root); -static GstFlowReturn gst_inferencedebug_transform_ip (GstBaseTransform * trans, +static GstFlowReturn gst_inference_debug_transform_ip (GstBaseTransform * trans, GstBuffer * buf); -struct _GstInferencedebug +struct _GstInferenceDebug { - GstBaseTransform base_inferencedebug; + GstBaseTransform base_inference_debug; }; - /* pad templates */ -static GstStaticPadTemplate gst_inferencedebug_src_template = +static GstStaticPadTemplate gst_inference_debug_src_template = GST_STATIC_PAD_TEMPLATE ("src", GST_PAD_SRC, GST_PAD_ALWAYS, GST_STATIC_CAPS_ANY); -static GstStaticPadTemplate gst_inferencedebug_sink_template = +static GstStaticPadTemplate gst_inference_debug_sink_template = GST_STATIC_PAD_TEMPLATE ("sink", GST_PAD_SINK, GST_PAD_ALWAYS, GST_STATIC_CAPS_ANY); - /* class initialization */ -G_DEFINE_TYPE_WITH_CODE (GstInferencedebug, gst_inferencedebug, +G_DEFINE_TYPE_WITH_CODE (GstInferenceDebug, gst_inference_debug, GST_TYPE_BASE_TRANSFORM, - GST_DEBUG_CATEGORY_INIT (gst_inferencedebug_debug_category, + GST_DEBUG_CATEGORY_INIT (gst_inference_debug_debug_category, "inferencedebug", 0, "debug category for inferencedebug element")); static void -gst_inferencedebug_class_init (GstInferencedebugClass * klass) +gst_inference_debug_class_init (GstInferenceDebugClass * klass) { GstBaseTransformClass *base_transform_class = GST_BASE_TRANSFORM_CLASS (klass); @@ -92,9 +88,9 @@ gst_inferencedebug_class_init (GstInferencedebugClass * klass) /* Setting up pads and setting metadata should be moved to base_class_init if you intend to subclass this class. */ gst_element_class_add_static_pad_template (GST_ELEMENT_CLASS (klass), - &gst_inferencedebug_src_template); + &gst_inference_debug_src_template); gst_element_class_add_static_pad_template (GST_ELEMENT_CLASS (klass), - &gst_inferencedebug_sink_template); + &gst_inference_debug_sink_template); gst_element_class_set_static_metadata (GST_ELEMENT_CLASS (klass), "Inference Debug", "Generic", @@ -102,17 +98,16 @@ gst_inferencedebug_class_init (GstInferencedebugClass * klass) ""); base_transform_class->transform_ip = - GST_DEBUG_FUNCPTR (gst_inferencedebug_transform_ip); + GST_DEBUG_FUNCPTR (gst_inference_debug_transform_ip); } static void -gst_inferencedebug_init (GstInferencedebug * inferencedebug) +gst_inference_debug_init (GstInferenceDebug * inferencedebug) { } - static void -gst_inferencedebug_print_predictions (GstInferencedebug * inferencedebug, +gst_inference_debug_print_predictions (GstInferenceDebug * inferencedebug, GstInferencePrediction * root) { gchar *prediction_tree = NULL; @@ -128,9 +123,9 @@ gst_inferencedebug_print_predictions (GstInferencedebug * inferencedebug, } static GstFlowReturn -gst_inferencedebug_transform_ip (GstBaseTransform * trans, GstBuffer * buf) +gst_inference_debug_transform_ip (GstBaseTransform * trans, GstBuffer * buf) { - GstInferencedebug *inferencedebug = GST_INFERENCEDEBUG (trans); + GstInferenceDebug *inferencedebug = GST_INFERENCE_DEBUG (trans); GstInferenceMeta *meta; GST_DEBUG_OBJECT (inferencedebug, "transform_ip"); @@ -146,7 +141,7 @@ gst_inferencedebug_transform_ip (GstBaseTransform * trans, GstBuffer * buf) g_return_val_if_fail (meta->prediction, GST_FLOW_ERROR); - gst_inferencedebug_print_predictions (inferencedebug, meta->prediction); + gst_inference_debug_print_predictions (inferencedebug, meta->prediction); return GST_FLOW_OK; } @@ -155,10 +150,9 @@ static gboolean plugin_init (GstPlugin * plugin) { return gst_element_register (plugin, "inferencedebug", GST_RANK_NONE, - GST_TYPE_INFERENCEDEBUG); + GST_TYPE_INFERENCE_DEBUG); } - GST_PLUGIN_DEFINE (GST_VERSION_MAJOR, GST_VERSION_MINOR, inferencedebug, diff --git a/gst/inferencedebug/gstinferencedebug.h b/gst/inferencedebug/gstinferencedebug.h index 055250fc..5899e850 100644 --- a/gst/inferencedebug/gstinferencedebug.h +++ b/gst/inferencedebug/gstinferencedebug.h @@ -19,15 +19,15 @@ * */ -#ifndef _GST_INFERENCEDEBUG_H_ -#define _GST_INFERENCEDEBUG_H_ +#ifndef _GST_INFERENCE_DEBUG_H_ +#define _GST_INFERENCE_DEBUG_H_ #include G_BEGIN_DECLS -#define GST_TYPE_INFERENCEDEBUG (gst_inferencedebug_get_type()) -G_DECLARE_FINAL_TYPE (GstInferencedebug, gst_inferencedebug, GST, - INFERENCEDEBUG, GstBaseTransform) +#define GST_TYPE_INFERENCE_DEBUG (gst_inference_debug_get_type()) +G_DECLARE_FINAL_TYPE (GstInferenceDebug, gst_inference_debug, GST, + INFERENCE_DEBUG, GstBaseTransform) G_END_DECLS #endif From cc78ec25c4b92220450d6b3a38adbd1fc5b33a53 Mon Sep 17 00:00:00 2001 From: c-trejos Date: Fri, 24 Jan 2020 09:15:51 -0600 Subject: [PATCH 141/188] Remove labels pointer guard --- ext/opencv/gstinferenceoverlay.cc | 2 -- 1 file changed, 2 deletions(-) diff --git a/ext/opencv/gstinferenceoverlay.cc b/ext/opencv/gstinferenceoverlay.cc index eb1a33a6..97ec3669 100644 --- a/ext/opencv/gstinferenceoverlay.cc +++ b/ext/opencv/gstinferenceoverlay.cc @@ -155,7 +155,6 @@ gst_get_meta (GstInferencePrediction *pred, cv::Mat &cv_mat, gdouble font_scale, cv::Mat alpha_overlay; gint width, height,x, y = 0; g_return_if_fail (pred != NULL); - g_return_if_fail (labels_list != NULL); list = gst_inference_prediction_get_children(pred); @@ -256,7 +255,6 @@ gst_inference_overlay_process_meta (GstInferenceBaseOverlay *inference_overlay, g_return_val_if_fail (inference_overlay != NULL ,GST_FLOW_ERROR); g_return_val_if_fail (frame != NULL ,GST_FLOW_ERROR); g_return_val_if_fail (meta != NULL ,GST_FLOW_ERROR); - g_return_val_if_fail (labels_list != NULL ,GST_FLOW_ERROR); switch (GST_VIDEO_FRAME_FORMAT (frame)) { case GST_VIDEO_FORMAT_RGB: From ddb0cd005a458afb71bbbb886b439d3bb1653e14 Mon Sep 17 00:00:00 2001 From: c-trejos Date: Fri, 24 Jan 2020 10:02:39 -0600 Subject: [PATCH 142/188] Override labels property on inferenceoverlay --- ext/opencv/gstinferenceoverlay.cc | 58 ++++++++++++++++++++++++++++++- 1 file changed, 57 insertions(+), 1 deletion(-) diff --git a/ext/opencv/gstinferenceoverlay.cc b/ext/opencv/gstinferenceoverlay.cc index 97ec3669..4494f36d 100644 --- a/ext/opencv/gstinferenceoverlay.cc +++ b/ext/opencv/gstinferenceoverlay.cc @@ -32,6 +32,14 @@ #include "opencv2/highgui.hpp" #endif +#define DEFAULT_LABELS NULL + +enum +{ + PROP_0, + PROP_LABELS +}; + static const cv::Scalar colors[] = { @@ -58,6 +66,10 @@ GST_DEBUG_CATEGORY_STATIC (gst_inference_overlay_debug_category); #define GST_CAT_DEFAULT gst_inference_overlay_debug_category /* prototypes */ +static void gst_inference_overlay_set_property (GObject * object, + guint property_id, const GValue * value, GParamSpec * pspec); +static void gst_inference_overlay_get_property (GObject * object, + guint property_id, GValue * value, GParamSpec * pspec); static GstFlowReturn gst_inference_overlay_process_meta (GstInferenceBaseOverlay *inference_overlay, GstVideoFrame *frame, GstMeta *meta, gdouble font_scale, gint thickness, @@ -86,11 +98,19 @@ static void gst_inference_overlay_class_init (GstInferenceOverlayClass *klass) { GstInferenceBaseOverlayClass * io_class = GST_INFERENCE_BASE_OVERLAY_CLASS (klass); - + GObjectClass *gobject_class = G_OBJECT_CLASS (klass); + + gobject_class->set_property = gst_inference_overlay_set_property; + gobject_class->get_property = gst_inference_overlay_get_property; + gst_element_class_set_static_metadata (GST_ELEMENT_CLASS (klass), "inferenceoverlay", "Filter", "Overlays Inferece metadata on input buffer", "Lenin Torres "); + g_object_class_install_property (gobject_class, PROP_LABELS, + g_param_spec_string ("labels", "labels", + "(Deprecated) Semicolon separated string containing inference labels.", + DEFAULT_LABELS, G_PARAM_READWRITE)); io_class->process_meta = GST_DEBUG_FUNCPTR (gst_inference_overlay_process_meta); @@ -101,6 +121,42 @@ static void gst_inference_overlay_init (GstInferenceOverlay *inference_overlay) { } +void +gst_inference_overlay_set_property (GObject * object, guint property_id, + const GValue * value, GParamSpec * pspec) +{ + GstInferenceOverlay *inference_overlay = + GST_INFERENCE_OVERLAY (object); + + GST_DEBUG_OBJECT (inference_overlay, "set_property"); + + switch (property_id) { + case PROP_LABELS: + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); + break; + } +} + +void +gst_inference_overlay_get_property (GObject * object, guint property_id, + GValue * value, GParamSpec * pspec) +{ + GstInferenceOverlay *inference_overlay = + GST_INFERENCE_OVERLAY (object); + + GST_DEBUG_OBJECT (inference_overlay, "get_property"); + + switch (property_id) { + case PROP_LABELS: + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); + break; + } +} + void draw_line(cv::Mat &img, cv::Point pt1, cv::Point pt2, cv::Scalar color, gint thickness, LineStyleBoundingBox style, gint gap) { From ebb230c9813afbfab8567c9195345210b60d43ec Mon Sep 17 00:00:00 2001 From: Michael Gruner Date: Fri, 24 Jan 2020 11:46:14 -0600 Subject: [PATCH 143/188] Fix lock on cleared mutex --- gst-libs/gst/r2inference/gstinferenceprediction.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/gst-libs/gst/r2inference/gstinferenceprediction.c b/gst-libs/gst/r2inference/gstinferenceprediction.c index 18249fb9..cdb66d90 100644 --- a/gst-libs/gst/r2inference/gstinferenceprediction.c +++ b/gst-libs/gst/r2inference/gstinferenceprediction.c @@ -38,6 +38,7 @@ struct _PredictionFindData guint64 prediction_id; }; +static void gst_inference_prediction_free (GstInferencePrediction * self); static GstInferencePrediction *prediction_copy (const GstInferencePrediction * self); static void prediction_free (GstInferencePrediction * obj); @@ -100,7 +101,7 @@ gst_inference_prediction_new (void) gst_mini_object_init (GST_MINI_OBJECT_CAST (self), 0, gst_inference_prediction_get_type (), (GstMiniObjectCopyFunction) gst_inference_prediction_copy, NULL, - (GstMiniObjectFreeFunction) prediction_free); + (GstMiniObjectFreeFunction) gst_inference_prediction_free); g_mutex_init (&self->mutex); @@ -433,6 +434,14 @@ prediction_free (GstInferencePrediction * self) g_node_destroy (self->predictions); self->predictions = NULL; } +} + +static void +gst_inference_prediction_free (GstInferencePrediction * self) +{ + g_return_if_fail (self); + + prediction_free (self); g_mutex_clear (&self->mutex); } From 5c4866db51e3e398ef3bedcfd2645340182a29f4 Mon Sep 17 00:00:00 2001 From: rrcarlosrodriguez Date: Fri, 24 Jan 2020 12:09:30 -0600 Subject: [PATCH 144/188] Set BoundingBox X,Y coordinates as signed numbers --- gst-libs/gst/r2inference/gstinferenceprediction.c | 4 ++-- gst-libs/gst/r2inference/gstinferenceprediction.h | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/gst-libs/gst/r2inference/gstinferenceprediction.c b/gst-libs/gst/r2inference/gstinferenceprediction.c index cdb66d90..38ed10e0 100644 --- a/gst-libs/gst/r2inference/gstinferenceprediction.c +++ b/gst-libs/gst/r2inference/gstinferenceprediction.c @@ -236,8 +236,8 @@ bounding_box_to_string (BoundingBox * bbox, gint level) g_return_val_if_fail (bbox, NULL); return g_strdup_printf ("{\n" - "%*s x : %u\n" - "%*s y : %u\n" + "%*s x : %d\n" + "%*s y : %d\n" "%*s width : %u\n" "%*s height : %u\n" "%*s}", diff --git a/gst-libs/gst/r2inference/gstinferenceprediction.h b/gst-libs/gst/r2inference/gstinferenceprediction.h index bc1853ab..1e33847d 100644 --- a/gst-libs/gst/r2inference/gstinferenceprediction.h +++ b/gst-libs/gst/r2inference/gstinferenceprediction.h @@ -43,8 +43,8 @@ typedef struct _GstInferencePrediction GstInferencePrediction; */ struct _BoundingBox { - guint x; - guint y; + gint x; + gint y; guint width; guint height; }; From 450d170d6b9d1b57a1fca91576d4a747ced47442 Mon Sep 17 00:00:00 2001 From: rrcarlosrodriguez Date: Fri, 24 Jan 2020 14:58:25 -0600 Subject: [PATCH 145/188] Overwrite prediction in detectioncrop element --- gst/inferencecrop/gstdetectioncrop.cc | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/gst/inferencecrop/gstdetectioncrop.cc b/gst/inferencecrop/gstdetectioncrop.cc index bc267478..6505509f 100644 --- a/gst/inferencecrop/gstdetectioncrop.cc +++ b/gst/inferencecrop/gstdetectioncrop.cc @@ -428,9 +428,11 @@ gst_detection_crop_new_buffer (GstPad *pad, GstPadProbeInfo *info, (gint) left); croped_buffer = gst_buffer_copy (buffer); - dmeta = (GstInferenceMeta *) gst_buffer_add_meta (croped_buffer, - GST_INFERENCE_META_INFO, - NULL); + + dmeta = (GstInferenceMeta *) gst_buffer_get_meta (croped_buffer, + GST_INFERENCE_META_API_TYPE); + + gst_inference_prediction_unref (dmeta->prediction); dmeta->prediction = gst_inference_prediction_copy (pred); From 811fd231919bf781930b87efb192c02903b42358 Mon Sep 17 00:00:00 2001 From: rrcarlosrodriguez Date: Fri, 24 Jan 2020 15:34:16 -0600 Subject: [PATCH 146/188] Ignore overlay if the prediction is disabled --- ext/opencv/gstinferenceoverlay.cc | 6 ++++++ gst-libs/gst/r2inference/gstinferenceprediction.c | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/ext/opencv/gstinferenceoverlay.cc b/ext/opencv/gstinferenceoverlay.cc index 4494f36d..04754f43 100644 --- a/ext/opencv/gstinferenceoverlay.cc +++ b/ext/opencv/gstinferenceoverlay.cc @@ -220,6 +220,12 @@ gst_get_meta (GstInferencePrediction *pred, cv::Mat &cv_mat, gdouble font_scale, gst_get_meta (predict, cv_mat, font_scale, thickness, labels_list, num_labels, style); } + + if (!pred->enabled) { + /* Ignore overlay if the prediction is disabled */ + return; + } + box = pred->bbox; for (iter = pred->classifications; iter != NULL; iter = g_list_next(iter)) { diff --git a/gst-libs/gst/r2inference/gstinferenceprediction.c b/gst-libs/gst/r2inference/gstinferenceprediction.c index 38ed10e0..ac216e36 100644 --- a/gst-libs/gst/r2inference/gstinferenceprediction.c +++ b/gst-libs/gst/r2inference/gstinferenceprediction.c @@ -404,7 +404,7 @@ prediction_reset (GstInferencePrediction * self) g_return_if_fail (self); self->prediction_id = get_new_id (); - self->enabled = FALSE; + self->enabled = TRUE; bounding_box_reset (&self->bbox); From 16e56e7a34e6ef7c4d24aa1a164b8aecd58d3115 Mon Sep 17 00:00:00 2001 From: rrcarlosrodriguez Date: Fri, 24 Jan 2020 15:54:30 -0600 Subject: [PATCH 147/188] Free prediction children list --- ext/opencv/gstinferenceoverlay.cc | 3 +++ gst/inferencefilter/gstinferencefilter.c | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/ext/opencv/gstinferenceoverlay.cc b/ext/opencv/gstinferenceoverlay.cc index 04754f43..c6a3f77f 100644 --- a/ext/opencv/gstinferenceoverlay.cc +++ b/ext/opencv/gstinferenceoverlay.cc @@ -302,6 +302,9 @@ gst_get_meta (GstInferencePrediction *pred, cv::Mat &cv_mat, gdouble font_scale, } } + if (NULL != list) { + g_slist_free (list); + } } static diff --git a/gst/inferencefilter/gstinferencefilter.c b/gst/inferencefilter/gstinferencefilter.c index 03607875..b1e7c1ea 100644 --- a/gst/inferencefilter/gstinferencefilter.c +++ b/gst/inferencefilter/gstinferencefilter.c @@ -227,6 +227,10 @@ gst_inferencefilter_filter_enable (GstInferencefilter * inferencefilter, gst_inferencefilter_filter_enable (inferencefilter, predict, class_id, reset); } + + if (NULL != iter_child) { + g_slist_free (iter_child); + } } static GstFlowReturn From f4934788414ca1328754eb9fc77a37774953a953 Mon Sep 17 00:00:00 2001 From: rrcarlosrodriguez Date: Fri, 24 Jan 2020 17:18:52 -0600 Subject: [PATCH 148/188] Disable predictions without a class in the inferencefilter --- gst/inferencefilter/gstinferencefilter.c | 28 ++++++++++++++---------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/gst/inferencefilter/gstinferencefilter.c b/gst/inferencefilter/gstinferencefilter.c index b1e7c1ea..5b821088 100644 --- a/gst/inferencefilter/gstinferencefilter.c +++ b/gst/inferencefilter/gstinferencefilter.c @@ -205,18 +205,22 @@ gst_inferencefilter_filter_enable (GstInferencefilter * inferencefilter, g_return_if_fail (inferencefilter); g_return_if_fail (root); - for (iter = root->classifications; iter != NULL; iter = g_list_next (iter)) { - GstInferenceClassification *classification = - (GstInferenceClassification *) iter->data; - if (classification->class_id == class_id || reset) { - GST_DEBUG_OBJECT (inferencefilter, "Enabling classification id %d", - classification->class_id); - root->enabled = TRUE; - break; - } else { - GST_DEBUG_OBJECT (inferencefilter, "Disabling classification id %d", - classification->class_id); - root->enabled = FALSE; + if (NULL == root->classifications) { + root->enabled = FALSE; + } else { + for (iter = root->classifications; iter != NULL; iter = g_list_next (iter)) { + GstInferenceClassification *classification = + (GstInferenceClassification *) iter->data; + if (classification->class_id == class_id || reset) { + GST_DEBUG_OBJECT (inferencefilter, "Enabling classification id %d", + classification->class_id); + root->enabled = TRUE; + break; + } else { + GST_DEBUG_OBJECT (inferencefilter, "Disabling classification id %d", + classification->class_id); + root->enabled = FALSE; + } } } From da7f0f1d9ad11b437d19a4bfee3deb17aa3b15a5 Mon Sep 17 00:00:00 2001 From: Carlos Rodriguez Date: Fri, 24 Jan 2020 08:05:04 -0600 Subject: [PATCH 149/188] Decouple Model and Bypass pads buffer processing Changing the collectpads approach to handle each pad logic separately --- gst-libs/gst/r2inference/gstvideoinference.c | 481 +++++++++++++------ 1 file changed, 321 insertions(+), 160 deletions(-) diff --git a/gst-libs/gst/r2inference/gstvideoinference.c b/gst-libs/gst/r2inference/gstvideoinference.c index df38c360..0b4ac4c6 100644 --- a/gst-libs/gst/r2inference/gstvideoinference.c +++ b/gst-libs/gst/r2inference/gstvideoinference.c @@ -92,6 +92,12 @@ struct _GstVideoInferencePrivate gchar *model_location; + GMutex mtx_model_queue; + GMutex mtx_bypass_queue; + + GQueue *model_queue; + GQueue *bypass_queue; + gchar *labels; gchar **labels_list; gint num_labels; @@ -126,10 +132,12 @@ static gboolean gst_video_inference_start (GstVideoInference * self); static gboolean gst_video_inference_stop (GstVideoInference * self); static GstPad *gst_video_inference_create_pad (GstVideoInference * self, GstPadTemplate * templ, const gchar * name, GstVideoInferencePad ** data); -static GstFlowReturn gst_video_inference_collected (GstCollectPads * pads, - gpointer user_data); -static GstFlowReturn gst_video_inference_pop_buffer (GstVideoInference * self, - GstCollectPads * cpads, GstCollectData * data, GstBuffer ** buffer); +static GstFlowReturn gst_video_inference_process_bypass (GstVideoInference * + self, GstBuffer * buffer, GstVideoInferencePad * pad); +static GstFlowReturn gst_video_inference_process_model (GstVideoInference * + self, GstBuffer * buffer, GstVideoInferencePad * pad); +static GstFlowReturn gst_video_inference_buffer_function (GstCollectPads * pads, + GstCollectData * data, GstBuffer * buffer, gpointer user_data); static GstFlowReturn gst_video_inference_forward_buffer (GstVideoInference * self, GstBuffer * buffer, GstPad * pad); static gboolean gst_video_inference_model_buffer_process (GstVideoInference * @@ -143,12 +151,6 @@ static gboolean gst_video_inference_predict (GstVideoInference * self, GstVideoInferencePrivate * priv, GstVideoFrame * frame, gpointer * pred, gsize * pred_size); -static gboolean gst_video_inference_postprocess (GstVideoInference * self, - GstVideoInferenceClass * klass, const gpointer prediction_data, - gsize prediction_size, GstBuffer * buffer_model, - GstVideoInferencePad * pad_model, GstBuffer * buffer_bypass, - GstVideoInferencePad * pad_bypass); - static GstIterator *gst_video_inference_iterate_internal_links (GstPad * pad, GstObject * parent); static gboolean gst_video_inference_sink_event (GstCollectPads * pads, @@ -169,11 +171,10 @@ static void video_inference_map_buffers (GstVideoInferencePad * data, GstBuffer * inbuf, GstVideoFrame * inframe, GstVideoFrame * outframe); static gboolean video_inference_prepare_postprocess (const GstMetaInfo * meta_info, GstBuffer * buffer, GstVideoInfo * video_info, - GstVideoFrame * out_frame); + GstVideoFrame * out_frame, GstMeta ** out_meta); static void video_inference_buffer_unref (GstBuffer * buffer); static void video_inference_frame_unmap (GstBuffer * buffer, GstVideoFrame * frame); -static void video_inference_remove_meta (GstBuffer * buffer, GstMeta * meta); static GstMeta *video_inference_transform_meta (GstBuffer * buffer_model, GstVideoInfo * info_model, GstMeta * meta_model, GstBuffer * buffer_bypass, GstVideoInfo * info_bypass); @@ -275,8 +276,17 @@ gst_video_inference_init (GstVideoInference * self) priv->inference_meta_info = gst_inference_meta_get_info (); priv->cpads = gst_collect_pads_new (); - gst_collect_pads_set_function (priv->cpads, gst_video_inference_collected, - (gpointer) (self)); + + g_mutex_init (&priv->mtx_model_queue); + g_mutex_init (&priv->mtx_bypass_queue); + + priv->model_queue = g_queue_new (); + priv->bypass_queue = g_queue_new (); + + /* Use buffer function to handle each pad buffer independently */ + gst_collect_pads_set_buffer_function (priv->cpads, + gst_video_inference_buffer_function, (gpointer) self); + gst_collect_pads_set_event_function (priv->cpads, gst_video_inference_sink_event, (gpointer) (self)); @@ -794,7 +804,8 @@ gst_video_inference_model_buffer_process (GstVideoInference * self, static gboolean video_inference_prepare_postprocess (const GstMetaInfo * meta_info, - GstBuffer * buffer, GstVideoInfo * video_info, GstVideoFrame * out_frame) + GstBuffer * buffer, GstVideoInfo * video_info, GstVideoFrame * out_frame, + GstMeta ** out_meta) { GstMapFlags flags; @@ -806,6 +817,23 @@ video_inference_prepare_postprocess (const GstMetaInfo * meta_info, return TRUE; } + if (out_meta) { + GstInferenceMeta *imeta = NULL; + + g_return_val_if_fail (gst_buffer_is_writable (buffer), FALSE); + out_meta[0] = gst_buffer_add_meta (buffer, meta_info, NULL); + + /* Create new meta only if the buffer didn't have one */ + if (!out_meta[1]) { + out_meta[1] = + gst_buffer_add_meta (buffer, gst_inference_meta_get_info (), NULL); + + imeta = (GstInferenceMeta *) out_meta[1]; + imeta->prediction->bbox.width = video_info->width; + imeta->prediction->bbox.height = video_info->height; + } + } + flags = (GstMapFlags) (GST_MAP_READ | GST_VIDEO_FRAME_MAP_FLAG_NO_REF); gst_video_frame_map (out_frame, video_info, buffer, flags); @@ -822,15 +850,6 @@ video_inference_frame_unmap (GstBuffer * buffer, GstVideoFrame * frame) } } -static void -video_inference_remove_meta (GstBuffer * buffer, GstMeta * meta) -{ - if (NULL != buffer && NULL != meta) { - g_return_if_fail (gst_buffer_is_writable (buffer)); - gst_buffer_remove_meta (buffer, meta); - } -} - static GstMeta * video_inference_transform_meta (GstBuffer * buffer_model, GstVideoInfo * info_model, GstMeta * meta_model, GstBuffer * buffer_bypass, @@ -861,204 +880,340 @@ video_inference_transform_meta (GstBuffer * buffer_model, return meta_bypass; } -static gboolean -gst_video_inference_postprocess (GstVideoInference * self, - GstVideoInferenceClass * klass, const gpointer prediction_data, - gsize prediction_size, GstBuffer * buffer_model, - GstVideoInferencePad * pad_model, GstBuffer * buffer_bypass, - GstVideoInferencePad * pad_bypass) +static void +video_inference_buffer_unref (GstBuffer * buffer) { + if (NULL != buffer) { + gst_buffer_unref (buffer); + } +} + +static GstFlowReturn +gst_video_inference_process_model (GstVideoInference * self, GstBuffer * buffer, + GstVideoInferencePad * pad) +{ + GstVideoInferencePrivate *priv = GST_VIDEO_INFERENCE_PRIVATE (self); + GstVideoInferenceClass *klass = GST_VIDEO_INFERENCE_GET_CLASS (self); + GstFlowReturn ret = GST_FLOW_OK; + GstMeta *current_meta = NULL; GstMeta *meta_model[2] = { NULL }; - GstMeta *meta_bypass[2] = { NULL }; GstVideoFrame frame_model; - GstVideoFrame frame_bypass; GstVideoInfo *info_model = NULL; - GstVideoInfo *info_bypass = NULL; + GstBuffer *buffer_model = NULL; + gpointer prediction_data = NULL; + gsize prediction_size; gboolean pred_valid = FALSE; - GstVideoInferencePrivate *priv = GST_VIDEO_INFERENCE_PRIVATE (self); - g_return_val_if_fail (self, FALSE); - g_return_val_if_fail (klass, FALSE); - g_return_val_if_fail (prediction_data, FALSE); - g_return_val_if_fail (prediction_size, FALSE); - g_return_val_if_fail (buffer_model, FALSE); - g_return_val_if_fail (pad_model, FALSE); + g_return_val_if_fail (self != NULL, GST_FLOW_ERROR); + g_return_val_if_fail (buffer != NULL, GST_FLOW_ERROR); + g_return_val_if_fail (pad != NULL, GST_FLOW_ERROR); - info_model = &(pad_model->info); - info_bypass = pad_bypass ? &(pad_bypass->info) : NULL; + GST_LOG_OBJECT (self, "Processing model buffer"); - /* Subclass didn't implement a post-process, dont fail, just ignore */ if (NULL == klass->postprocess) { - return TRUE; - } - - meta_model[0] = - gst_buffer_get_meta (buffer_model, gst_detection_meta_api_get_type ()); - if (!meta_model[0]) { - meta_model[0] = - gst_buffer_add_meta (buffer_model, klass->inference_meta_info, NULL); + GST_ELEMENT_ERROR (self, STREAM, FAILED, + ("Subclass didn't implement post-process"), (NULL)); + ret = GST_FLOW_ERROR; + goto buffer_free; } - meta_model[1] = + buffer_model = gst_buffer_make_writable (buffer); + current_meta = gst_buffer_get_meta (buffer_model, gst_inference_meta_api_get_type ()); - if (!meta_model[1]) { - GstInferenceMeta *imeta; - - meta_model[1] = - gst_buffer_add_meta (buffer_model, gst_inference_meta_get_info (), - NULL); - - imeta = (GstInferenceMeta *) meta_model[1]; - imeta->prediction->bbox.width = info_model->width; - imeta->prediction->bbox.height = info_model->height; + if (current_meta) { + /* Check if root is enabled to be processed, if not, just forward buffer */ + GstInferenceMeta *inference_meta = (GstInferenceMeta *) current_meta; + GstInferencePrediction *root = inference_meta->prediction; + if (!root->enabled) { + GST_INFO_OBJECT (self, + "Current Prediction is not enabled, bypassing processing..."); + goto forward_buffer; + } } - if (!video_inference_prepare_postprocess (klass->inference_meta_info, - buffer_model, info_model, &frame_model)) { - return FALSE; + /* Run preprocess and inference on the model and generate prediction */ + if (!gst_video_inference_model_buffer_process (self, klass, priv, + buffer_model, &prediction_data, &prediction_size)) { + ret = GST_FLOW_ERROR; + goto buffer_free; } + /* Assign already created inferencemeta, no need to create a new one */ + meta_model[1] = current_meta; + + /* Prepare postprocess */ + info_model = &(pad->info); if (!video_inference_prepare_postprocess (klass->inference_meta_info, - buffer_bypass, info_bypass, &frame_bypass)) { - return FALSE; + buffer_model, info_model, &frame_model, meta_model)) { + ret = GST_FLOW_ERROR; + goto buffer_free; } - - GST_LOG_OBJECT (self, "Calling frame postprocess"); - if (!klass->postprocess (self, prediction_data, prediction_size, meta_model, - info_model, &pred_valid, priv->labels_list, priv->num_labels)) { - GST_ELEMENT_ERROR (self, STREAM, FAILED, - ("Subclass failed at postprocess"), (NULL)); - return FALSE; + /* Subclass Processing */ + if (!klass->postprocess (self, prediction_data, prediction_size, + meta_model, info_model, &pred_valid, priv->labels_list, + priv->num_labels)) { + GST_ELEMENT_ERROR (self, STREAM, FAILED, ("Subclass failed at preprocess"), + (NULL)); + ret = GST_FLOW_ERROR; + goto buffer_free; } - if (pred_valid) { - GstVideoFrame *pbpass = buffer_bypass ? &frame_bypass : NULL; - meta_bypass[0] = - video_inference_transform_meta (buffer_model, info_model, meta_model[0], - buffer_bypass, info_bypass); - - g_signal_emit (self, gst_video_inference_signals[NEW_PREDICTION_SIGNAL], - 0, meta_model[0], &frame_model, meta_bypass[0], pbpass); - + /* Check if bypass pad was requested, if not, forward buffer */ + if (NULL == priv->sink_bypass) { + GST_INFO_OBJECT (self, + "There is no sinkpad for bypass, forwarding model buffer..."); + goto forward_buffer; } else { - video_inference_remove_meta (buffer_model, meta_model[0]); + /* Queue buffer */ + GST_LOG_OBJECT (self, "Queue model buffer"); + g_mutex_lock (&priv->mtx_model_queue); + g_queue_push_head (priv->model_queue, (gpointer) buffer_model); + g_mutex_unlock (&priv->mtx_model_queue); + goto buffer_free; } - meta_bypass[1] = - video_inference_transform_meta (buffer_model, info_model, meta_model[1], - buffer_bypass, info_bypass); +forward_buffer: + /* Keep previous results */ + ret |= gst_video_inference_forward_buffer (self, buffer_model, + priv->src_model); +buffer_free: video_inference_frame_unmap (buffer_model, &frame_model); - video_inference_frame_unmap (buffer_bypass, &frame_bypass); - return TRUE; + if (GST_FLOW_OK != ret) { + /* Free if forward failed */ + video_inference_buffer_unref (buffer_model); + } + g_free (prediction_data); + + return ret; } -static GstFlowReturn -gst_video_inference_pop_buffer (GstVideoInference * self, - GstCollectPads * cpads, GstCollectData * data, GstBuffer ** buffer) +static gboolean +gst_inference_is_prediction_enabled (GNode * node, gpointer data) { - g_return_val_if_fail (self, GST_FLOW_ERROR); - g_return_val_if_fail (buffer, GST_FLOW_ERROR); + GstInferencePrediction *root = (GstInferencePrediction *) node->data; + gboolean *enabled; - *buffer = NULL; + g_return_val_if_fail (root != NULL, TRUE); + g_return_val_if_fail (data != NULL, TRUE); - if (NULL == data) { - return GST_FLOW_OK; - } + enabled = (gboolean *) data; - *buffer = gst_collect_pads_pop (cpads, data); - if (NULL == *buffer) { - GST_INFO_OBJECT (self, "EOS requested on %" GST_PTR_FORMAT, data->pad); - return GST_FLOW_EOS; + if (root->enabled) { + /* This prediction is enabled, mark enabled flag and terminate traverse */ + *enabled = TRUE; + return TRUE; } - - *buffer = gst_buffer_make_writable (*buffer); - - GST_LOG_OBJECT (self, "Popped %" GST_PTR_FORMAT " from %" GST_PTR_FORMAT, - *buffer, data->pad); - return GST_FLOW_OK; + return FALSE; } static void -video_inference_buffer_unref (GstBuffer * buffer) +video_inference_notify (GstVideoInference * self, GstBuffer * model_buffer, + GstMeta * meta_model[2], GstBuffer * bypass_buffer, + GstMeta * meta_bypass[2]) { - if (NULL != buffer) { - gst_buffer_unref (buffer); - } + GstVideoInferencePrivate *priv = GST_VIDEO_INFERENCE_PRIVATE (self); + GstVideoFrame frame_model; + GstVideoFrame frame_bypass; + GstVideoInfo *info_model = NULL; + GstVideoInfo *info_bypass = NULL; + GstMapFlags flags; + + g_return_if_fail (model_buffer); + g_return_if_fail (meta_model); + + info_model = &(priv->sink_model_data->info); + info_bypass = &(priv->sink_bypass_data->info); + + flags = (GstMapFlags) (GST_MAP_READ | GST_VIDEO_FRAME_MAP_FLAG_NO_REF); + gst_video_frame_map (&frame_model, info_model, model_buffer, flags); + gst_video_frame_map (&frame_bypass, info_bypass, bypass_buffer, flags); + + /* Emit inference signal */ + g_signal_emit (self, gst_video_inference_signals[NEW_PREDICTION_SIGNAL], 0, + meta_model[0], &frame_model, meta_bypass[0], &frame_bypass); + g_signal_emit (self, gst_video_inference_signals[NEW_INFERENCE_SIGNAL], 0, + meta_model[1], &frame_model, meta_bypass[1], &frame_bypass); + + video_inference_frame_unmap (model_buffer, &frame_model); + video_inference_frame_unmap (bypass_buffer, &frame_bypass); } static GstFlowReturn -gst_video_inference_collected (GstCollectPads * pads, gpointer user_data) +gst_video_inference_process_bypass (GstVideoInference * self, + GstBuffer * buffer, GstVideoInferencePad * pad) { - GstVideoInference *self = GST_VIDEO_INFERENCE (user_data); - GstVideoInferenceClass *klass = GST_VIDEO_INFERENCE_GET_CLASS (self); GstVideoInferencePrivate *priv = GST_VIDEO_INFERENCE_PRIVATE (self); + GstVideoInferenceClass *klass = GST_VIDEO_INFERENCE_GET_CLASS (self); GstFlowReturn ret = GST_FLOW_OK; - GstBuffer *buffer_model = NULL; - GstBuffer *buffer_bypass = NULL; - gpointer prediction_data = NULL; - gsize prediction_size; + GstMeta *current_meta = NULL; + GstBuffer *bypass_buffer = NULL; + GstBuffer *model_buffer = NULL; + gboolean model_empty = FALSE; - ret = - gst_video_inference_pop_buffer (self, pads, - (GstCollectData *) priv->sink_model_data, &buffer_model); - if (GST_FLOW_OK != ret) { - goto out; - } + g_return_val_if_fail (self != NULL, GST_FLOW_ERROR); + g_return_val_if_fail (buffer != NULL, GST_FLOW_ERROR); + g_return_val_if_fail (pad != NULL, GST_FLOW_ERROR); - ret = - gst_video_inference_pop_buffer (self, pads, - (GstCollectData *) priv->sink_bypass_data, &buffer_bypass); - if (GST_FLOW_OK != ret) { - goto model_free; + GST_LOG_OBJECT (self, "Processing bypass buffer"); + + /* Check if model pad was requested, if not, forward buffer */ + if (NULL == priv->sink_model) { + GST_INFO_OBJECT (self, + "There is no sinkpad for model, forwarding bypass buffer..."); + goto forward_buffer; } - if (buffer_model) { - /* Run preprocess and inference on the model and generate prediction */ - if (!gst_video_inference_model_buffer_process (self, klass, priv, - buffer_model, &prediction_data, &prediction_size)) { - ret = GST_FLOW_ERROR; - goto bypass_free; + bypass_buffer = gst_buffer_make_writable (buffer); + current_meta = gst_buffer_get_meta (bypass_buffer, + gst_inference_meta_api_get_type ()); + if (current_meta) { + /* Check if at least one node is enabled to be processed, if not, just forward buffer */ + GstInferenceMeta *imeta = (GstInferenceMeta *) current_meta; + gboolean enabled = FALSE; + + g_node_traverse (imeta->prediction->predictions, G_LEVEL_ORDER, + G_TRAVERSE_ALL, -1, gst_inference_is_prediction_enabled, &enabled); + if (!enabled) { + GST_INFO_OBJECT (self, + "There is no predictions enabled, bypassing processing..."); + goto forward_buffer; } + } + + /* Queue this new buffer at the head and dequeue the older one */ + GST_LOG_OBJECT (self, "Queue bypass buffer and get older one"); + g_mutex_lock (&priv->mtx_bypass_queue); + g_queue_push_head (priv->bypass_queue, (gpointer) bypass_buffer); + bypass_buffer = (GstBuffer *) g_queue_pop_tail (priv->bypass_queue); + g_mutex_unlock (&priv->mtx_bypass_queue); + + while (!model_empty) { + /* Dequeue oldest model buffer from tail */ + g_mutex_lock (&priv->mtx_model_queue); + GST_LOG_OBJECT (self, "Dequeue model buffer"); + model_buffer = (GstBuffer *) g_queue_pop_tail (priv->model_queue); + g_mutex_unlock (&priv->mtx_model_queue); + + if (NULL == model_buffer) { + /* Model queue is empty */ + model_empty = TRUE; + } else { + GstInferencePrediction *root_model = NULL; + GstInferencePrediction *root_bypass = NULL; + GstMeta *meta_model[2] = { NULL }; + GstMeta *meta_bypass[2] = { NULL }; + GstVideoInfo *info_model = &(priv->sink_model_data->info); + GstVideoInfo *info_bypass = &(priv->sink_bypass_data->info); + + /* Get both metas from model buffer */ + meta_model[0] = gst_buffer_get_meta (model_buffer, + klass->inference_meta_info->api); + meta_model[1] = gst_buffer_get_meta (model_buffer, + gst_inference_meta_api_get_type ()); + + root_model = ((GstInferenceMeta *) meta_model[1])->prediction; + + /* If bypass doesn't have meta, just transfer the model meta */ + current_meta = gst_buffer_get_meta (bypass_buffer, + gst_inference_meta_api_get_type ()); + + if (current_meta) { + /* Check if model and bypass IDs match */ + GST_LOG_OBJECT (self, "Checking if model and bypass IDs match"); + root_bypass = gst_inference_prediction_find (((GstInferenceMeta *) + current_meta)->prediction, root_model->prediction_id); + if (NULL == root_bypass) { + /* Queue model buffer to tail again */ + g_mutex_lock (&priv->mtx_model_queue); + GST_LOG_OBJECT (self, "Queue model buffer"); + g_queue_push_tail (priv->model_queue, (gpointer) model_buffer); + g_mutex_unlock (&priv->mtx_model_queue); + goto forward_buffer; + } + } + + /* Transfer meta from model to bypass */ + GST_LOG_OBJECT (self, "Transfering meta from model to bypass"); + + meta_bypass[0] = + video_inference_transform_meta (model_buffer, info_model, + (GstMeta *) meta_model[0], bypass_buffer, info_bypass); + meta_bypass[1] = + video_inference_transform_meta (model_buffer, info_model, + (GstMeta *) meta_model[1], bypass_buffer, info_bypass); + + /* Notify prediction */ + video_inference_notify (self, model_buffer, meta_model, bypass_buffer, + meta_bypass); + + /* Forward buffer to model src pad */ + GST_LOG_OBJECT (self, "Forward model buffer"); + ret = gst_video_inference_forward_buffer (self, model_buffer, + priv->src_model); - /* Have the subclass analyze the prediction and generate model and bypass metas */ - if (!gst_video_inference_postprocess (self, klass, prediction_data, - prediction_size, buffer_model, priv->sink_model_data, buffer_bypass, - priv->sink_bypass_data)) { - ret = GST_FLOW_ERROR; - goto bypass_free; + if (GST_FLOW_OK != ret) { + /* Free if forward failed */ + video_inference_buffer_unref (model_buffer); + } + model_buffer = NULL; } } - /* Forward buffer to model src pad */ - ret = gst_video_inference_forward_buffer (self, buffer_model, - priv->src_model); + /* Queue old bypass buffer at the tail */ + GST_LOG_OBJECT (self, "Queue bypass buffer"); + g_mutex_lock (&priv->mtx_bypass_queue); + g_queue_push_tail (priv->bypass_queue, (gpointer) bypass_buffer); + g_mutex_unlock (&priv->mtx_bypass_queue); + + return ret; + +forward_buffer: + /* Forward buffer to bypass src pad */ + GST_LOG_OBJECT (self, "Forward bypass buffer"); + ret = + gst_video_inference_forward_buffer (self, bypass_buffer, + priv->src_bypass); - /* We don't own this buffer anymore, don't free it */ - buffer_model = NULL; if (GST_FLOW_OK != ret) { - goto bypass_free; + /* Free if forward failed */ + video_inference_buffer_unref (bypass_buffer); } - /* Forward buffer to bypass src pad */ - ret = gst_video_inference_forward_buffer (self, - buffer_bypass, priv->src_bypass); + return ret; +} + +static GstFlowReturn +gst_video_inference_buffer_function (GstCollectPads * pads, + GstCollectData * data, GstBuffer * buffer, gpointer user_data) +{ + GstVideoInference *self = (GstVideoInference *) user_data; + GstVideoInferencePad *pad = (GstVideoInferencePad *) data; + GstFlowReturn ret = GST_FLOW_OK; - /* We don't own this buffer anymore, don't free it */ - buffer_bypass = NULL; + if (!buffer) { + /* EOS reached the pad */ + ret = GST_FLOW_EOS; + goto out; + } - goto out; + g_return_val_if_fail (pads != NULL, GST_FLOW_ERROR); + g_return_val_if_fail (self != NULL, GST_FLOW_ERROR); -bypass_free: - video_inference_buffer_unref (buffer_bypass); + if (!g_strcmp0 (GST_PAD_NAME (data->pad), "sink_model")) { + GST_LOG_OBJECT (self, "Model buffer arrived, processing it..."); + ret = gst_video_inference_process_model (self, buffer, pad); + goto out; + } -model_free: - video_inference_buffer_unref (buffer_model); + if (!g_strcmp0 (GST_PAD_NAME (data->pad), "sink_bypass")) { + GST_LOG_OBJECT (self, "Bypass buffer arrived, processing it..."); + ret = gst_video_inference_process_bypass (self, buffer, pad); + goto out; + } out: - g_free (prediction_data); - return ret; } @@ -1233,6 +1388,12 @@ gst_video_inference_finalize (GObject * object) g_free (priv->labels_list); priv->labels_list = NULL; + g_mutex_clear (&priv->mtx_model_queue); + g_mutex_clear (&priv->mtx_bypass_queue); + + g_queue_free (priv->model_queue); + g_queue_free (priv->bypass_queue); + g_clear_object (&priv->backend); G_OBJECT_CLASS (gst_video_inference_parent_class)->finalize (object); From a8054358c36330f2f0ba1373fb6652228c70318c Mon Sep 17 00:00:00 2001 From: Michael Gruner Date: Fri, 24 Jan 2020 10:55:09 -0600 Subject: [PATCH 150/188] Compare pads using the pointer instead of the name --- gst-libs/gst/r2inference/gstvideoinference.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/gst-libs/gst/r2inference/gstvideoinference.c b/gst-libs/gst/r2inference/gstvideoinference.c index 0b4ac4c6..2a10a768 100644 --- a/gst-libs/gst/r2inference/gstvideoinference.c +++ b/gst-libs/gst/r2inference/gstvideoinference.c @@ -1189,6 +1189,7 @@ gst_video_inference_buffer_function (GstCollectPads * pads, GstCollectData * data, GstBuffer * buffer, gpointer user_data) { GstVideoInference *self = (GstVideoInference *) user_data; + GstVideoInferencePrivate *priv = NULL; GstVideoInferencePad *pad = (GstVideoInferencePad *) data; GstFlowReturn ret = GST_FLOW_OK; @@ -1201,13 +1202,13 @@ gst_video_inference_buffer_function (GstCollectPads * pads, g_return_val_if_fail (pads != NULL, GST_FLOW_ERROR); g_return_val_if_fail (self != NULL, GST_FLOW_ERROR); - if (!g_strcmp0 (GST_PAD_NAME (data->pad), "sink_model")) { + priv = GST_VIDEO_INFERENCE_PRIVATE (self); + + if (data->pad == priv->sink_model) { GST_LOG_OBJECT (self, "Model buffer arrived, processing it..."); ret = gst_video_inference_process_model (self, buffer, pad); goto out; - } - - if (!g_strcmp0 (GST_PAD_NAME (data->pad), "sink_bypass")) { + } else { GST_LOG_OBJECT (self, "Bypass buffer arrived, processing it..."); ret = gst_video_inference_process_bypass (self, buffer, pad); goto out; From 7dccb062ceaf42835dd6e2e70fc93cd061525487 Mon Sep 17 00:00:00 2001 From: Michael Gruner Date: Fri, 24 Jan 2020 12:14:47 -0600 Subject: [PATCH 151/188] Simplify buffer preparation for postprocessing --- gst-libs/gst/r2inference/gstvideoinference.c | 47 +++++++------------- 1 file changed, 17 insertions(+), 30 deletions(-) diff --git a/gst-libs/gst/r2inference/gstvideoinference.c b/gst-libs/gst/r2inference/gstvideoinference.c index 2a10a768..e2777f92 100644 --- a/gst-libs/gst/r2inference/gstvideoinference.c +++ b/gst-libs/gst/r2inference/gstvideoinference.c @@ -171,7 +171,7 @@ static void video_inference_map_buffers (GstVideoInferencePad * data, GstBuffer * inbuf, GstVideoFrame * inframe, GstVideoFrame * outframe); static gboolean video_inference_prepare_postprocess (const GstMetaInfo * meta_info, GstBuffer * buffer, GstVideoInfo * video_info, - GstVideoFrame * out_frame, GstMeta ** out_meta); + GstMeta ** out_meta); static void video_inference_buffer_unref (GstBuffer * buffer); static void video_inference_frame_unmap (GstBuffer * buffer, GstVideoFrame * frame); @@ -804,39 +804,28 @@ gst_video_inference_model_buffer_process (GstVideoInference * self, static gboolean video_inference_prepare_postprocess (const GstMetaInfo * meta_info, - GstBuffer * buffer, GstVideoInfo * video_info, GstVideoFrame * out_frame, - GstMeta ** out_meta) + GstBuffer * buffer, GstVideoInfo * video_info, GstMeta ** out_meta) { - GstMapFlags flags; + GstInferenceMeta *imeta = NULL; g_return_val_if_fail (meta_info, FALSE); - g_return_val_if_fail (out_frame, FALSE); - - /* No pad requested, continue without meta */ - if (NULL == buffer || NULL == video_info) { - return TRUE; - } - - if (out_meta) { - GstInferenceMeta *imeta = NULL; + g_return_val_if_fail (buffer, FALSE); + g_return_val_if_fail (video_info, FALSE); + g_return_val_if_fail (out_meta, FALSE); - g_return_val_if_fail (gst_buffer_is_writable (buffer), FALSE); - out_meta[0] = gst_buffer_add_meta (buffer, meta_info, NULL); + g_return_val_if_fail (gst_buffer_is_writable (buffer), FALSE); + out_meta[0] = gst_buffer_add_meta (buffer, meta_info, NULL); - /* Create new meta only if the buffer didn't have one */ - if (!out_meta[1]) { - out_meta[1] = - gst_buffer_add_meta (buffer, gst_inference_meta_get_info (), NULL); + /* Create new meta only if the buffer didn't have one */ + if (!out_meta[1]) { + out_meta[1] = + gst_buffer_add_meta (buffer, gst_inference_meta_get_info (), NULL); - imeta = (GstInferenceMeta *) out_meta[1]; - imeta->prediction->bbox.width = video_info->width; - imeta->prediction->bbox.height = video_info->height; - } + imeta = (GstInferenceMeta *) out_meta[1]; + imeta->prediction->bbox.width = video_info->width; + imeta->prediction->bbox.height = video_info->height; } - flags = (GstMapFlags) (GST_MAP_READ | GST_VIDEO_FRAME_MAP_FLAG_NO_REF); - gst_video_frame_map (out_frame, video_info, buffer, flags); - return TRUE; } @@ -897,7 +886,6 @@ gst_video_inference_process_model (GstVideoInference * self, GstBuffer * buffer, GstFlowReturn ret = GST_FLOW_OK; GstMeta *current_meta = NULL; GstMeta *meta_model[2] = { NULL }; - GstVideoFrame frame_model; GstVideoInfo *info_model = NULL; GstBuffer *buffer_model = NULL; gpointer prediction_data = NULL; @@ -944,10 +932,11 @@ gst_video_inference_process_model (GstVideoInference * self, GstBuffer * buffer, /* Prepare postprocess */ info_model = &(pad->info); if (!video_inference_prepare_postprocess (klass->inference_meta_info, - buffer_model, info_model, &frame_model, meta_model)) { + buffer_model, info_model, meta_model)) { ret = GST_FLOW_ERROR; goto buffer_free; } + /* Subclass Processing */ if (!klass->postprocess (self, prediction_data, prediction_size, meta_model, info_model, &pred_valid, priv->labels_list, @@ -978,8 +967,6 @@ gst_video_inference_process_model (GstVideoInference * self, GstBuffer * buffer, priv->src_model); buffer_free: - video_inference_frame_unmap (buffer_model, &frame_model); - if (GST_FLOW_OK != ret) { /* Free if forward failed */ video_inference_buffer_unref (buffer_model); From ebdb233068709c920a03f82cba8fbf89e4b09247 Mon Sep 17 00:00:00 2001 From: Michael Gruner Date: Fri, 24 Jan 2020 14:33:52 -0600 Subject: [PATCH 152/188] Simplify code by removing unneeded checks --- gst-libs/gst/r2inference/gstvideoinference.c | 54 ++++---------------- 1 file changed, 11 insertions(+), 43 deletions(-) diff --git a/gst-libs/gst/r2inference/gstvideoinference.c b/gst-libs/gst/r2inference/gstvideoinference.c index e2777f92..6d2354b4 100644 --- a/gst-libs/gst/r2inference/gstvideoinference.c +++ b/gst-libs/gst/r2inference/gstvideoinference.c @@ -172,9 +172,6 @@ static void video_inference_map_buffers (GstVideoInferencePad * data, static gboolean video_inference_prepare_postprocess (const GstMetaInfo * meta_info, GstBuffer * buffer, GstVideoInfo * video_info, GstMeta ** out_meta); -static void video_inference_buffer_unref (GstBuffer * buffer); -static void video_inference_frame_unmap (GstBuffer * buffer, - GstVideoFrame * frame); static GstMeta *video_inference_transform_meta (GstBuffer * buffer_model, GstVideoInfo * info_model, GstMeta * meta_model, GstBuffer * buffer_bypass, GstVideoInfo * info_bypass); @@ -829,16 +826,6 @@ video_inference_prepare_postprocess (const GstMetaInfo * meta_info, return TRUE; } -static void -video_inference_frame_unmap (GstBuffer * buffer, GstVideoFrame * frame) -{ - g_return_if_fail (frame); - - if (NULL != buffer) { - gst_video_frame_unmap (frame); - } -} - static GstMeta * video_inference_transform_meta (GstBuffer * buffer_model, GstVideoInfo * info_model, GstMeta * meta_model, GstBuffer * buffer_bypass, @@ -869,14 +856,6 @@ video_inference_transform_meta (GstBuffer * buffer_model, return meta_bypass; } -static void -video_inference_buffer_unref (GstBuffer * buffer) -{ - if (NULL != buffer) { - gst_buffer_unref (buffer); - } -} - static GstFlowReturn gst_video_inference_process_model (GstVideoInference * self, GstBuffer * buffer, GstVideoInferencePad * pad) @@ -934,7 +913,7 @@ gst_video_inference_process_model (GstVideoInference * self, GstBuffer * buffer, if (!video_inference_prepare_postprocess (klass->inference_meta_info, buffer_model, info_model, meta_model)) { ret = GST_FLOW_ERROR; - goto buffer_free; + goto prediction_free; } /* Subclass Processing */ @@ -944,7 +923,7 @@ gst_video_inference_process_model (GstVideoInference * self, GstBuffer * buffer, GST_ELEMENT_ERROR (self, STREAM, FAILED, ("Subclass failed at preprocess"), (NULL)); ret = GST_FLOW_ERROR; - goto buffer_free; + goto prediction_free; } /* Check if bypass pad was requested, if not, forward buffer */ @@ -958,21 +937,20 @@ gst_video_inference_process_model (GstVideoInference * self, GstBuffer * buffer, g_mutex_lock (&priv->mtx_model_queue); g_queue_push_head (priv->model_queue, (gpointer) buffer_model); g_mutex_unlock (&priv->mtx_model_queue); - goto buffer_free; + goto out; } forward_buffer: - /* Keep previous results */ - ret |= gst_video_inference_forward_buffer (self, buffer_model, + ret = gst_video_inference_forward_buffer (self, gst_buffer_ref (buffer_model), priv->src_model); -buffer_free: - if (GST_FLOW_OK != ret) { - /* Free if forward failed */ - video_inference_buffer_unref (buffer_model); - } +prediction_free: g_free (prediction_data); +buffer_free: + gst_buffer_unref (buffer_model); + +out: return ret; } @@ -1023,8 +1001,8 @@ video_inference_notify (GstVideoInference * self, GstBuffer * model_buffer, g_signal_emit (self, gst_video_inference_signals[NEW_INFERENCE_SIGNAL], 0, meta_model[1], &frame_model, meta_bypass[1], &frame_bypass); - video_inference_frame_unmap (model_buffer, &frame_model); - video_inference_frame_unmap (bypass_buffer, &frame_bypass); + gst_video_frame_unmap (&frame_model); + gst_video_frame_unmap (&frame_bypass); } static GstFlowReturn @@ -1139,11 +1117,6 @@ gst_video_inference_process_bypass (GstVideoInference * self, GST_LOG_OBJECT (self, "Forward model buffer"); ret = gst_video_inference_forward_buffer (self, model_buffer, priv->src_model); - - if (GST_FLOW_OK != ret) { - /* Free if forward failed */ - video_inference_buffer_unref (model_buffer); - } model_buffer = NULL; } } @@ -1163,11 +1136,6 @@ gst_video_inference_process_bypass (GstVideoInference * self, gst_video_inference_forward_buffer (self, bypass_buffer, priv->src_bypass); - if (GST_FLOW_OK != ret) { - /* Free if forward failed */ - video_inference_buffer_unref (bypass_buffer); - } - return ret; } From 0e4f97239d6860668029c6dd0264f175c5c7daac Mon Sep 17 00:00:00 2001 From: Michael Gruner Date: Fri, 24 Jan 2020 14:40:58 -0600 Subject: [PATCH 153/188] Improve ambiguous method name --- gst-libs/gst/r2inference/gstvideoinference.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/gst-libs/gst/r2inference/gstvideoinference.c b/gst-libs/gst/r2inference/gstvideoinference.c index 6d2354b4..55e36ad9 100644 --- a/gst-libs/gst/r2inference/gstvideoinference.c +++ b/gst-libs/gst/r2inference/gstvideoinference.c @@ -140,7 +140,7 @@ static GstFlowReturn gst_video_inference_buffer_function (GstCollectPads * pads, GstCollectData * data, GstBuffer * buffer, gpointer user_data); static GstFlowReturn gst_video_inference_forward_buffer (GstVideoInference * self, GstBuffer * buffer, GstPad * pad); -static gboolean gst_video_inference_model_buffer_process (GstVideoInference * +static gboolean gst_video_inference_model_run_prediction (GstVideoInference * self, GstVideoInferenceClass * klass, GstVideoInferencePrivate * priv, GstBuffer * buffer, gpointer * prediction_data, gsize * prediction_size); @@ -759,7 +759,7 @@ gst_video_inference_predict (GstVideoInference * self, } static gboolean -gst_video_inference_model_buffer_process (GstVideoInference * self, +gst_video_inference_model_run_prediction (GstVideoInference * self, GstVideoInferenceClass * klass, GstVideoInferencePrivate * priv, GstBuffer * buffer, gpointer * prediction_data, gsize * prediction_size) { @@ -899,7 +899,7 @@ gst_video_inference_process_model (GstVideoInference * self, GstBuffer * buffer, } /* Run preprocess and inference on the model and generate prediction */ - if (!gst_video_inference_model_buffer_process (self, klass, priv, + if (!gst_video_inference_model_run_prediction (self, klass, priv, buffer_model, &prediction_data, &prediction_size)) { ret = GST_FLOW_ERROR; goto buffer_free; From 57ed218ec71fcde491d3731f6ecc9bcc4178b674 Mon Sep 17 00:00:00 2001 From: Michael Gruner Date: Fri, 24 Jan 2020 14:43:28 -0600 Subject: [PATCH 154/188] Demote per buffer logs to GST_LOG --- gst-libs/gst/r2inference/gstvideoinference.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gst-libs/gst/r2inference/gstvideoinference.c b/gst-libs/gst/r2inference/gstvideoinference.c index 55e36ad9..28903128 100644 --- a/gst-libs/gst/r2inference/gstvideoinference.c +++ b/gst-libs/gst/r2inference/gstvideoinference.c @@ -928,7 +928,7 @@ gst_video_inference_process_model (GstVideoInference * self, GstBuffer * buffer, /* Check if bypass pad was requested, if not, forward buffer */ if (NULL == priv->sink_bypass) { - GST_INFO_OBJECT (self, + GST_LOG_OBJECT (self, "There is no sinkpad for bypass, forwarding model buffer..."); goto forward_buffer; } else { @@ -1025,7 +1025,7 @@ gst_video_inference_process_bypass (GstVideoInference * self, /* Check if model pad was requested, if not, forward buffer */ if (NULL == priv->sink_model) { - GST_INFO_OBJECT (self, + GST_LOG_OBJECT (self, "There is no sinkpad for model, forwarding bypass buffer..."); goto forward_buffer; } From 3a918bb52775896fece1045549c04609b0c7830a Mon Sep 17 00:00:00 2001 From: Michael Gruner Date: Fri, 24 Jan 2020 14:44:19 -0600 Subject: [PATCH 155/188] Cast using gobject macros --- gst-libs/gst/r2inference/gstvideoinference.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gst-libs/gst/r2inference/gstvideoinference.c b/gst-libs/gst/r2inference/gstvideoinference.c index 28903128..8367820e 100644 --- a/gst-libs/gst/r2inference/gstvideoinference.c +++ b/gst-libs/gst/r2inference/gstvideoinference.c @@ -1051,14 +1051,14 @@ gst_video_inference_process_bypass (GstVideoInference * self, GST_LOG_OBJECT (self, "Queue bypass buffer and get older one"); g_mutex_lock (&priv->mtx_bypass_queue); g_queue_push_head (priv->bypass_queue, (gpointer) bypass_buffer); - bypass_buffer = (GstBuffer *) g_queue_pop_tail (priv->bypass_queue); + bypass_buffer = GST_BUFFER_CAST (g_queue_pop_tail (priv->bypass_queue)); g_mutex_unlock (&priv->mtx_bypass_queue); while (!model_empty) { /* Dequeue oldest model buffer from tail */ g_mutex_lock (&priv->mtx_model_queue); GST_LOG_OBJECT (self, "Dequeue model buffer"); - model_buffer = (GstBuffer *) g_queue_pop_tail (priv->model_queue); + model_buffer = GST_BUFFER_CAST (g_queue_pop_tail (priv->model_queue)); g_mutex_unlock (&priv->mtx_model_queue); if (NULL == model_buffer) { From ba9af1eaffe1b67d1fceea2bd4dac44a4802c5ba Mon Sep 17 00:00:00 2001 From: Michael Gruner Date: Fri, 24 Jan 2020 15:07:35 -0600 Subject: [PATCH 156/188] Move enabled query to prediction module --- .../gst/r2inference/gstinferenceprediction.c | 43 +++++++++++++++++++ .../gst/r2inference/gstinferenceprediction.h | 10 +++++ gst-libs/gst/r2inference/gstvideoinference.c | 28 ++---------- 3 files changed, 57 insertions(+), 24 deletions(-) diff --git a/gst-libs/gst/r2inference/gstinferenceprediction.c b/gst-libs/gst/r2inference/gstinferenceprediction.c index ac216e36..7ced6633 100644 --- a/gst-libs/gst/r2inference/gstinferenceprediction.c +++ b/gst-libs/gst/r2inference/gstinferenceprediction.c @@ -59,6 +59,8 @@ static void prediction_scale_ip (GstInferencePrediction * self, static GSList *prediction_get_children_unlocked (GstInferencePrediction * self); static void prediction_merge (GstInferencePrediction * src, GstInferencePrediction * dst); +static void prediction_get_enabled (GstInferencePrediction * self, + GList ** enabled); static GstInferenceClassification * classification_copy (GstInferenceClassification * from, gpointer data); @@ -74,6 +76,7 @@ static gboolean node_scale_ip (GNode * node, gpointer data); static gpointer node_scale (gconstpointer, gpointer data); static gboolean node_assign (GNode * node, gpointer data); static gboolean node_find (GNode * node, gpointer data); +static gboolean node_get_enabled (GNode * node, gpointer data); static void compute_factors (GstVideoInfo * from, GstVideoInfo * to, gdouble * hfactor, gdouble * vfactor); @@ -744,3 +747,43 @@ gst_inference_prediction_merge (GstInferencePrediction * src, GST_INFERENCE_PREDICTION_UNLOCK (dst); GST_INFERENCE_PREDICTION_UNLOCK (src); } + +static void +prediction_get_enabled (GstInferencePrediction * self, GList ** found) +{ + g_return_if_fail (self); + g_return_if_fail (found); + + if (self->enabled) { + *found = g_list_append (*found, self); + } +} + +static gboolean +node_get_enabled (GNode * node, gpointer data) +{ + GstInferencePrediction *self = NULL; + GList **found = (GList **) data; + + g_return_val_if_fail (node, TRUE); + g_return_val_if_fail (found, TRUE); + + self = (GstInferencePrediction *) node->data; + + prediction_get_enabled (self, found); + + return FALSE; +} + +GList * +gst_inference_prediction_get_enabled (GstInferencePrediction * self) +{ + GList *found = NULL; + + g_return_val_if_fail (self, NULL); + + g_node_traverse (self->predictions, G_IN_ORDER, G_TRAVERSE_ALL, -1, + node_get_enabled, &found); + + return found; +} diff --git a/gst-libs/gst/r2inference/gstinferenceprediction.h b/gst-libs/gst/r2inference/gstinferenceprediction.h index 1e33847d..b9fd0bf7 100644 --- a/gst-libs/gst/r2inference/gstinferenceprediction.h +++ b/gst-libs/gst/r2inference/gstinferenceprediction.h @@ -231,6 +231,16 @@ void gst_inference_prediction_scale_ip (GstInferencePrediction * self, GstInferencePrediction * gst_inference_prediction_find (GstInferencePrediction * self, guint64 id); +/** + * gst_inference_prediction_get_enabled: + * @self: the root prediction + * + * Traverse the prediction three saving the predictions that are enabled. + * + * Returns: a GList of predictions that are enabled. + */ +GList * gst_inference_prediction_get_enabled (GstInferencePrediction * self); + /** * gst_inference_prediction_merge: * @src: the source prediction diff --git a/gst-libs/gst/r2inference/gstvideoinference.c b/gst-libs/gst/r2inference/gstvideoinference.c index 8367820e..a441ee95 100644 --- a/gst-libs/gst/r2inference/gstvideoinference.c +++ b/gst-libs/gst/r2inference/gstvideoinference.c @@ -954,25 +954,6 @@ gst_video_inference_process_model (GstVideoInference * self, GstBuffer * buffer, return ret; } -static gboolean -gst_inference_is_prediction_enabled (GNode * node, gpointer data) -{ - GstInferencePrediction *root = (GstInferencePrediction *) node->data; - gboolean *enabled; - - g_return_val_if_fail (root != NULL, TRUE); - g_return_val_if_fail (data != NULL, TRUE); - - enabled = (gboolean *) data; - - if (root->enabled) { - /* This prediction is enabled, mark enabled flag and terminate traverse */ - *enabled = TRUE; - return TRUE; - } - return FALSE; -} - static void video_inference_notify (GstVideoInference * self, GstBuffer * model_buffer, GstMeta * meta_model[2], GstBuffer * bypass_buffer, @@ -1034,17 +1015,16 @@ gst_video_inference_process_bypass (GstVideoInference * self, current_meta = gst_buffer_get_meta (bypass_buffer, gst_inference_meta_api_get_type ()); if (current_meta) { - /* Check if at least one node is enabled to be processed, if not, just forward buffer */ GstInferenceMeta *imeta = (GstInferenceMeta *) current_meta; - gboolean enabled = FALSE; + GList *found = gst_inference_prediction_get_enabled (imeta->prediction); - g_node_traverse (imeta->prediction->predictions, G_LEVEL_ORDER, - G_TRAVERSE_ALL, -1, gst_inference_is_prediction_enabled, &enabled); - if (!enabled) { + if (!found) { GST_INFO_OBJECT (self, "There is no predictions enabled, bypassing processing..."); goto forward_buffer; } + + g_list_free (found); } /* Queue this new buffer at the head and dequeue the older one */ From ba66b9d08080890dc010e9dca3bd585129ac9893 Mon Sep 17 00:00:00 2001 From: Michael Gruner Date: Fri, 24 Jan 2020 15:47:02 -0600 Subject: [PATCH 157/188] Flush pad queue when stopping the pipe --- gst-libs/gst/r2inference/gstvideoinference.c | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/gst-libs/gst/r2inference/gstvideoinference.c b/gst-libs/gst/r2inference/gstvideoinference.c index a441ee95..07a21b42 100644 --- a/gst-libs/gst/r2inference/gstvideoinference.c +++ b/gst-libs/gst/r2inference/gstvideoinference.c @@ -175,6 +175,7 @@ static gboolean video_inference_prepare_postprocess (const GstMetaInfo * static GstMeta *video_inference_transform_meta (GstBuffer * buffer_model, GstVideoInfo * info_model, GstMeta * meta_model, GstBuffer * buffer_bypass, GstVideoInfo * info_bypass); +static void video_inference_flush_queue (GQueue * queue, GMutex * mutex); static guint gst_video_inference_signals[LAST_SIGNAL] = { 0 }; @@ -454,6 +455,9 @@ gst_video_inference_stop (GstVideoInference * self) GST_INFO_OBJECT (self, "Stopping video inference"); + video_inference_flush_queue (priv->model_queue, &priv->mtx_model_queue); + video_inference_flush_queue (priv->bypass_queue, &priv->mtx_bypass_queue); + if (!gst_backend_stop (priv->backend, &err)) { GST_ELEMENT_ERROR (self, LIBRARY, INIT, ("Could not stop the selected backend: (%s)", err->message), (NULL)); @@ -1303,6 +1307,21 @@ gst_video_inference_src_event (GstPad * pad, GstObject * parent, return gst_collect_pads_src_event_default (priv->cpads, pad, event); } +static void +video_inference_flush_queue (GQueue * queue, GMutex * mutex) +{ + GstBuffer *buf = NULL; + + g_return_if_fail (queue); + g_return_if_fail (mutex); + + g_mutex_lock (mutex); + while ((buf = GST_BUFFER_CAST (g_queue_pop_tail (queue)))) { + gst_buffer_unref (buf); + } + g_mutex_unlock (mutex); +} + static void gst_video_inference_finalize (GObject * object) { From 6e6b693fa2511aca4f3a58413b47732de6d0d7a3 Mon Sep 17 00:00:00 2001 From: rrcarlosrodriguez Date: Fri, 24 Jan 2020 16:53:50 -0600 Subject: [PATCH 158/188] Fix bypass buffer forward if model sinkpad is not requested --- gst-libs/gst/r2inference/gstvideoinference.c | 1 + 1 file changed, 1 insertion(+) diff --git a/gst-libs/gst/r2inference/gstvideoinference.c b/gst-libs/gst/r2inference/gstvideoinference.c index 07a21b42..f42e42dc 100644 --- a/gst-libs/gst/r2inference/gstvideoinference.c +++ b/gst-libs/gst/r2inference/gstvideoinference.c @@ -1012,6 +1012,7 @@ gst_video_inference_process_bypass (GstVideoInference * self, if (NULL == priv->sink_model) { GST_LOG_OBJECT (self, "There is no sinkpad for model, forwarding bypass buffer..."); + bypass_buffer = buffer; goto forward_buffer; } From b20a7562a42b3cd2bdeddde8e627d317c544928f Mon Sep 17 00:00:00 2001 From: Michael Gruner Date: Fri, 24 Jan 2020 17:06:01 -0600 Subject: [PATCH 159/188] Do not scale if only a class was appended --- gst-libs/gst/r2inference/gstinferencemeta.c | 10 ++++-- .../gst/r2inference/gstinferenceprediction.c | 34 +++++++++++++------ .../gst/r2inference/gstinferenceprediction.h | 4 ++- 3 files changed, 33 insertions(+), 15 deletions(-) diff --git a/gst-libs/gst/r2inference/gstinferencemeta.c b/gst-libs/gst/r2inference/gstinferencemeta.c index de113193..326b28a9 100644 --- a/gst-libs/gst/r2inference/gstinferencemeta.c +++ b/gst-libs/gst/r2inference/gstinferencemeta.c @@ -201,6 +201,7 @@ gst_inference_meta_transform_existing_meta (GstBuffer * dest, GstMeta * meta, GstInferenceMeta *dmeta, *smeta; GstInferencePrediction *pred = NULL; gboolean ret = TRUE; + gboolean needs_scale = FALSE; g_return_val_if_fail (dest, FALSE); g_return_val_if_fail (meta, FALSE); @@ -223,7 +224,7 @@ gst_inference_meta_transform_existing_meta (GstBuffer * dest, GstMeta * meta, g_return_val_if_reached (FALSE); } - gst_inference_prediction_merge (smeta->prediction, pred); + needs_scale = gst_inference_prediction_merge (smeta->prediction, pred); if (GST_META_TRANSFORM_IS_COPY (type)) { GST_LOG ("Copy detection metadata"); @@ -232,10 +233,13 @@ gst_inference_meta_transform_existing_meta (GstBuffer * dest, GstMeta * meta, goto out; } + if (GST_VIDEO_META_TRANSFORM_IS_SCALE (type)) { - GstVideoMetaTransform *trans = (GstVideoMetaTransform *) data; - gst_inference_prediction_scale_ip (pred, trans->out_info, trans->in_info); + if (needs_scale) { + GstVideoMetaTransform *trans = (GstVideoMetaTransform *) data; + gst_inference_prediction_scale_ip (pred, trans->out_info, trans->in_info); + } goto out; } diff --git a/gst-libs/gst/r2inference/gstinferenceprediction.c b/gst-libs/gst/r2inference/gstinferenceprediction.c index 7ced6633..574bc4ad 100644 --- a/gst-libs/gst/r2inference/gstinferenceprediction.c +++ b/gst-libs/gst/r2inference/gstinferenceprediction.c @@ -57,7 +57,7 @@ static GstInferencePrediction *prediction_scale (const GstInferencePrediction * static void prediction_scale_ip (GstInferencePrediction * self, GstVideoInfo * to, GstVideoInfo * from); static GSList *prediction_get_children_unlocked (GstInferencePrediction * self); -static void prediction_merge (GstInferencePrediction * src, +static gboolean prediction_merge (GstInferencePrediction * src, GstInferencePrediction * dst); static void prediction_get_enabled (GstInferencePrediction * self, GList ** enabled); @@ -680,16 +680,17 @@ classification_merge (GList * src, GList ** dst) } } -static void +static gboolean prediction_merge (GstInferencePrediction * src, GstInferencePrediction * dst) { GSList *src_children = prediction_get_children_unlocked (src); GSList *iter = NULL; GSList *new_children = NULL; + gboolean new_added = FALSE; - g_return_if_fail (src); - g_return_if_fail (dst); - g_return_if_fail (src->prediction_id == dst->prediction_id); + g_return_val_if_fail (src, FALSE); + g_return_val_if_fail (dst, FALSE); + g_return_val_if_fail (src->prediction_id == dst->prediction_id, FALSE); /* Two things might've happened: * 1) A new class was added @@ -714,7 +715,7 @@ prediction_merge (GstInferencePrediction * src, GstInferencePrediction * dst) } /* Recurse into the children */ - prediction_merge (current, found); + new_added = prediction_merge (current, found); gst_inference_prediction_unref (found); } @@ -726,26 +727,37 @@ prediction_merge (GstInferencePrediction * src, GstInferencePrediction * dst) gst_inference_prediction_copy ((GstInferencePrediction *) iter->data); gst_inference_prediction_append (dst, prediction); } + + new_added |= new_children ? TRUE : FALSE; + + g_slist_free_full (new_children, + (GDestroyNotify) gst_inference_prediction_unref); + + return new_added; } -void +gboolean gst_inference_prediction_merge (GstInferencePrediction * src, GstInferencePrediction * dst) { - g_return_if_fail (src); - g_return_if_fail (dst); + gboolean needs_scale = FALSE; + + g_return_val_if_fail (src, FALSE); + g_return_val_if_fail (dst, FALSE); if (src == dst) { - return; + return needs_scale; } GST_INFERENCE_PREDICTION_LOCK (src); GST_INFERENCE_PREDICTION_LOCK (dst); - prediction_merge (src, dst); + needs_scale = prediction_merge (src, dst); GST_INFERENCE_PREDICTION_UNLOCK (dst); GST_INFERENCE_PREDICTION_UNLOCK (src); + + return needs_scale; } static void diff --git a/gst-libs/gst/r2inference/gstinferenceprediction.h b/gst-libs/gst/r2inference/gstinferenceprediction.h index b9fd0bf7..79a66640 100644 --- a/gst-libs/gst/r2inference/gstinferenceprediction.h +++ b/gst-libs/gst/r2inference/gstinferenceprediction.h @@ -247,8 +247,10 @@ GList * gst_inference_prediction_get_enabled (GstInferencePrediction * self); * @dst: the destination prediction * * Copies the extra information from src to dst. + * + * Returns: TRUE if new sub-predictions were added, FALSE otherwise. */ -void gst_inference_prediction_merge (GstInferencePrediction * src, GstInferencePrediction * dst); +gboolean gst_inference_prediction_merge (GstInferencePrediction * src, GstInferencePrediction * dst); /** * GST_INFERENCE_PREDICTION_LOCK: From 7caeb704742c39e7efcaf7587c51f7ec1b06995a Mon Sep 17 00:00:00 2001 From: Michael Gruner Date: Wed, 29 Jan 2020 21:46:04 -0600 Subject: [PATCH 160/188] Send a gap event if no buffer is to be pushed --- gst/inferencecrop/gstdetectioncrop.cc | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/gst/inferencecrop/gstdetectioncrop.cc b/gst/inferencecrop/gstdetectioncrop.cc index 6505509f..3b2a7592 100644 --- a/gst/inferencecrop/gstdetectioncrop.cc +++ b/gst/inferencecrop/gstdetectioncrop.cc @@ -93,7 +93,8 @@ enum { struct _GstDetectionCrop { GstBin parent; - GstPad *pad; + GstPad *sinkpad; + GstPad *srcpad; CropElement *element; gint width_ratio; gint height_ratio; @@ -149,7 +150,7 @@ gst_detection_crop_init (GstDetectionCrop *self) { GstElement *element; GstPad *sinkpad, *sinkgpad, *srcpad, *srcgpad; - self->pad = NULL; + self->sinkpad = NULL; self->element = new VideoCrop (); self->width_ratio = PROP_CROP_RATIO_DEFAULT_WIDTH; self->height_ratio = PROP_CROP_RATIO_DEFAULT_HEIGHT; @@ -165,7 +166,7 @@ gst_detection_crop_init (GstDetectionCrop *self) { sinkpad = self->element->GetSinkPad (); g_return_if_fail (sinkpad); - self->pad = GST_PAD(gst_object_ref(sinkpad)); + self->sinkpad = GST_PAD(gst_object_ref(sinkpad)); sinkgpad = gst_ghost_pad_new ("sink", sinkpad); gst_pad_set_active (sinkgpad, TRUE); @@ -179,6 +180,8 @@ gst_detection_crop_init (GstDetectionCrop *self) { srcpad = self->element->GetSrcPad (); g_return_if_fail (srcpad); + self->srcpad = GST_PAD(gst_object_ref(srcpad)); + srcgpad = gst_ghost_pad_new ("src", srcpad); gst_pad_set_active (srcgpad, TRUE); gst_element_add_pad (GST_ELEMENT (self), srcgpad); @@ -194,8 +197,10 @@ gst_detection_crop_finalize (GObject *object) { g_return_if_fail(self); delete (self->element); - gst_object_unref (self->pad); - self->pad = NULL; + gst_object_unref (self->sinkpad); + self->sinkpad = NULL; + gst_object_unref (self->srcpad); + self->srcpad = NULL; G_OBJECT_CLASS (gst_detection_crop_parent_class)->finalize (object); } @@ -391,6 +396,7 @@ gst_detection_crop_new_buffer (GstPad *pad, GstPadProbeInfo *info, GstPadProbeReturn ret = GST_PAD_PROBE_DROP; GList *list = NULL; GList *iter = NULL; + gboolean gap = TRUE; GST_OBJECT_LOCK (self); crop_width_ratio = self->width_ratio; @@ -441,15 +447,21 @@ gst_detection_crop_new_buffer (GstPad *pad, GstPadProbeInfo *info, dmeta->prediction->bbox.width = self->width - right - left; dmeta->prediction->bbox.height = self->height - top - bottom; - if (GST_FLOW_OK != gst_pad_chain(self->pad, croped_buffer)) { + if (GST_FLOW_OK != gst_pad_chain(self->sinkpad, croped_buffer)) { GST_ELEMENT_ERROR(self, CORE, FAILED, ("Failed to push a new buffer into crop element"), (NULL)); } + gap = FALSE; } ret = GST_PAD_PROBE_DROP; out: + if (gap) { + gst_pad_push_event (self->srcpad, gst_event_new_gap (GST_BUFFER_TIMESTAMP (buffer), + GST_BUFFER_DURATION (buffer))); + } + return ret; } @@ -459,7 +471,7 @@ plugin_init (GstPlugin *plugin) { ret = gst_element_register (plugin, "detectioncrop", GST_RANK_NONE, - GST_TYPE_DETECTION_CROP); + GST_TYPE_DETECTION_CROP); return ret; } From 91fddc89505555865606dd1eb88c40946d98771f Mon Sep 17 00:00:00 2001 From: Michael Gruner Date: Wed, 29 Jan 2020 21:47:35 -0600 Subject: [PATCH 161/188] Lock videoinference pads in non-waiting mode --- gst-libs/gst/r2inference/gstvideoinference.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gst-libs/gst/r2inference/gstvideoinference.c b/gst-libs/gst/r2inference/gstvideoinference.c index f42e42dc..96dbd662 100644 --- a/gst-libs/gst/r2inference/gstvideoinference.c +++ b/gst-libs/gst/r2inference/gstvideoinference.c @@ -536,7 +536,7 @@ gst_video_inference_create_pad (GstVideoInference * self, *data = (GstVideoInferencePad *) gst_collect_pads_add_pad (priv->cpads, pad, - sizeof (GstVideoInferencePad), NULL, TRUE); + sizeof (GstVideoInferencePad), NULL, FALSE); if (NULL == *data) { GST_ERROR_OBJECT (self, "Unable to add pad %s to collect pads", name); goto free_pad; From fa2bafdc247eed883de10ca48d47e78c45e370b1 Mon Sep 17 00:00:00 2001 From: Michael Gruner Date: Thu, 30 Jan 2020 10:25:34 -0600 Subject: [PATCH 162/188] Free linked lists after usage --- gst-libs/gst/r2inference/gstinferenceprediction.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/gst-libs/gst/r2inference/gstinferenceprediction.c b/gst-libs/gst/r2inference/gstinferenceprediction.c index 574bc4ad..35bda23c 100644 --- a/gst-libs/gst/r2inference/gstinferenceprediction.c +++ b/gst-libs/gst/r2inference/gstinferenceprediction.c @@ -271,6 +271,8 @@ prediction_children_to_string (GstInferencePrediction * self, gint level) g_free (child); } + g_slist_free (subpreds); + return g_string_free (string, FALSE); } @@ -421,14 +423,11 @@ static void prediction_free (GstInferencePrediction * self) { GSList *children = prediction_get_children_unlocked (self); - GSList *iter = NULL; - for (iter = children; iter != NULL; iter = g_slist_next (iter)) { - GstInferencePrediction *child = (GstInferencePrediction *) iter->data; - - gst_inference_prediction_unref (child); - } + /* Free all children recursively */ + g_slist_free_full (children, (GDestroyNotify) gst_inference_prediction_unref); + /* Now free our classifications */ g_list_free_full (self->classifications, (GDestroyNotify) gst_inference_classification_unref); self->classifications = NULL; @@ -730,6 +729,7 @@ prediction_merge (GstInferencePrediction * src, GstInferencePrediction * dst) new_added |= new_children ? TRUE : FALSE; + g_slist_free (src_children); g_slist_free_full (new_children, (GDestroyNotify) gst_inference_prediction_unref); From ddcb20089a976471bde977ee2cd9a95de9fd7a09 Mon Sep 17 00:00:00 2001 From: Michael Gruner Date: Thu, 30 Jan 2020 10:26:03 -0600 Subject: [PATCH 163/188] Unref prediction leaked reference --- gst-libs/gst/r2inference/gstvideoinference.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gst-libs/gst/r2inference/gstvideoinference.c b/gst-libs/gst/r2inference/gstvideoinference.c index 96dbd662..fb0e871e 100644 --- a/gst-libs/gst/r2inference/gstvideoinference.c +++ b/gst-libs/gst/r2inference/gstvideoinference.c @@ -1081,6 +1081,8 @@ gst_video_inference_process_bypass (GstVideoInference * self, g_queue_push_tail (priv->model_queue, (gpointer) model_buffer); g_mutex_unlock (&priv->mtx_model_queue); goto forward_buffer; + } else { + gst_inference_prediction_unref (root_bypass); } } From 94da57ee9e962a2aeef7afdd13195d8bae2f5612 Mon Sep 17 00:00:00 2001 From: Michael Gruner Date: Fri, 31 Jan 2020 11:02:22 -0600 Subject: [PATCH 164/188] Fix inferencebaseoverlay debug category --- gst-libs/gst/opencv/gstinferencebaseoverlay.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gst-libs/gst/opencv/gstinferencebaseoverlay.c b/gst-libs/gst/opencv/gstinferencebaseoverlay.c index f790e40f..6f1dabe3 100644 --- a/gst-libs/gst/opencv/gstinferencebaseoverlay.c +++ b/gst-libs/gst/opencv/gstinferencebaseoverlay.c @@ -99,7 +99,7 @@ gst_inference_base_overlay_transform_frame_ip (GstVideoFilter * trans, G_DEFINE_TYPE_WITH_CODE (GstInferenceBaseOverlay, gst_inference_base_overlay, GST_TYPE_VIDEO_FILTER, GST_DEBUG_CATEGORY_INIT (gst_inference_base_overlay_debug_category, - "inferenceoverlay", 0, "debug category for inferenceoverlay class"); + "inferencebaseoverlay", 0, "debug category for inferenceoverlay class"); G_ADD_PRIVATE (GstInferenceBaseOverlay)); #define GST_INFERENCE_BASE_OVERLAY_PRIVATE(self) \ From ce6491d9bf98d08866bf0659a654ea937b772993 Mon Sep 17 00:00:00 2001 From: Michael Gruner Date: Fri, 31 Jan 2020 11:04:45 -0600 Subject: [PATCH 165/188] Pass in stride to the cvMat constructor to account for odd channels --- ext/opencv/gstclassificationoverlay.cc | 32 ++++---------- ext/opencv/gstdetectionoverlay.cc | 25 +++-------- ext/opencv/gstinferenceoverlay.cc | 42 +++++-------------- gst-libs/gst/opencv/Makefile.am | 8 +++- ...seoverlay.c => gstinferencebaseoverlay.cc} | 42 +++++++++++++++---- gst-libs/gst/opencv/gstinferencebaseoverlay.h | 10 ++++- 6 files changed, 72 insertions(+), 87 deletions(-) rename gst-libs/gst/opencv/{gstinferencebaseoverlay.c => gstinferencebaseoverlay.cc} (90%) diff --git a/ext/opencv/gstclassificationoverlay.cc b/ext/opencv/gstclassificationoverlay.cc index 3528f3fe..45ab304c 100644 --- a/ext/opencv/gstclassificationoverlay.cc +++ b/ext/opencv/gstclassificationoverlay.cc @@ -39,10 +39,10 @@ GST_DEBUG_CATEGORY_STATIC (gst_classification_overlay_debug_category); #define GST_CAT_DEFAULT gst_classification_overlay_debug_category /* prototypes */ -static GstFlowReturn -gst_classification_overlay_process_meta (GstInferenceBaseOverlay * - inference_overlay, GstVideoFrame * frame, GstMeta * meta, - gdouble font_scale, gint thickness, gchar ** labels_list, gint num_labels, LineStyleBoundingBox style); +static GstFlowReturn gst_classification_overlay_process_meta + (GstInferenceBaseOverlay * inference_overlay, cv::Mat &cv_mat, + GstVideoFrame * frame, GstMeta * meta, gdouble font_scale, gint thickness, + gchar ** labels_list, gint num_labels, LineStyleBoundingBox style); enum { @@ -94,29 +94,16 @@ gst_classification_overlay_init (GstClassificationOverlay * } static GstFlowReturn -gst_classification_overlay_process_meta (GstInferenceBaseOverlay * - inference_overlay, GstVideoFrame * frame, GstMeta * meta, - gdouble font_scale, gint thickness, gchar ** labels_list, gint num_labels, LineStyleBoundingBox style) +gst_classification_overlay_process_meta (GstInferenceBaseOverlay * inference_overlay, + cv::Mat &cv_mat, GstVideoFrame * frame, GstMeta * meta, gdouble font_scale, + gint thickness, gchar ** labels_list, gint num_labels, LineStyleBoundingBox style) { GstClassificationMeta *class_meta; - gint index, i, width, height, channels; + gint index, i; gdouble max, current; - cv::Mat cv_mat; cv::String str; cv::Size size; - switch (GST_VIDEO_FRAME_FORMAT (frame)) { - case GST_VIDEO_FORMAT_RGB: - case GST_VIDEO_FORMAT_BGR: - channels = 3; - break; - default: - channels = 4; - break; - } - width = GST_VIDEO_FRAME_COMP_STRIDE (frame, 0) / channels; - height = GST_VIDEO_FRAME_HEIGHT (frame); - class_meta = (GstClassificationMeta *) meta; /* Get the most probable label */ @@ -134,8 +121,7 @@ gst_classification_overlay_process_meta (GstInferenceBaseOverlay * } else { str = cv::format ("Label #%d prob:%f", index, max); } - cv_mat = cv::Mat (height, width, CV_MAKETYPE (CV_8U, channels), - (char *) frame->data[0]); + /* Put string on screen * 10*font_scale+16 aproximates text's rendered size on screen as a * lineal function to avoid using cv::getTextSize diff --git a/ext/opencv/gstdetectionoverlay.cc b/ext/opencv/gstdetectionoverlay.cc index a0409e61..52882da2 100644 --- a/ext/opencv/gstdetectionoverlay.cc +++ b/ext/opencv/gstdetectionoverlay.cc @@ -50,8 +50,8 @@ GST_DEBUG_CATEGORY_STATIC (gst_detection_overlay_debug_category); #define GST_CAT_DEFAULT gst_detection_overlay_debug_category /* prototypes */ -static GstFlowReturn -gst_detection_overlay_process_meta (GstInferenceBaseOverlay * inference_overlay, +static GstFlowReturn gst_detection_overlay_process_meta + (GstInferenceBaseOverlay * inference_overlay, cv::Mat &cv_mat, GstVideoFrame * frame, GstMeta * meta, gdouble font_scale, gint thickness, gchar ** labels_list, gint num_labels, LineStyleBoundingBox style); @@ -104,31 +104,16 @@ gst_detection_overlay_init (GstDetectionOverlay * detection_overlay) static GstFlowReturn gst_detection_overlay_process_meta (GstInferenceBaseOverlay * inference_overlay, - GstVideoFrame * frame, GstMeta * meta, gdouble font_scale, gint thickness, - gchar ** labels_list, gint num_labels, LineStyleBoundingBox style) + cv::Mat &cv_mat, GstVideoFrame * frame, GstMeta * meta, gdouble font_scale, + gint thickness, gchar ** labels_list, gint num_labels, LineStyleBoundingBox style) { GstDetectionMeta *detect_meta; - gint i, width, height, channels; - cv::Mat cv_mat; + gint i; cv::Size size; cv::String str; BBox box; - switch (GST_VIDEO_FRAME_FORMAT (frame)) { - case GST_VIDEO_FORMAT_RGB: - case GST_VIDEO_FORMAT_BGR: - channels = 3; - break; - default: - channels = 4; - break; - } - width = GST_VIDEO_FRAME_COMP_STRIDE (frame, 0) / channels; - height = GST_VIDEO_FRAME_HEIGHT (frame); - detect_meta = (GstDetectionMeta *) meta; - cv_mat = cv::Mat (height, width, CV_MAKETYPE (CV_8U, channels), - (char *) frame->data[0]); for (i = 0; i < detect_meta->num_boxes; ++i) { box = detect_meta->boxes[i]; if (num_labels > box.label) { diff --git a/ext/opencv/gstinferenceoverlay.cc b/ext/opencv/gstinferenceoverlay.cc index c6a3f77f..bcd48c01 100644 --- a/ext/opencv/gstinferenceoverlay.cc +++ b/ext/opencv/gstinferenceoverlay.cc @@ -25,12 +25,6 @@ #include "gstinferenceoverlay.h" #include "gst/r2inference/gstinferencemeta.h" -#ifdef OCV_VERSION_LT_3_2 -#include "opencv2/highgui/highgui.hpp" -#else -#include "opencv2/imgproc.hpp" -#include "opencv2/highgui.hpp" -#endif #define DEFAULT_LABELS NULL @@ -70,12 +64,12 @@ static void gst_inference_overlay_set_property (GObject * object, guint property_id, const GValue * value, GParamSpec * pspec); static void gst_inference_overlay_get_property (GObject * object, guint property_id, GValue * value, GParamSpec * pspec); -static GstFlowReturn -gst_inference_overlay_process_meta (GstInferenceBaseOverlay *inference_overlay, - GstVideoFrame *frame, GstMeta *meta, gdouble font_scale, gint thickness, - gchar **labels_list, gint num_labels, LineStyleBoundingBox style); -void draw_line(cv::Mat &img, cv::Point pt1, cv::Point pt2, cv::Scalar color, - gint thickness, LineStyleBoundingBox style, gint gap); +static GstFlowReturn gst_inference_overlay_process_meta + (GstInferenceBaseOverlay *inference_overlay, cv::Mat &cv_mat, GstVideoFrame *frame, + GstMeta *meta, gdouble font_scale, gint thickness, gchar **labels_list, + gint num_labels, LineStyleBoundingBox style); +static void draw_line(cv::Mat &img, cv::Point pt1, cv::Point pt2, cv::Scalar color, + gint thickness, LineStyleBoundingBox style, gint gap); struct _GstInferenceOverlay { GstInferenceBaseOverlay @@ -310,33 +304,17 @@ gst_get_meta (GstInferencePrediction *pred, cv::Mat &cv_mat, gdouble font_scale, static GstFlowReturn gst_inference_overlay_process_meta (GstInferenceBaseOverlay *inference_overlay, - GstVideoFrame *frame, GstMeta *meta, gdouble font_scale, gint thickness, - gchar **labels_list, gint num_labels, LineStyleBoundingBox style) { - GstInferenceMeta * - detect_meta; - gint width, height, channels; - cv::Mat cv_mat; + cv::Mat &cv_mat, GstVideoFrame *frame, GstMeta *meta, gdouble font_scale, + gint thickness, gchar **labels_list, gint num_labels, LineStyleBoundingBox style) +{ + GstInferenceMeta *detect_meta; g_return_val_if_fail (inference_overlay != NULL ,GST_FLOW_ERROR); g_return_val_if_fail (frame != NULL ,GST_FLOW_ERROR); g_return_val_if_fail (meta != NULL ,GST_FLOW_ERROR); - switch (GST_VIDEO_FRAME_FORMAT (frame)) { - case GST_VIDEO_FORMAT_RGB: - case GST_VIDEO_FORMAT_BGR: - channels = 3; - break; - default: - channels = 4; - break; - } - width = GST_VIDEO_FRAME_COMP_STRIDE (frame, 0) / channels; - height = GST_VIDEO_FRAME_HEIGHT (frame); - detect_meta = (GstInferenceMeta *) meta; - cv_mat = cv::Mat (height, width, CV_MAKETYPE (CV_8U, channels), - (char *) frame->data[0]); gst_get_meta (detect_meta->prediction, cv_mat, font_scale, thickness, labels_list, num_labels, style); diff --git a/gst-libs/gst/opencv/Makefile.am b/gst-libs/gst/opencv/Makefile.am index 7f94479d..8d83bb38 100644 --- a/gst-libs/gst/opencv/Makefile.am +++ b/gst-libs/gst/opencv/Makefile.am @@ -2,7 +2,7 @@ lib_LTLIBRARIES = libgstinferencebaseoverlay-@GST_API_VERSION@.la libgstinferencebaseoverlay_@GST_API_VERSION@_la_SOURCES= \ - gstinferencebaseoverlay.c + gstinferencebaseoverlay.cc libgstinferencebaseoverlay_@GST_API_VERSION@_la_CFLAGS= \ $(GST_CFLAGS) \ @@ -10,6 +10,12 @@ libgstinferencebaseoverlay_@GST_API_VERSION@_la_CFLAGS= \ $(GST_PLUGINS_BASE_CFLAGS) \ $(R2INFERENCE_CFLAGS) +libgstinferencebaseoverlay_@GST_API_VERSION@_la_CXXFLAGS= \ + $(GST_CFLAGS) \ + $(GST_BASE_CFLAGS) \ + $(GST_PLUGINS_BASE_CFLAGS) \ + $(R2INFERENCE_CFLAGS) + libgstinferencebaseoverlay_@GST_API_VERSION@_la_LIBADD= \ $(GST_LIBS) \ $(GST_BASE_LIBS) \ diff --git a/gst-libs/gst/opencv/gstinferencebaseoverlay.c b/gst-libs/gst/opencv/gstinferencebaseoverlay.cc similarity index 90% rename from gst-libs/gst/opencv/gstinferencebaseoverlay.c rename to gst-libs/gst/opencv/gstinferencebaseoverlay.cc index 6f1dabe3..da4ec2ab 100644 --- a/gst-libs/gst/opencv/gstinferencebaseoverlay.c +++ b/gst-libs/gst/opencv/gstinferencebaseoverlay.cc @@ -201,7 +201,7 @@ gst_inference_base_overlay_set_property (GObject * object, guint property_id, priv->labels); break; case PROP_STYLE: - priv->style = g_value_get_enum (value); + priv->style = (LineStyleBoundingBox)g_value_get_enum (value); GST_DEBUG_OBJECT (inference_overlay, "Changed box style to %d", priv->style); break; @@ -310,20 +310,44 @@ gst_inference_base_overlay_transform_frame_ip (GstVideoFilter * trans, GST_INFERENCE_BASE_OVERLAY_PRIVATE (inference_overlay); GstMeta *meta; GstFlowReturn ret = GST_FLOW_ERROR; + gint width, height, channels; + cv::Mat cv_mat; + gint sizes[2] = { 0, 0 }; + gsize steps[2] = { 0, 0 }; + const gint num_dims = 2; + gchar * data = NULL; + + g_return_val_if_fail (io_class->process_meta != NULL, GST_FLOW_ERROR); meta = gst_buffer_get_meta (frame->buffer, io_class->meta_type); if (NULL == meta) { GST_LOG_OBJECT (trans, "No inference meta found"); ret = GST_FLOW_OK; - } else { - GST_LOG_OBJECT (trans, "Valid inference meta found"); - if (io_class->process_meta != NULL) { - ret = - io_class->process_meta (inference_overlay, frame, meta, - priv->font_scale, priv->thickness, priv->labels_list, - priv->num_labels, priv->style); - } + goto out; } + GST_LOG_OBJECT (trans, "Valid inference meta found"); + channels = GST_VIDEO_FRAME_N_COMPONENTS (frame); + width = GST_VIDEO_FRAME_WIDTH (frame); + height = GST_VIDEO_FRAME_HEIGHT (frame); + data = (gchar *)GST_VIDEO_FRAME_PLANE_DATA (frame, 0); + + sizes[0] = height; + sizes[1] = width; + + /* This is not a mistake, it's oddly inverted */ + steps[1] = height; + steps[0] = GST_VIDEO_FRAME_COMP_STRIDE (frame, 0); + + GST_LOG_OBJECT (trans, "width: %d, height: %d, stride: %" G_GSIZE_FORMAT + ", channels: %d", width, height, steps[0], channels); + + cv_mat = cv::Mat (num_dims, (gint *)sizes, CV_MAKETYPE (CV_8U, channels), data, + (gsize *)steps); + + ret = io_class->process_meta (inference_overlay, cv_mat, frame, meta, priv->font_scale, + priv->thickness, priv->labels_list, priv->num_labels, priv->style); + +out: return ret; } diff --git a/gst-libs/gst/opencv/gstinferencebaseoverlay.h b/gst-libs/gst/opencv/gstinferencebaseoverlay.h index a1713279..32a6effe 100644 --- a/gst-libs/gst/opencv/gstinferencebaseoverlay.h +++ b/gst-libs/gst/opencv/gstinferencebaseoverlay.h @@ -24,6 +24,12 @@ #include #include +#ifdef OCV_VERSION_LT_3_2 +#include "opencv2/highgui/highgui.hpp" +#else +#include "opencv2/imgproc.hpp" +#include "opencv2/highgui.hpp" +#endif G_BEGIN_DECLS @@ -52,8 +58,8 @@ struct _GstInferenceBaseOverlayClass GstVideoFilterClass parent_class; GstFlowReturn (* process_meta) (GstInferenceBaseOverlay * inference_base_overlay, - GstVideoFrame * frame, GstMeta* meta, gdouble font_scale, gint thickness, - gchar **labels_list, gint num_labels, LineStyleBoundingBox style); + cv::Mat &mat, GstVideoFrame * frame, GstMeta* meta, gdouble font_scale, + gint thickness, gchar **labels_list, gint num_labels, LineStyleBoundingBox style); GType meta_type; }; From e371ff66756d02060a3f9a9fd97ce11ba790cd06 Mon Sep 17 00:00:00 2001 From: Michael Gruner Date: Fri, 31 Jan 2020 11:14:41 -0600 Subject: [PATCH 166/188] Indent overlay elements and base class --- ext/opencv/gstclassificationoverlay.cc | 35 ++-- ext/opencv/gstdetectionoverlay.cc | 25 ++- ext/opencv/gstinferenceoverlay.cc | 190 +++++++++--------- .../gst/opencv/gstinferencebaseoverlay.cc | 19 +- 4 files changed, 140 insertions(+), 129 deletions(-) diff --git a/ext/opencv/gstclassificationoverlay.cc b/ext/opencv/gstclassificationoverlay.cc index 45ab304c..8b54c2f6 100644 --- a/ext/opencv/gstclassificationoverlay.cc +++ b/ext/opencv/gstclassificationoverlay.cc @@ -25,24 +25,21 @@ #include "gstclassificationoverlay.h" #include "gst/r2inference/gstinferencemeta.h" -#ifdef OCV_VERSION_LT_3_2 -#include "opencv2/highgui/highgui.hpp" -#else -#include "opencv2/imgproc.hpp" -#include "opencv2/highgui.hpp" -#endif +/* *INDENT-OFF* */ static const cv::Scalar black = cv::Scalar (0, 0, 0, 0); static const cv::Scalar white = cv::Scalar (255, 255, 255, 255); +/* *INDENT-ON* */ GST_DEBUG_CATEGORY_STATIC (gst_classification_overlay_debug_category); #define GST_CAT_DEFAULT gst_classification_overlay_debug_category /* prototypes */ -static GstFlowReturn gst_classification_overlay_process_meta - (GstInferenceBaseOverlay * inference_overlay, cv::Mat &cv_mat, - GstVideoFrame * frame, GstMeta * meta, gdouble font_scale, gint thickness, - gchar ** labels_list, gint num_labels, LineStyleBoundingBox style); +static GstFlowReturn +gst_classification_overlay_process_meta (GstInferenceBaseOverlay * + inference_overlay, cv::Mat & cv_mat, GstVideoFrame * frame, GstMeta * meta, + gdouble font_scale, gint thickness, gchar ** labels_list, gint num_labels, + LineStyleBoundingBox style); enum { @@ -70,7 +67,8 @@ G_DEFINE_TYPE_WITH_CODE (GstClassificationOverlay, gst_classification_overlay, static void gst_classification_overlay_class_init (GstClassificationOverlayClass * klass) { - GstInferenceBaseOverlayClass *io_class = GST_INFERENCE_BASE_OVERLAY_CLASS (klass); + GstInferenceBaseOverlayClass *io_class = + GST_INFERENCE_BASE_OVERLAY_CLASS (klass); gst_element_class_set_static_metadata (GST_ELEMENT_CLASS (klass), "classificationoverlay", "Filter", @@ -94,9 +92,10 @@ gst_classification_overlay_init (GstClassificationOverlay * } static GstFlowReturn -gst_classification_overlay_process_meta (GstInferenceBaseOverlay * inference_overlay, - cv::Mat &cv_mat, GstVideoFrame * frame, GstMeta * meta, gdouble font_scale, - gint thickness, gchar ** labels_list, gint num_labels, LineStyleBoundingBox style) +gst_classification_overlay_process_meta (GstInferenceBaseOverlay * + inference_overlay, cv::Mat & cv_mat, GstVideoFrame * frame, GstMeta * meta, + gdouble font_scale, gint thickness, gchar ** labels_list, gint num_labels, + LineStyleBoundingBox style) { GstClassificationMeta *class_meta; gint index, i; @@ -126,10 +125,10 @@ gst_classification_overlay_process_meta (GstInferenceBaseOverlay * inference_ove * 10*font_scale+16 aproximates text's rendered size on screen as a * lineal function to avoid using cv::getTextSize */ - cv::putText (cv_mat, str, cv::Point (0, 10*font_scale+16), cv::FONT_HERSHEY_PLAIN, - font_scale, white, thickness + (thickness*0.5)); - cv::putText (cv_mat, str, cv::Point (0, 10*font_scale+16), cv::FONT_HERSHEY_PLAIN, - font_scale, black, thickness); + cv::putText (cv_mat, str, cv::Point (0, 10 * font_scale + 16), + cv::FONT_HERSHEY_PLAIN, font_scale, white, thickness + (thickness * 0.5)); + cv::putText (cv_mat, str, cv::Point (0, 10 * font_scale + 16), + cv::FONT_HERSHEY_PLAIN, font_scale, black, thickness); return GST_FLOW_OK; } diff --git a/ext/opencv/gstdetectionoverlay.cc b/ext/opencv/gstdetectionoverlay.cc index 52882da2..1ee31c5b 100644 --- a/ext/opencv/gstdetectionoverlay.cc +++ b/ext/opencv/gstdetectionoverlay.cc @@ -25,13 +25,8 @@ #include "gstdetectionoverlay.h" #include "gst/r2inference/gstinferencemeta.h" -#ifdef OCV_VERSION_LT_3_2 -#include "opencv2/highgui/highgui.hpp" -#else -#include "opencv2/imgproc.hpp" -#include "opencv2/highgui.hpp" -#endif +/* *INDENT-OFF* */ static const cv::Scalar colors[] = { cv::Scalar (254, 254, 254), cv::Scalar (239, 211, 127), cv::Scalar (225, 169, 0), cv::Scalar (211, 127, 254), @@ -44,16 +39,18 @@ static const cv::Scalar colors[] = { cv::Scalar (28, 42, 127), cv::Scalar (14, 84, 0), cv::Scalar (0, 254, 254), cv::Scalar (14, 211, 127) }; +/* *INDENT-ON* */ + #define N_C (sizeof (colors)/sizeof (cv::Scalar)) GST_DEBUG_CATEGORY_STATIC (gst_detection_overlay_debug_category); #define GST_CAT_DEFAULT gst_detection_overlay_debug_category /* prototypes */ -static GstFlowReturn gst_detection_overlay_process_meta - (GstInferenceBaseOverlay * inference_overlay, cv::Mat &cv_mat, - GstVideoFrame * frame, GstMeta * meta, gdouble font_scale, gint thickness, - gchar ** labels_list, gint num_labels, LineStyleBoundingBox style); +static GstFlowReturn gst_detection_overlay_process_meta (GstInferenceBaseOverlay + * inference_overlay, cv::Mat & cv_mat, GstVideoFrame * frame, + GstMeta * meta, gdouble font_scale, gint thickness, gchar ** labels_list, + gint num_labels, LineStyleBoundingBox style); enum { @@ -80,7 +77,8 @@ G_DEFINE_TYPE_WITH_CODE (GstDetectionOverlay, gst_detection_overlay, static void gst_detection_overlay_class_init (GstDetectionOverlayClass * klass) { - GstInferenceBaseOverlayClass *io_class = GST_INFERENCE_BASE_OVERLAY_CLASS (klass); + GstInferenceBaseOverlayClass *io_class = + GST_INFERENCE_BASE_OVERLAY_CLASS (klass); gst_element_class_set_static_metadata (GST_ELEMENT_CLASS (klass), "detectionoverlay", "Filter", @@ -104,8 +102,9 @@ gst_detection_overlay_init (GstDetectionOverlay * detection_overlay) static GstFlowReturn gst_detection_overlay_process_meta (GstInferenceBaseOverlay * inference_overlay, - cv::Mat &cv_mat, GstVideoFrame * frame, GstMeta * meta, gdouble font_scale, - gint thickness, gchar ** labels_list, gint num_labels, LineStyleBoundingBox style) + cv::Mat & cv_mat, GstVideoFrame * frame, GstMeta * meta, gdouble font_scale, + gint thickness, gchar ** labels_list, gint num_labels, + LineStyleBoundingBox style) { GstDetectionMeta *detect_meta; gint i; diff --git a/ext/opencv/gstinferenceoverlay.cc b/ext/opencv/gstinferenceoverlay.cc index bcd48c01..76676906 100644 --- a/ext/opencv/gstinferenceoverlay.cc +++ b/ext/opencv/gstinferenceoverlay.cc @@ -34,9 +34,8 @@ enum PROP_LABELS }; -static const -cv::Scalar -colors[] = { +/* *INDENT-OFF* */ +static const cv::Scalar colors[] = { cv::Scalar (254, 254, 254), cv::Scalar (239, 211, 127), cv::Scalar (225, 169, 0), cv::Scalar (211, 127, 254), cv::Scalar (197, 84, 127), cv::Scalar (183, 42, 0), @@ -48,6 +47,7 @@ colors[] = { cv::Scalar (28, 42, 127), cv::Scalar (14, 84, 0), cv::Scalar (0, 254, 254), cv::Scalar (14, 211, 127) }; +/* *INDENT-ON* */ #define N_C (sizeof (colors)/sizeof (cv::Scalar)) @@ -64,63 +64,64 @@ static void gst_inference_overlay_set_property (GObject * object, guint property_id, const GValue * value, GParamSpec * pspec); static void gst_inference_overlay_get_property (GObject * object, guint property_id, GValue * value, GParamSpec * pspec); -static GstFlowReturn gst_inference_overlay_process_meta - (GstInferenceBaseOverlay *inference_overlay, cv::Mat &cv_mat, GstVideoFrame *frame, - GstMeta *meta, gdouble font_scale, gint thickness, gchar **labels_list, +static GstFlowReturn gst_inference_overlay_process_meta (GstInferenceBaseOverlay + * inference_overlay, cv::Mat & cv_mat, GstVideoFrame * frame, + GstMeta * meta, gdouble font_scale, gint thickness, gchar ** labels_list, gint num_labels, LineStyleBoundingBox style); -static void draw_line(cv::Mat &img, cv::Point pt1, cv::Point pt2, cv::Scalar color, - gint thickness, LineStyleBoundingBox style, gint gap); +static void draw_line (cv::Mat & img, cv::Point pt1, cv::Point pt2, + cv::Scalar color, gint thickness, LineStyleBoundingBox style, gint gap); -struct _GstInferenceOverlay { - GstInferenceBaseOverlay - parent; +struct _GstInferenceOverlay +{ + GstInferenceBaseOverlay parent; }; -struct _GstInferenceOverlayClass { - GstInferenceBaseOverlay - parent; +struct _GstInferenceOverlayClass +{ + GstInferenceBaseOverlay parent; }; /* class initialization */ G_DEFINE_TYPE_WITH_CODE (GstInferenceOverlay, gst_inference_overlay, - GST_TYPE_INFERENCE_BASE_OVERLAY, - GST_DEBUG_CATEGORY_INIT (gst_inference_overlay_debug_category, - "inferenceoverlay", 0, "debug category for inference_overlay element")); + GST_TYPE_INFERENCE_BASE_OVERLAY, + GST_DEBUG_CATEGORY_INIT (gst_inference_overlay_debug_category, + "inferenceoverlay", 0, "debug category for inference_overlay element")); static void -gst_inference_overlay_class_init (GstInferenceOverlayClass *klass) { - GstInferenceBaseOverlayClass * - io_class = GST_INFERENCE_BASE_OVERLAY_CLASS (klass); +gst_inference_overlay_class_init (GstInferenceOverlayClass * klass) +{ + GstInferenceBaseOverlayClass *io_class = + GST_INFERENCE_BASE_OVERLAY_CLASS (klass); GObjectClass *gobject_class = G_OBJECT_CLASS (klass); - + gobject_class->set_property = gst_inference_overlay_set_property; gobject_class->get_property = gst_inference_overlay_get_property; - + gst_element_class_set_static_metadata (GST_ELEMENT_CLASS (klass), - "inferenceoverlay", "Filter", - "Overlays Inferece metadata on input buffer", - "Lenin Torres "); + "inferenceoverlay", "Filter", + "Overlays Inferece metadata on input buffer", + "Lenin Torres "); g_object_class_install_property (gobject_class, PROP_LABELS, g_param_spec_string ("labels", "labels", "(Deprecated) Semicolon separated string containing inference labels.", DEFAULT_LABELS, G_PARAM_READWRITE)); io_class->process_meta = - GST_DEBUG_FUNCPTR (gst_inference_overlay_process_meta); + GST_DEBUG_FUNCPTR (gst_inference_overlay_process_meta); io_class->meta_type = GST_INFERENCE_META_API_TYPE; } static void -gst_inference_overlay_init (GstInferenceOverlay *inference_overlay) { +gst_inference_overlay_init (GstInferenceOverlay * inference_overlay) +{ } void gst_inference_overlay_set_property (GObject * object, guint property_id, const GValue * value, GParamSpec * pspec) { - GstInferenceOverlay *inference_overlay = - GST_INFERENCE_OVERLAY (object); + GstInferenceOverlay *inference_overlay = GST_INFERENCE_OVERLAY (object); GST_DEBUG_OBJECT (inference_overlay, "set_property"); @@ -137,8 +138,7 @@ void gst_inference_overlay_get_property (GObject * object, guint property_id, GValue * value, GParamSpec * pspec) { - GstInferenceOverlay *inference_overlay = - GST_INFERENCE_OVERLAY (object); + GstInferenceOverlay *inference_overlay = GST_INFERENCE_OVERLAY (object); GST_DEBUG_OBJECT (inference_overlay, "get_property"); @@ -151,28 +151,29 @@ gst_inference_overlay_get_property (GObject * object, guint property_id, } } -void draw_line(cv::Mat &img, cv::Point pt1, cv::Point pt2, cv::Scalar color, - gint thickness, LineStyleBoundingBox style, gint gap) { - +void +draw_line (cv::Mat & img, cv::Point pt1, cv::Point pt2, cv::Scalar color, + gint thickness, LineStyleBoundingBox style, gint gap) +{ gfloat dx = pt1.x - pt2.x; gfloat dy = pt1.y - pt2.y; - gfloat dist = sqrt(dx * dx + dy * dy); + gfloat dist = sqrt (dx * dx + dy * dy); - std::vector pts; + std::vector < cv::Point > pts; for (gint i = 0; i < dist; i += gap) { - gfloat r = (gfloat)i / dist; - gint x = gint((pt1.x * (1.0 - r) + pt2.x * r) + .5); - gint y = gint((pt1.y * (1.0 - r) + pt2.y * r) + .5); - cv::Point p = cv::Point(x, y); - pts.push_back(p); + gfloat r = (gfloat) i / dist; + gint x = gint ((pt1.x * (1.0 - r) + pt2.x * r) + .5); + gint y = gint ((pt1.y * (1.0 - r) + pt2.y * r) + .5); + cv::Point p = cv::Point (x, y); + pts.push_back (p); } - gint pts_size = pts.size(); + gint pts_size = pts.size (); if (DOTTED == style) { for (gint i = 0; i < pts_size; i++) { - cv::circle(img, pts[i], thickness, color, -1); + cv::circle (img, pts[i], thickness, color, -1); } } else { cv::Point s = pts[0]; @@ -183,16 +184,17 @@ void draw_line(cv::Mat &img, cv::Point pt1, cv::Point pt2, cv::Scalar color, e = pts[i]; if (1 == i % 2) { - cv::line(img, s, e, color, thickness); + cv::line (img, s, e, color, thickness); } } } } static void -gst_get_meta (GstInferencePrediction *pred, cv::Mat &cv_mat, gdouble font_scale, - gint thickness, - gchar **labels_list, gint num_labels, LineStyleBoundingBox style) { +gst_get_meta (GstInferencePrediction * pred, cv::Mat & cv_mat, + gdouble font_scale, gint thickness, gchar ** labels_list, gint num_labels, + LineStyleBoundingBox style) +{ cv::Size size; cv::String label; GList *iter = NULL; @@ -203,16 +205,19 @@ gst_get_meta (GstInferencePrediction *pred, cv::Mat &cv_mat, gdouble font_scale, gint classes = 0; gdouble alpha = 0.5; cv::Mat alpha_overlay; - gint width, height,x, y = 0; + gint width, height, x, y = 0; + g_return_if_fail (pred != NULL); - list = gst_inference_prediction_get_children(pred); + list = gst_inference_prediction_get_children (pred); - for (tree_iter = list; tree_iter != NULL; tree_iter = g_slist_next(tree_iter)) { - GstInferencePrediction *predict = (GstInferencePrediction *)tree_iter->data; + for (tree_iter = list; tree_iter != NULL; + tree_iter = g_slist_next (tree_iter)) { + GstInferencePrediction *predict = + (GstInferencePrediction *) tree_iter->data; gst_get_meta (predict, cv_mat, font_scale, thickness, - labels_list, num_labels, style); + labels_list, num_labels, style); } if (!pred->enabled) { @@ -222,77 +227,82 @@ gst_get_meta (GstInferencePrediction *pred, cv::Mat &cv_mat, gdouble font_scale, box = pred->bbox; - for (iter = pred->classifications; iter != NULL; iter = g_list_next(iter)) { + for (iter = pred->classifications; iter != NULL; iter = g_list_next (iter)) { GstInferenceClassification *classification = (GstInferenceClassification *) iter->data; classes++; - if (classification->num_classes > classification->class_id && classification->labels != NULL) { - label = cv::format ("%s Prob: %f", classification->labels[classification->class_id], - classification->class_prob); + if (classification->num_classes > classification->class_id + && classification->labels != NULL) { + label = + cv::format ("%s Prob: %f", + classification->labels[classification->class_id], + classification->class_prob); } else { label = cv::format ("Label #%d Prob: %f", classification->class_id, - classification->class_prob); + classification->class_prob); } - cv::putText (cv_mat, label, cv::Point (box.x + box.width, box.y + classes * OVERLAY_WIDTH), - cv::FONT_HERSHEY_PLAIN, font_scale, cv::Scalar::all(0), thickness); + cv::putText (cv_mat, label, cv::Point (box.x + box.width, + box.y + classes * OVERLAY_WIDTH), cv::FONT_HERSHEY_PLAIN, + font_scale, cv::Scalar::all (0), thickness); } cv::Size text = cv::getTextSize (label, cv::FONT_HERSHEY_PLAIN, font_scale, - thickness, 0); + thickness, 0); if ((box.x + box.width) < 0) { x = 0; - } else if ((int)(box.x + box.width) >= cv_mat.cols) { + } else if ((int) (box.x + box.width) >= cv_mat.cols) { x = cv_mat.cols - 1; } else { x = box.x + box.width; } - if ((int)(x + box.width + text.width) >= cv_mat.cols) { + if ((int) (x + box.width + text.width) >= cv_mat.cols) { width = cv_mat.cols - x - 1; } else { width = text.width; } - if ((int)(box.y + OVERLAY_HEIGHT * classes ) >= cv_mat.rows) { + if ((int) (box.y + OVERLAY_HEIGHT * classes) >= cv_mat.rows) { y = cv_mat.rows - 1; - }else if ((int)box.y < 0) { + } else if ((int) box.y < 0) { y = 1; - }else { + } else { y = box.y; } - if ((int)(y + (OVERLAY_HEIGHT * classes)) >= cv_mat.rows) { + if ((int) (y + (OVERLAY_HEIGHT * classes)) >= cv_mat.rows) { height = cv_mat.rows - y - 1; } else { - height = OVERLAY_HEIGHT * classes; + height = OVERLAY_HEIGHT * classes; } if (width && height) { - cv::Mat rectangle (height, width, cv_mat.type(), cv::Scalar (255, 255, 255) ); + cv::Mat rectangle (height, width, cv_mat.type (), cv::Scalar (255, 255, + 255)); cv::Rect roi (x, y, width, height); cv::addWeighted (cv_mat (roi), alpha, rectangle, 1.0 - alpha, 0.0, - cv_mat (roi) ); + cv_mat (roi)); } - if (FALSE == G_NODE_IS_ROOT(pred->predictions)) { + if (FALSE == G_NODE_IS_ROOT (pred->predictions)) { if (0 == style) { cv::rectangle (cv_mat, cv::Point (box.x, box.y), - cv::Point (box.x + box.width, box.y + box.height), - colors[CHOSEN_COLOR % N_C], thickness); - }else { + cv::Point (box.x + box.width, box.y + box.height), + colors[CHOSEN_COLOR % N_C], thickness); + } else { draw_line (cv_mat, cv::Point (box.x, box.y), - cv::Point (box.x + box.width, box.y), - colors[CHOSEN_COLOR % N_C], thickness, style, LINES_GAP); + cv::Point (box.x + box.width, box.y), + colors[CHOSEN_COLOR % N_C], thickness, style, LINES_GAP); draw_line (cv_mat, cv::Point (box.x, box.y), - cv::Point (box.x, box.y + box.height), - colors[CHOSEN_COLOR % N_C], thickness, style, LINES_GAP); + cv::Point (box.x, box.y + box.height), + colors[CHOSEN_COLOR % N_C], thickness, style, LINES_GAP); draw_line (cv_mat, cv::Point (box.x + box.width, box.y), - cv::Point (box.x + box.width, box.y + box.height), - colors[CHOSEN_COLOR % N_C], thickness, style, LINES_GAP); + cv::Point (box.x + box.width, box.y + box.height), + colors[CHOSEN_COLOR % N_C], thickness, style, LINES_GAP); draw_line (cv_mat, cv::Point (box.x, box.y + box.height), - cv::Point (box.x + box.width, box.y + box.height), - colors[CHOSEN_COLOR % N_C], thickness, style, LINES_GAP); + cv::Point (box.x + box.width, box.y + box.height), + colors[CHOSEN_COLOR % N_C], thickness, style, LINES_GAP); } } @@ -301,22 +311,22 @@ gst_get_meta (GstInferencePrediction *pred, cv::Mat &cv_mat, gdouble font_scale, } } -static -GstFlowReturn -gst_inference_overlay_process_meta (GstInferenceBaseOverlay *inference_overlay, - cv::Mat &cv_mat, GstVideoFrame *frame, GstMeta *meta, gdouble font_scale, - gint thickness, gchar **labels_list, gint num_labels, LineStyleBoundingBox style) +static GstFlowReturn +gst_inference_overlay_process_meta (GstInferenceBaseOverlay * inference_overlay, + cv::Mat & cv_mat, GstVideoFrame * frame, GstMeta * meta, gdouble font_scale, + gint thickness, gchar ** labels_list, gint num_labels, + LineStyleBoundingBox style) { GstInferenceMeta *detect_meta; - g_return_val_if_fail (inference_overlay != NULL ,GST_FLOW_ERROR); - g_return_val_if_fail (frame != NULL ,GST_FLOW_ERROR); - g_return_val_if_fail (meta != NULL ,GST_FLOW_ERROR); + g_return_val_if_fail (inference_overlay != NULL, GST_FLOW_ERROR); + g_return_val_if_fail (frame != NULL, GST_FLOW_ERROR); + g_return_val_if_fail (meta != NULL, GST_FLOW_ERROR); detect_meta = (GstInferenceMeta *) meta; gst_get_meta (detect_meta->prediction, cv_mat, font_scale, thickness, - labels_list, num_labels, style); + labels_list, num_labels, style); return GST_FLOW_OK; } diff --git a/gst-libs/gst/opencv/gstinferencebaseoverlay.cc b/gst-libs/gst/opencv/gstinferencebaseoverlay.cc index da4ec2ab..9afb38d1 100644 --- a/gst-libs/gst/opencv/gstinferencebaseoverlay.cc +++ b/gst-libs/gst/opencv/gstinferencebaseoverlay.cc @@ -201,7 +201,7 @@ gst_inference_base_overlay_set_property (GObject * object, guint property_id, priv->labels); break; case PROP_STYLE: - priv->style = (LineStyleBoundingBox)g_value_get_enum (value); + priv->style = (LineStyleBoundingBox) g_value_get_enum (value); GST_DEBUG_OBJECT (inference_overlay, "Changed box style to %d", priv->style); break; @@ -310,12 +310,12 @@ gst_inference_base_overlay_transform_frame_ip (GstVideoFilter * trans, GST_INFERENCE_BASE_OVERLAY_PRIVATE (inference_overlay); GstMeta *meta; GstFlowReturn ret = GST_FLOW_ERROR; - gint width, height, channels; + gint width, height, channels; cv::Mat cv_mat; gint sizes[2] = { 0, 0 }; gsize steps[2] = { 0, 0 }; const gint num_dims = 2; - gchar * data = NULL; + gchar *data = NULL; g_return_val_if_fail (io_class->process_meta != NULL, GST_FLOW_ERROR); @@ -330,7 +330,7 @@ gst_inference_base_overlay_transform_frame_ip (GstVideoFilter * trans, channels = GST_VIDEO_FRAME_N_COMPONENTS (frame); width = GST_VIDEO_FRAME_WIDTH (frame); height = GST_VIDEO_FRAME_HEIGHT (frame); - data = (gchar *)GST_VIDEO_FRAME_PLANE_DATA (frame, 0); + data = (gchar *) GST_VIDEO_FRAME_PLANE_DATA (frame, 0); sizes[0] = height; sizes[1] = width; @@ -342,11 +342,14 @@ gst_inference_base_overlay_transform_frame_ip (GstVideoFilter * trans, GST_LOG_OBJECT (trans, "width: %d, height: %d, stride: %" G_GSIZE_FORMAT ", channels: %d", width, height, steps[0], channels); - cv_mat = cv::Mat (num_dims, (gint *)sizes, CV_MAKETYPE (CV_8U, channels), data, - (gsize *)steps); + cv_mat = + cv::Mat (num_dims, (gint *) sizes, CV_MAKETYPE (CV_8U, channels), data, + (gsize *) steps); - ret = io_class->process_meta (inference_overlay, cv_mat, frame, meta, priv->font_scale, - priv->thickness, priv->labels_list, priv->num_labels, priv->style); + ret = + io_class->process_meta (inference_overlay, cv_mat, frame, meta, + priv->font_scale, priv->thickness, priv->labels_list, priv->num_labels, + priv->style); out: return ret; From 65d2046ad0177a6dc1163a4a753a23d0ffca5009 Mon Sep 17 00:00:00 2001 From: Michael Gruner Date: Fri, 31 Jan 2020 11:47:22 -0600 Subject: [PATCH 167/188] Use pixel stride to query channels to get 4 on RGBx (instead of 3) --- gst-libs/gst/opencv/gstinferencebaseoverlay.cc | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/gst-libs/gst/opencv/gstinferencebaseoverlay.cc b/gst-libs/gst/opencv/gstinferencebaseoverlay.cc index 9afb38d1..3bde2118 100644 --- a/gst-libs/gst/opencv/gstinferencebaseoverlay.cc +++ b/gst-libs/gst/opencv/gstinferencebaseoverlay.cc @@ -327,7 +327,9 @@ gst_inference_base_overlay_transform_frame_ip (GstVideoFilter * trans, } GST_LOG_OBJECT (trans, "Valid inference meta found"); - channels = GST_VIDEO_FRAME_N_COMPONENTS (frame); + + /* Use pixel stride instead of num components because RGBx reports 3 channels */ + channels = GST_VIDEO_FRAME_COMP_PSTRIDE (frame, 0); width = GST_VIDEO_FRAME_WIDTH (frame); height = GST_VIDEO_FRAME_HEIGHT (frame); data = (gchar *) GST_VIDEO_FRAME_PLANE_DATA (frame, 0); @@ -336,7 +338,7 @@ gst_inference_base_overlay_transform_frame_ip (GstVideoFilter * trans, sizes[1] = width; /* This is not a mistake, it's oddly inverted */ - steps[1] = height; + steps[1] = channels; steps[0] = GST_VIDEO_FRAME_COMP_STRIDE (frame, 0); GST_LOG_OBJECT (trans, "width: %d, height: %d, stride: %" G_GSIZE_FORMAT From b453027d38ca85205924ef50d83824b13315693c Mon Sep 17 00:00:00 2001 From: tvlenin Date: Mon, 20 Jan 2020 15:47:34 -0600 Subject: [PATCH 168/188] Add new meta support to inception models --- ext/r2inference/gstinceptionv1.c | 66 ++++++++++++++++++++++++++++++-- ext/r2inference/gstinceptionv3.c | 65 +++++++++++++++++++++++++++++-- ext/r2inference/gstinceptionv4.c | 58 ++++++++++++++++++++++++++-- 3 files changed, 177 insertions(+), 12 deletions(-) diff --git a/ext/r2inference/gstinceptionv1.c b/ext/r2inference/gstinceptionv1.c index 4e3949a7..d6924889 100644 --- a/ext/r2inference/gstinceptionv1.c +++ b/ext/r2inference/gstinceptionv1.c @@ -60,6 +60,13 @@ static gboolean gst_inceptionv1_postprocess (GstVideoInference * vi, const gpointer prediction, gsize predsize, GstMeta * meta_model[2], GstVideoInfo * info_model, gboolean * valid_prediction, gchar ** labels_list, gint num_labels); +static gboolean gst_inceptionv1_postprocess_old (GstVideoInference * vi, + const gpointer prediction, gsize predsize, GstMeta * meta_model, + GstVideoInfo * info_model, gboolean * valid_prediction); +static gboolean gst_inceptionv1_postprocess_new (GstVideoInference * vi, + const gpointer prediction, gsize predsize, GstMeta * meta_model, + GstVideoInfo * info_model, gboolean * valid_prediction, + gchar ** labels_list, gint num_labels); static gboolean gst_inceptionv1_start (GstVideoInference * vi); static gboolean gst_inceptionv1_stop (GstVideoInference * vi); @@ -145,12 +152,13 @@ gst_inceptionv1_preprocess (GstVideoInference * vi, } static gboolean -gst_inceptionv1_postprocess (GstVideoInference * vi, const gpointer prediction, - gsize predsize, GstMeta * meta_model[2], GstVideoInfo * info_model, - gboolean * valid_prediction, gchar ** labels_list, gint num_labels) +gst_inceptionv1_postprocess_old (GstVideoInference * vi, + const gpointer prediction, gsize predsize, GstMeta * meta_model, + GstVideoInfo * info_model, gboolean * valid_prediction) { - GstClassificationMeta *class_meta = (GstClassificationMeta *) meta_model[0]; + GstClassificationMeta *class_meta = (GstClassificationMeta *) meta_model; GstDebugLevel gst_debug_level = GST_LEVEL_LOG; + GST_LOG_OBJECT (vi, "Postprocess"); gst_fill_classification_meta (class_meta, prediction, predsize); @@ -162,6 +170,56 @@ gst_inceptionv1_postprocess (GstVideoInference * vi, const gpointer prediction, return TRUE; } +static gboolean +gst_inceptionv1_postprocess_new (GstVideoInference * vi, + const gpointer prediction, gsize predsize, GstMeta * meta_model, + GstVideoInfo * info_model, gboolean * valid_prediction, + gchar ** labels_list, gint num_labels) +{ + GstInferenceMeta *imeta = NULL; + GstInferenceClassification *c = NULL; + GstInferencePrediction *root = NULL; + + g_return_val_if_fail (vi != NULL, FALSE); + g_return_val_if_fail (meta_model != NULL, FALSE); + g_return_val_if_fail (info_model != NULL, FALSE); + + GST_LOG_OBJECT (vi, "Postprocess Meta"); + + imeta = (GstInferenceMeta *) meta_model; + + root = imeta->prediction; + if (!root) { + GST_ERROR_OBJECT (vi, "Prediction is not part of the Inference Meta"); + return FALSE; + } + + c = gst_create_class_from_prediction (vi, prediction, predsize, labels_list, + num_labels); + gst_inference_prediction_append_classification (root, c); + gst_inference_print_predictions (vi, gst_inceptionv1_debug_category, imeta); + + *valid_prediction = TRUE; + return TRUE; +} + +static gboolean +gst_inceptionv1_postprocess (GstVideoInference * vi, const gpointer prediction, + gsize predsize, GstMeta * meta_model[2], GstVideoInfo * info_model, + gboolean * valid_prediction, gchar ** labels_list, gint num_labels) +{ + gboolean ret = TRUE; + + ret &= + gst_inceptionv1_postprocess_old (vi, prediction, predsize, meta_model[0], + info_model, valid_prediction); + ret &= + gst_inceptionv1_postprocess_new (vi, prediction, predsize, meta_model[1], + info_model, valid_prediction, labels_list, num_labels); + + return ret; +} + static gboolean gst_inceptionv1_start (GstVideoInference * vi) { diff --git a/ext/r2inference/gstinceptionv3.c b/ext/r2inference/gstinceptionv3.c index 741cdde0..7f89b0a3 100644 --- a/ext/r2inference/gstinceptionv3.c +++ b/ext/r2inference/gstinceptionv3.c @@ -59,6 +59,13 @@ static gboolean gst_inceptionv3_postprocess (GstVideoInference * vi, const gpointer prediction, gsize predsize, GstMeta * meta_model[2], GstVideoInfo * info_model, gboolean * valid_prediction, gchar ** labels_list, gint num_labels); +static gboolean gst_inceptionv3_postprocess_old (GstVideoInference * vi, + const gpointer prediction, gsize predsize, GstMeta * meta_model, + GstVideoInfo * info_model, gboolean * valid_prediction); +static gboolean gst_inceptionv3_postprocess_new (GstVideoInference * vi, + const gpointer prediction, gsize predsize, GstMeta * meta_model, + GstVideoInfo * info_model, gboolean * valid_prediction, + gchar ** labels_list, gint num_labels); static gboolean gst_inceptionv3_start (GstVideoInference * vi); static gboolean gst_inceptionv3_stop (GstVideoInference * vi); @@ -144,11 +151,11 @@ gst_inceptionv3_preprocess (GstVideoInference * vi, } static gboolean -gst_inceptionv3_postprocess (GstVideoInference * vi, const gpointer prediction, - gsize predsize, GstMeta * meta_model[2], GstVideoInfo * info_model, - gboolean * valid_prediction, gchar ** labels_list, gint num_labels) +gst_inceptionv3_postprocess_old (GstVideoInference * vi, + const gpointer prediction, gsize predsize, GstMeta * meta_model, + GstVideoInfo * info_model, gboolean * valid_prediction) { - GstClassificationMeta *class_meta = (GstClassificationMeta *) meta_model[0]; + GstClassificationMeta *class_meta = (GstClassificationMeta *) meta_model; GstDebugLevel gst_debug_level = GST_LEVEL_LOG; GST_LOG_OBJECT (vi, "Postprocess"); @@ -162,6 +169,56 @@ gst_inceptionv3_postprocess (GstVideoInference * vi, const gpointer prediction, return TRUE; } +static gboolean +gst_inceptionv3_postprocess_new (GstVideoInference * vi, + const gpointer prediction, gsize predsize, GstMeta * meta_model, + GstVideoInfo * info_model, gboolean * valid_prediction, + gchar ** labels_list, gint num_labels) +{ + GstInferenceMeta *imeta = NULL; + GstInferenceClassification *c = NULL; + GstInferencePrediction *root = NULL; + + g_return_val_if_fail (vi != NULL, FALSE); + g_return_val_if_fail (meta_model != NULL, FALSE); + g_return_val_if_fail (info_model != NULL, FALSE); + + GST_LOG_OBJECT (vi, "Postprocess Meta"); + + imeta = (GstInferenceMeta *) meta_model; + + root = imeta->prediction; + if (!root) { + GST_ERROR_OBJECT (vi, "Prediction is not part of the Inference Meta"); + return FALSE; + } + + c = gst_create_class_from_prediction (vi, prediction, predsize, labels_list, + num_labels); + gst_inference_prediction_append_classification (root, c); + gst_inference_print_predictions (vi, gst_inceptionv3_debug_category, imeta); + + *valid_prediction = TRUE; + return TRUE; +} + +static gboolean +gst_inceptionv3_postprocess (GstVideoInference * vi, const gpointer prediction, + gsize predsize, GstMeta * meta_model[2], GstVideoInfo * info_model, + gboolean * valid_prediction, gchar ** labels_list, gint num_labels) +{ + gboolean ret = TRUE; + + ret &= + gst_inceptionv3_postprocess_old (vi, prediction, predsize, meta_model[0], + info_model, valid_prediction); + ret &= + gst_inceptionv3_postprocess_new (vi, prediction, predsize, meta_model[1], + info_model, valid_prediction, labels_list, num_labels); + + return ret; +} + static gboolean gst_inceptionv3_start (GstVideoInference * vi) { diff --git a/ext/r2inference/gstinceptionv4.c b/ext/r2inference/gstinceptionv4.c index a6f0dba3..24081dc4 100644 --- a/ext/r2inference/gstinceptionv4.c +++ b/ext/r2inference/gstinceptionv4.c @@ -211,11 +211,11 @@ gst_inceptionv4_preprocess (GstVideoInference * vi, } static gboolean -gst_inceptionv4_postprocess (GstVideoInference * vi, const gpointer prediction, - gsize predsize, GstMeta * meta_model[2], GstVideoInfo * info_model, - gboolean * valid_prediction, gchar ** labels_list, gint num_labels) +gst_inceptionv4_postprocess_old (GstVideoInference * vi, + const gpointer prediction, gsize predsize, GstMeta * meta_model, + GstVideoInfo * info_model, gboolean * valid_prediction) { - GstClassificationMeta *class_meta = (GstClassificationMeta *) meta_model[0]; + GstClassificationMeta *class_meta = (GstClassificationMeta *) meta_model; GstDebugLevel gst_debug_level = GST_LEVEL_LOG; GST_LOG_OBJECT (vi, "Postprocess"); @@ -229,6 +229,56 @@ gst_inceptionv4_postprocess (GstVideoInference * vi, const gpointer prediction, return TRUE; } +static gboolean +gst_inceptionv4_postprocess_new (GstVideoInference * vi, + const gpointer prediction, gsize predsize, GstMeta * meta_model, + GstVideoInfo * info_model, gboolean * valid_prediction, + gchar ** labels_list, gint num_labels) +{ + GstInferenceMeta *imeta = NULL; + GstInferenceClassification *c = NULL; + GstInferencePrediction *root = NULL; + + g_return_val_if_fail (vi != NULL, FALSE); + g_return_val_if_fail (meta_model != NULL, FALSE); + g_return_val_if_fail (info_model != NULL, FALSE); + + GST_LOG_OBJECT (vi, "Postprocess Meta"); + + imeta = (GstInferenceMeta *) meta_model; + + root = imeta->prediction; + if (!root) { + GST_ERROR_OBJECT (vi, "Prediction is not part of the Inference Meta"); + return FALSE; + } + + c = gst_create_class_from_prediction (vi, prediction, predsize, labels_list, + num_labels); + gst_inference_prediction_append_classification (root, c); + gst_inference_print_predictions (vi, gst_inceptionv4_debug_category, imeta); + + *valid_prediction = TRUE; + return TRUE; +} + +static gboolean +gst_inceptionv4_postprocess (GstVideoInference * vi, const gpointer prediction, + gsize predsize, GstMeta * meta_model[2], GstVideoInfo * info_model, + gboolean * valid_prediction, gchar ** labels_list, gint num_labels) +{ + gboolean ret = TRUE; + + ret &= + gst_inceptionv4_postprocess_old (vi, prediction, predsize, meta_model[0], + info_model, valid_prediction); + ret &= + gst_inceptionv4_postprocess_new (vi, prediction, predsize, meta_model[1], + info_model, valid_prediction, labels_list, num_labels); + + return ret; +} + static gboolean gst_inceptionv4_start (GstVideoInference * vi) { From 22341f47860bfae823f90fd223ffb7681b234d85 Mon Sep 17 00:00:00 2001 From: tvlenin Date: Mon, 20 Jan 2020 15:55:44 -0600 Subject: [PATCH 169/188] Add new meta support to tinyyolov3 --- ext/r2inference/gsttinyyolov3.c | 100 ++++++++++++++++++++++++++++++-- 1 file changed, 94 insertions(+), 6 deletions(-) diff --git a/ext/r2inference/gsttinyyolov3.c b/ext/r2inference/gsttinyyolov3.c index d7406745..e7ff510d 100644 --- a/ext/r2inference/gsttinyyolov3.c +++ b/ext/r2inference/gsttinyyolov3.c @@ -75,6 +75,15 @@ static gboolean gst_tinyyolov3_postprocess (GstVideoInference * vi, const gpointer prediction, gsize predsize, GstMeta * meta_model[2], GstVideoInfo * info_model, gboolean * valid_prediction, gchar ** labels_list, gint num_labels); +static gboolean +gst_tinyyolov3_postprocess_old (GstVideoInference * vi, + const gpointer prediction, gsize predsize, GstMeta * meta_model, + GstVideoInfo * info_model, gboolean * valid_prediction); +static gboolean +gst_tinyyolov3_postprocess_new (GstVideoInference * vi, + const gpointer prediction, gsize predsize, GstMeta * meta_model, + GstVideoInfo * info_model, gboolean * valid_prediction, + gchar ** labels_list, gint num_labels); static gboolean gst_tinyyolov3_start (GstVideoInference * vi); static gboolean gst_tinyyolov3_stop (GstVideoInference * vi); @@ -249,16 +258,43 @@ gst_tinyyolov3_postprocess (GstVideoInference * vi, const gpointer prediction, gsize predsize, GstMeta * meta_model[2], GstVideoInfo * info_model, gboolean * valid_prediction, gchar ** labels_list, gint num_labels) { - GstTinyyolov3 *tinyyolov3 = GST_TINYYOLOV3 (vi); - GstDetectionMeta *detect_meta = (GstDetectionMeta *) meta_model[0]; + gboolean ret = TRUE; - GST_LOG_OBJECT (vi, "Postprocess"); + g_return_val_if_fail (vi, FALSE); + g_return_val_if_fail (prediction, FALSE); + g_return_val_if_fail (meta_model, FALSE); + g_return_val_if_fail (info_model, FALSE); + g_return_val_if_fail (valid_prediction, FALSE); + + ret &= + gst_tinyyolov3_postprocess_old (vi, prediction, predsize, meta_model[0], + info_model, valid_prediction); + ret &= + gst_tinyyolov3_postprocess_new (vi, prediction, predsize, meta_model[1], + info_model, valid_prediction, labels_list, num_labels); + + return TRUE; +} + +static gboolean +gst_tinyyolov3_postprocess_old (GstVideoInference * vi, + const gpointer prediction, gsize predsize, GstMeta * meta_model, + GstVideoInfo * info_model, gboolean * valid_prediction) +{ + GstTinyyolov3 *tinyyolov3; + gdouble *probabilities = NULL; + + GstDetectionMeta *detect_meta = (GstDetectionMeta *) meta_model; + + g_return_val_if_fail (detect_meta, FALSE); + GST_LOG_OBJECT (vi, "Postprocess"); detect_meta->num_boxes = 0; + tinyyolov3 = GST_TINYYOLOV3 (vi); - gst_create_boxes_float (vi, prediction, - valid_prediction, &detect_meta->boxes, &detect_meta->num_boxes, - tinyyolov3->obj_thresh, tinyyolov3->prob_thresh, tinyyolov3->iou_thresh); + gst_create_boxes (vi, prediction, valid_prediction, + &detect_meta->boxes, &detect_meta->num_boxes, tinyyolov3->obj_thresh, + tinyyolov3->prob_thresh, tinyyolov3->iou_thresh, &probabilities); gst_inference_print_boxes (vi, gst_tinyyolov3_debug_category, detect_meta); @@ -267,6 +303,58 @@ gst_tinyyolov3_postprocess (GstVideoInference * vi, const gpointer prediction, return TRUE; } +static gboolean +gst_tinyyolov3_postprocess_new (GstVideoInference * vi, + const gpointer prediction, gsize predsize, GstMeta * meta_model, + GstVideoInfo * info_model, gboolean * valid_prediction, + gchar ** labels_list, gint num_labels) +{ + GstTinyyolov3 *tinyyolov3 = NULL; + GstInferenceMeta *imeta = NULL; + BBox *boxes = NULL; + gint num_boxes, i; + gdouble *probabilities = NULL; + + g_return_val_if_fail (vi != NULL, FALSE); + g_return_val_if_fail (meta_model != NULL, FALSE); + g_return_val_if_fail (info_model != NULL, FALSE); + + imeta = (GstInferenceMeta *) meta_model; + tinyyolov3 = GST_TINYYOLOV3 (vi); + + GST_LOG_OBJECT (tinyyolov3, "Postprocess Meta"); + + /* Create boxes from prediction data */ + gst_create_boxes (vi, prediction, valid_prediction, + &boxes, &num_boxes, tinyyolov3->obj_thresh, + tinyyolov3->prob_thresh, tinyyolov3->iou_thresh, &probabilities); + + GST_LOG_OBJECT (tinyyolov3, "Number of predictions: %d", num_boxes); + + if (NULL == imeta->prediction) { + imeta->prediction = gst_inference_prediction_new (); + imeta->prediction->bbox.width = info_model->width; + imeta->prediction->bbox.height = info_model->height; + } + + for (i = 0; i < num_boxes; i++) { + GstInferencePrediction *pred = + gst_create_prediction_from_box (vi, &boxes[i], labels_list, num_labels, + probabilities); + gst_inference_prediction_append (imeta->prediction, pred); + } + + /* Free boxes after creation */ + g_free (boxes); + + /* Log predictions */ + gst_inference_print_predictions (vi, gst_tinyyolov3_debug_category, imeta); + + *valid_prediction = (num_boxes > 0) ? TRUE : FALSE; + + return TRUE; +} + static gboolean gst_tinyyolov3_start (GstVideoInference * vi) { From af9d3f2735bd1bdda5a0246642438d37a16441a2 Mon Sep 17 00:00:00 2001 From: tvlenin Date: Mon, 20 Jan 2020 15:59:11 -0600 Subject: [PATCH 170/188] Add new meta support to facenetv1 --- ext/r2inference/gstfacenetv1.c | 70 +++++++++++++++++++++++++++++++--- 1 file changed, 64 insertions(+), 6 deletions(-) diff --git a/ext/r2inference/gstfacenetv1.c b/ext/r2inference/gstfacenetv1.c index 8a1af8e4..7e004e13 100644 --- a/ext/r2inference/gstfacenetv1.c +++ b/ext/r2inference/gstfacenetv1.c @@ -65,6 +65,13 @@ static gboolean gst_facenetv1_postprocess (GstVideoInference * vi, const gpointer prediction, gsize predsize, GstMeta * meta_model[2], GstVideoInfo * info_model, gboolean * valid_prediction, gchar ** labels_list, gint num_labels); +static gboolean gst_facenetv1_postprocess_old (GstVideoInference * vi, + const gpointer prediction, gsize predsize, GstMeta * meta_model, + GstVideoInfo * info_model, gboolean * valid_prediction); +static gboolean gst_facenetv1_postprocess_new (GstVideoInference * vi, + const gpointer prediction, gsize predsize, GstMeta * meta_model, + GstVideoInfo * info_model, gboolean * valid_prediction, + gchar ** labels_list, gint num_labels); static gboolean gst_facenetv1_start (GstVideoInference * vi); static gboolean gst_facenetv1_stop (GstVideoInference * vi); @@ -206,23 +213,74 @@ gst_facenetv1_preprocess (GstVideoInference * vi, } static gboolean -gst_facenetv1_postprocess (GstVideoInference * vi, const gpointer prediction, - gsize predsize, GstMeta * meta_model[2], GstVideoInfo * info_model, - gboolean * valid_prediction, gchar ** labels_list, gint num_labels) +gst_facenetv1_postprocess_old (GstVideoInference * vi, + const gpointer prediction, gsize predsize, GstMeta * meta_model, + GstVideoInfo * info_model, gboolean * valid_prediction) { - GstClassificationMeta *class_meta = (GstClassificationMeta *) meta_model[0]; + GstClassificationMeta *class_meta = (GstClassificationMeta *) meta_model; GstDebugLevel gst_debug_level = GST_LEVEL_LOG; GST_LOG_OBJECT (vi, "Postprocess"); gst_fill_classification_meta (class_meta, prediction, predsize); - gst_inference_print_embedding (vi, gst_facenetv1_debug_category, class_meta, - prediction, gst_debug_level); + gst_inference_print_highest_probability (vi, gst_facenetv1_debug_category, + class_meta, prediction, gst_debug_level); + *valid_prediction = TRUE; return TRUE; } +static gboolean +gst_facenetv1_postprocess_new (GstVideoInference * vi, + const gpointer prediction, gsize predsize, GstMeta * meta_model, + GstVideoInfo * info_model, gboolean * valid_prediction, + gchar ** labels_list, gint num_labels) +{ + GstInferenceMeta *imeta = NULL; + GstInferenceClassification *c = NULL; + GstInferencePrediction *root = NULL; + + g_return_val_if_fail (vi != NULL, FALSE); + g_return_val_if_fail (meta_model != NULL, FALSE); + g_return_val_if_fail (info_model != NULL, FALSE); + + GST_LOG_OBJECT (vi, "Postprocess Meta"); + + imeta = (GstInferenceMeta *) meta_model; + + root = imeta->prediction; + if (!root) { + GST_ERROR_OBJECT (vi, "Prediction is not part of the Inference Meta"); + return FALSE; + } + + c = gst_create_class_from_prediction (vi, prediction, predsize, labels_list, + num_labels); + gst_inference_prediction_append_classification (root, c); + gst_inference_print_predictions (vi, gst_facenetv1_debug_category, imeta); + + *valid_prediction = TRUE; + return TRUE; +} + +static gboolean +gst_facenetv1_postprocess (GstVideoInference * vi, const gpointer prediction, + gsize predsize, GstMeta * meta_model[2], GstVideoInfo * info_model, + gboolean * valid_prediction, gchar ** labels_list, gint num_labels) +{ + gboolean ret = TRUE; + + ret &= + gst_facenetv1_postprocess_old (vi, prediction, predsize, meta_model[0], + info_model, valid_prediction); + ret &= + gst_facenetv1_postprocess_new (vi, prediction, predsize, meta_model[1], + info_model, valid_prediction, labels_list, num_labels); + + return ret; +} + static gboolean gst_facenetv1_start (GstVideoInference * vi) { From 7f0fa5fd228a19a601f2c95935987466420b6f5d Mon Sep 17 00:00:00 2001 From: tvlenin Date: Mon, 20 Jan 2020 16:00:47 -0600 Subject: [PATCH 171/188] Add new meta support to mobilenetv2 --- ext/r2inference/gstmobilenetv2.c | 65 ++++++++++++++++++++++++++++++-- 1 file changed, 61 insertions(+), 4 deletions(-) diff --git a/ext/r2inference/gstmobilenetv2.c b/ext/r2inference/gstmobilenetv2.c index 761d87e9..7b9b44e0 100644 --- a/ext/r2inference/gstmobilenetv2.c +++ b/ext/r2inference/gstmobilenetv2.c @@ -58,6 +58,13 @@ static gboolean gst_mobilenetv2_postprocess (GstVideoInference * vi, const gpointer prediction, gsize predsize, GstMeta * meta_model[2], GstVideoInfo * info_model, gboolean * valid_prediction, gchar ** labels_list, gint num_labels); +static gboolean gst_mobilenetv2_postprocess_old (GstVideoInference * vi, + const gpointer prediction, gsize predsize, GstMeta * meta_model, + GstVideoInfo * info_model, gboolean * valid_prediction); +static gboolean gst_mobilenetv2_postprocess_new (GstVideoInference * vi, + const gpointer prediction, gsize predsize, GstMeta * meta_model, + GstVideoInfo * info_model, gboolean * valid_prediction, + gchar ** labels_list, gint num_labels); static gboolean gst_mobilenetv2_start (GstVideoInference * vi); static gboolean gst_mobilenetv2_stop (GstVideoInference * vi); @@ -143,11 +150,11 @@ gst_mobilenetv2_preprocess (GstVideoInference * vi, } static gboolean -gst_mobilenetv2_postprocess (GstVideoInference * vi, const gpointer prediction, - gsize predsize, GstMeta * meta_model[2], GstVideoInfo * info_model, - gboolean * valid_prediction, gchar ** labels_list, gint num_labels) +gst_mobilenetv2_postprocess_old (GstVideoInference * vi, + const gpointer prediction, gsize predsize, GstMeta * meta_model, + GstVideoInfo * info_model, gboolean * valid_prediction) { - GstClassificationMeta *class_meta = (GstClassificationMeta *) meta_model[0]; + GstClassificationMeta *class_meta = (GstClassificationMeta *) meta_model; GstDebugLevel gst_debug_level = GST_LEVEL_LOG; GST_LOG_OBJECT (vi, "Postprocess"); @@ -161,6 +168,56 @@ gst_mobilenetv2_postprocess (GstVideoInference * vi, const gpointer prediction, return TRUE; } +static gboolean +gst_mobilenetv2_postprocess_new (GstVideoInference * vi, + const gpointer prediction, gsize predsize, GstMeta * meta_model, + GstVideoInfo * info_model, gboolean * valid_prediction, + gchar ** labels_list, gint num_labels) +{ + GstInferenceMeta *imeta = NULL; + GstInferenceClassification *c = NULL; + GstInferencePrediction *root = NULL; + + g_return_val_if_fail (vi != NULL, FALSE); + g_return_val_if_fail (meta_model != NULL, FALSE); + g_return_val_if_fail (info_model != NULL, FALSE); + + GST_LOG_OBJECT (vi, "Postprocess Meta"); + + imeta = (GstInferenceMeta *) meta_model; + + root = imeta->prediction; + if (!root) { + GST_ERROR_OBJECT (vi, "Prediction is not part of the Inference Meta"); + return FALSE; + } + + c = gst_create_class_from_prediction (vi, prediction, predsize, labels_list, + num_labels); + gst_inference_prediction_append_classification (root, c); + gst_inference_print_predictions (vi, gst_mobilenetv2_debug_category, imeta); + + *valid_prediction = TRUE; + return TRUE; +} + +static gboolean +gst_mobilenetv2_postprocess (GstVideoInference * vi, const gpointer prediction, + gsize predsize, GstMeta * meta_model[2], GstVideoInfo * info_model, + gboolean * valid_prediction, gchar ** labels_list, gint num_labels) +{ + gboolean ret = TRUE; + + ret &= + gst_mobilenetv2_postprocess_old (vi, prediction, predsize, meta_model[0], + info_model, valid_prediction); + ret &= + gst_mobilenetv2_postprocess_new (vi, prediction, predsize, meta_model[1], + info_model, valid_prediction, labels_list, num_labels); + + return ret; +} + static gboolean gst_mobilenetv2_start (GstVideoInference * vi) { From ddc834eaca8657aa121b3f531149abaca94a6016 Mon Sep 17 00:00:00 2001 From: tvlenin Date: Mon, 20 Jan 2020 16:01:59 -0600 Subject: [PATCH 172/188] Add new meta support to restnet50v1 --- ext/r2inference/gstresnet50v1.c | 65 +++++++++++++++++++++++++++++++-- 1 file changed, 61 insertions(+), 4 deletions(-) diff --git a/ext/r2inference/gstresnet50v1.c b/ext/r2inference/gstresnet50v1.c index be39c830..b52a905a 100644 --- a/ext/r2inference/gstresnet50v1.c +++ b/ext/r2inference/gstresnet50v1.c @@ -62,6 +62,13 @@ static gboolean gst_resnet50v1_postprocess (GstVideoInference * vi, const gpointer prediction, gsize predsize, GstMeta * meta_model[2], GstVideoInfo * info_model, gboolean * valid_prediction, gchar ** labels_list, gint num_labels); +static gboolean gst_resnet50v1_postprocess_old (GstVideoInference * vi, + const gpointer prediction, gsize predsize, GstMeta * meta_model, + GstVideoInfo * info_model, gboolean * valid_prediction); +static gboolean gst_resnet50v1_postprocess_new (GstVideoInference * vi, + const gpointer prediction, gsize predsize, GstMeta * meta_model, + GstVideoInfo * info_model, gboolean * valid_prediction, + gchar ** labels_list, gint num_labels); static gboolean gst_resnet50v1_start (GstVideoInference * vi); static gboolean gst_resnet50v1_stop (GstVideoInference * vi); @@ -149,11 +156,11 @@ gst_resnet50v1_preprocess (GstVideoInference * vi, } static gboolean -gst_resnet50v1_postprocess (GstVideoInference * vi, const gpointer prediction, - gsize predsize, GstMeta * meta_model[2], GstVideoInfo * info_model, - gboolean * valid_prediction, gchar ** labels_list, gint num_labels) +gst_resnet50v1_postprocess_old (GstVideoInference * vi, + const gpointer prediction, gsize predsize, GstMeta * meta_model, + GstVideoInfo * info_model, gboolean * valid_prediction) { - GstClassificationMeta *class_meta = (GstClassificationMeta *) meta_model[0]; + GstClassificationMeta *class_meta = (GstClassificationMeta *) meta_model; GstDebugLevel gst_debug_level = GST_LEVEL_LOG; GST_LOG_OBJECT (vi, "Postprocess"); @@ -167,6 +174,56 @@ gst_resnet50v1_postprocess (GstVideoInference * vi, const gpointer prediction, return TRUE; } +static gboolean +gst_resnet50v1_postprocess_new (GstVideoInference * vi, + const gpointer prediction, gsize predsize, GstMeta * meta_model, + GstVideoInfo * info_model, gboolean * valid_prediction, + gchar ** labels_list, gint num_labels) +{ + GstInferenceMeta *imeta = NULL; + GstInferenceClassification *c = NULL; + GstInferencePrediction *root = NULL; + + g_return_val_if_fail (vi != NULL, FALSE); + g_return_val_if_fail (meta_model != NULL, FALSE); + g_return_val_if_fail (info_model != NULL, FALSE); + + GST_LOG_OBJECT (vi, "Postprocess Meta"); + + imeta = (GstInferenceMeta *) meta_model; + + root = imeta->prediction; + if (!root) { + GST_ERROR_OBJECT (vi, "Prediction is not part of the Inference Meta"); + return FALSE; + } + + c = gst_create_class_from_prediction (vi, prediction, predsize, labels_list, + num_labels); + gst_inference_prediction_append_classification (root, c); + gst_inference_print_predictions (vi, gst_resnet50v1_debug_category, imeta); + + *valid_prediction = TRUE; + return TRUE; +} + +static gboolean +gst_resnet50v1_postprocess (GstVideoInference * vi, const gpointer prediction, + gsize predsize, GstMeta * meta_model[2], GstVideoInfo * info_model, + gboolean * valid_prediction, gchar ** labels_list, gint num_labels) +{ + gboolean ret = TRUE; + + ret &= + gst_resnet50v1_postprocess_old (vi, prediction, predsize, meta_model[0], + info_model, valid_prediction); + ret &= + gst_resnet50v1_postprocess_new (vi, prediction, predsize, meta_model[1], + info_model, valid_prediction, labels_list, num_labels); + + return ret; +} + static gboolean gst_resnet50v1_start (GstVideoInference * vi) { From de6a3ba73ed871449b5dbb38670b820eb3650d3f Mon Sep 17 00:00:00 2001 From: Michael Gruner Date: Fri, 31 Jan 2020 18:02:06 -0600 Subject: [PATCH 173/188] Rename plugin entry point to build in c++ --- ext/opencv/Makefile.am | 2 +- ext/opencv/{gstinferenceoverlayplugin.c => gstplugin.cc} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename ext/opencv/{gstinferenceoverlayplugin.c => gstplugin.cc} (100%) diff --git a/ext/opencv/Makefile.am b/ext/opencv/Makefile.am index b55ade76..7762f6a6 100644 --- a/ext/opencv/Makefile.am +++ b/ext/opencv/Makefile.am @@ -3,7 +3,7 @@ plugin_LTLIBRARIES = libgstinferenceoverlayplugin.la libgstinferenceoverlayplugin_la_SOURCES = \ gstclassificationoverlay.cc \ gstdetectionoverlay.cc \ - gstinferenceoverlayplugin.c \ + gstplugin.cc \ gstinferenceoverlay.cc \ gstembeddingoverlay.cc diff --git a/ext/opencv/gstinferenceoverlayplugin.c b/ext/opencv/gstplugin.cc similarity index 100% rename from ext/opencv/gstinferenceoverlayplugin.c rename to ext/opencv/gstplugin.cc From 9147bbbcaf364d6f48bb6f67eacc3188e389e168 Mon Sep 17 00:00:00 2001 From: Michael Gruner Date: Fri, 31 Jan 2020 18:02:34 -0600 Subject: [PATCH 174/188] Use new overlay virtual method definition --- ext/opencv/gstembeddingoverlay.cc | 99 ++++++++++++++----------------- 1 file changed, 43 insertions(+), 56 deletions(-) diff --git a/ext/opencv/gstembeddingoverlay.cc b/ext/opencv/gstembeddingoverlay.cc index 47314111..5d7638c9 100644 --- a/ext/opencv/gstembeddingoverlay.cc +++ b/ext/opencv/gstembeddingoverlay.cc @@ -24,13 +24,8 @@ #endif #include "gstembeddingoverlay.h" + #include "gst/r2inference/gstinferencemeta.h" -#ifdef OCV_VERSION_LT_3_2 -#include "opencv2/highgui/highgui.hpp" -#else -#include "opencv2/imgproc.hpp" -#include "opencv2/highgui.hpp" -#endif #define DEFAULT_EMBEDDINGS NULL #define DEFAULT_NUM_EMBEDDINGS 0 @@ -38,9 +33,11 @@ #define MAX_LIKENESS_THRESH G_MAXDOUBLE #define DEFAULT_LIKENESS_THRESH 1.0 +/* *INDENT-OFF* */ static const cv::Scalar forest_green = cv::Scalar (11, 102, 35); static const cv::Scalar chilli_red = cv::Scalar (194, 24, 7); static const cv::Scalar white = cv::Scalar (255, 255, 255, 255); +/* *INDENT-ON* */ GST_DEBUG_CATEGORY_STATIC (gst_embedding_overlay_debug_category); #define GST_CAT_DEFAULT gst_embedding_overlay_debug_category @@ -51,13 +48,12 @@ static void gst_embedding_overlay_set_property (GObject * object, static void gst_embedding_overlay_get_property (GObject * object, guint property_id, GValue * value, GParamSpec * pspec); static void gst_embedding_overlay_finalize (GObject * object); -static GstFlowReturn -gst_embedding_overlay_process_meta (GstInferenceBaseOverlay * inference_overlay, - GstVideoFrame * frame, GstMeta * meta, gdouble font_scale, gint thickness, - gchar ** labels_list, gint num_labels, LineStyleBoundingBox style); -static gboolean -gst_embedding_overlay_set_embeddings (GstEmbeddingOverlay * embedding_overlay, - const GValue * value); +static GstFlowReturn gst_embedding_overlay_process_meta (GstInferenceBaseOverlay + * inference_overlay, cv::Mat & cv_mat, GstVideoFrame * frame, + GstMeta * meta, gdouble font_scale, gint thickness, gchar ** labels_list, + gint num_labels, LineStyleBoundingBox style); +static gboolean gst_embedding_overlay_set_embeddings (GstEmbeddingOverlay * + embedding_overlay, const GValue * value); enum { @@ -92,7 +88,8 @@ static void gst_embedding_overlay_class_init (GstEmbeddingOverlayClass * klass) { GObjectClass *gobject_class = G_OBJECT_CLASS (klass); - GstInferenceBaseOverlayClass *io_class = GST_INFERENCE_BASE_OVERLAY_CLASS (klass); + GstInferenceBaseOverlayClass *io_class = + GST_INFERENCE_BASE_OVERLAY_CLASS (klass); gobject_class->set_property = gst_embedding_overlay_set_property; gobject_class->get_property = gst_embedding_overlay_get_property; @@ -144,20 +141,20 @@ gst_embedding_overlay_set_property (GObject * object, guint property_id, switch (property_id) { case PROP_EMBEDDINGS: - if (gst_embedding_overlay_set_embeddings(embedding_overlay, value)){ + if (gst_embedding_overlay_set_embeddings (embedding_overlay, value)) { GST_DEBUG_OBJECT (embedding_overlay, "Changed inference labels %s", - embedding_overlay->embeddings); + embedding_overlay->embeddings); } else { GST_ERROR_OBJECT (embedding_overlay, "Failed setting embeddings"); } break; case PROP_LIKENESS_THRESH: - GST_OBJECT_LOCK(embedding_overlay); + GST_OBJECT_LOCK (embedding_overlay); embedding_overlay->likeness_thresh = g_value_get_double (value); GST_DEBUG_OBJECT (embedding_overlay, "Changed likeness threshold to %lf", embedding_overlay->likeness_thresh); - GST_OBJECT_UNLOCK(embedding_overlay); + GST_OBJECT_UNLOCK (embedding_overlay); break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); @@ -175,14 +172,14 @@ gst_embedding_overlay_get_property (GObject * object, guint property_id, switch (property_id) { case PROP_EMBEDDINGS: - GST_OBJECT_LOCK(embedding_overlay); + GST_OBJECT_LOCK (embedding_overlay); g_value_set_string (value, embedding_overlay->embeddings); - GST_OBJECT_UNLOCK(embedding_overlay); + GST_OBJECT_UNLOCK (embedding_overlay); break; case PROP_LIKENESS_THRESH: - GST_OBJECT_LOCK(embedding_overlay); + GST_OBJECT_LOCK (embedding_overlay); g_value_set_double (value, embedding_overlay->likeness_thresh); - GST_OBJECT_UNLOCK(embedding_overlay); + GST_OBJECT_UNLOCK (embedding_overlay); break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); @@ -210,14 +207,15 @@ gst_embedding_overlay_finalize (GObject * object) static GstFlowReturn gst_embedding_overlay_process_meta (GstInferenceBaseOverlay * inference_overlay, - GstVideoFrame * frame, GstMeta * meta, gdouble font_scale, gint thickness, - gchar ** labels_list, gint num_labels, LineStyleBoundingBox style) + cv::Mat & cv_mat, GstVideoFrame * frame, GstMeta * meta, gdouble font_scale, + gint thickness, gchar ** labels_list, gint num_labels, + LineStyleBoundingBox style) { - GstEmbeddingOverlay *embedding_overlay = GST_EMBEDDING_OVERLAY (inference_overlay); + GstEmbeddingOverlay *embedding_overlay = + GST_EMBEDDING_OVERLAY (inference_overlay); GstClassificationMeta *class_meta; - gint i, j, width, height, channels; + gint i, j, width, height; gdouble current, diff; - cv::Mat cv_mat; cv::String str; cv::Size size; cv::Scalar tmp_color; @@ -228,36 +226,23 @@ gst_embedding_overlay_process_meta (GstInferenceBaseOverlay * inference_overlay, gint embedding_size; gdouble likeness_thresh; - GST_OBJECT_LOCK(embedding_overlay); + GST_OBJECT_LOCK (embedding_overlay); embeddings_list = g_strdupv (embedding_overlay->embeddings_list); num_embeddings = embedding_overlay->num_embeddings; embedding_size = embedding_overlay->embedding_size; likeness_thresh = embedding_overlay->likeness_thresh; - GST_OBJECT_UNLOCK(embedding_overlay); + GST_OBJECT_UNLOCK (embedding_overlay); - if (num_embeddings == 0){ + if (num_embeddings == 0) { GST_WARNING_OBJECT (embedding_overlay, "Please set at least one valid embedding using the 'embeddings'" "property"); goto end; } - switch (GST_VIDEO_FRAME_FORMAT (frame)) { - case GST_VIDEO_FORMAT_RGB: - case GST_VIDEO_FORMAT_BGR: - channels = 3; - break; - default: - channels = 4; - break; - } - - width = GST_VIDEO_FRAME_COMP_STRIDE (frame, 0) / channels; - height = GST_VIDEO_FRAME_HEIGHT (frame); - class_meta = (GstClassificationMeta *) meta; - if (class_meta->num_labels != embedding_size){ + if (class_meta->num_labels != embedding_size) { GST_WARNING_OBJECT (embedding_overlay, "Provided embeddings and inference output have different sizes"); goto end; @@ -271,8 +256,7 @@ gst_embedding_overlay_process_meta (GstInferenceBaseOverlay * inference_overlay, diff = 0.0; for (j = 0; j < embedding_size; ++j) { current = class_meta->label_probs[j]; - current -= - atof (embeddings_list[i * embedding_size + j]); + current -= atof (embeddings_list[i * embedding_size + j]); current = current * current; diff = diff + current; } @@ -315,22 +299,25 @@ gst_embedding_overlay_process_meta (GstInferenceBaseOverlay * inference_overlay, break; } - cv_mat = cv::Mat (height, width, CV_MAKETYPE (CV_8U, channels), - (char *) frame->data[0]); + width = cv_mat.cols; + height = cv_mat.rows; + /* Put string on screen * 10*font_scale+16 aproximates text's rendered size on screen as a * lineal function to avoid using cv::getTextSize. * 5*thickness adds the border offset */ - cv::putText (cv_mat, str, cv::Point (5*thickness, 5*thickness+10*font_scale+16), - cv::FONT_HERSHEY_PLAIN, font_scale, white, thickness + (thickness*0.5)); - cv::putText (cv_mat, str, cv::Point (5*thickness, 5*thickness+10*font_scale+16), - cv::FONT_HERSHEY_PLAIN, font_scale, color, thickness); + cv::putText (cv_mat, str, cv::Point (5 * thickness, + 5 * thickness + 10 * font_scale + 16), cv::FONT_HERSHEY_PLAIN, + font_scale, white, thickness + (thickness * 0.5)); + cv::putText (cv_mat, str, cv::Point (5 * thickness, + 5 * thickness + 10 * font_scale + 16), cv::FONT_HERSHEY_PLAIN, + font_scale, color, thickness); cv::rectangle (cv_mat, cv::Point (0, 0), cv::Point (width, height), color, - 10*thickness); + 10 * thickness); end: - g_strfreev(embeddings_list); + g_strfreev (embeddings_list); return GST_FLOW_OK; } @@ -341,7 +328,7 @@ gst_embedding_overlay_set_embeddings (GstEmbeddingOverlay * embedding_overlay, g_return_val_if_fail (embedding_overlay, FALSE); g_return_val_if_fail (value, FALSE); - GST_OBJECT_LOCK(embedding_overlay); + GST_OBJECT_LOCK (embedding_overlay); if (embedding_overlay->embeddings != NULL) { g_free (embedding_overlay->embeddings); } @@ -359,7 +346,7 @@ gst_embedding_overlay_set_embeddings (GstEmbeddingOverlay * embedding_overlay, embedding_overlay->embedding_size = g_strv_length (embedding_overlay->embeddings_list) / embedding_overlay->num_embeddings; - GST_OBJECT_UNLOCK(embedding_overlay); + GST_OBJECT_UNLOCK (embedding_overlay); return TRUE; } From c8ff355557834a6ce7568aea99dc62597e0601c7 Mon Sep 17 00:00:00 2001 From: tvlenin Date: Fri, 31 Jan 2020 14:31:05 -0600 Subject: [PATCH 175/188] Add tensorflow lite backend --- gst-libs/gst/r2inference/Makefile.am | 2 + .../gst/r2inference/gstinferencebackends.cc | 4 ++ gst-libs/gst/r2inference/gsttflite.cc | 54 +++++++++++++++++++ gst-libs/gst/r2inference/gsttflite.h | 35 ++++++++++++ 4 files changed, 95 insertions(+) create mode 100644 gst-libs/gst/r2inference/gsttflite.cc create mode 100644 gst-libs/gst/r2inference/gsttflite.h diff --git a/gst-libs/gst/r2inference/Makefile.am b/gst-libs/gst/r2inference/Makefile.am index d59b1972..f99d2062 100644 --- a/gst-libs/gst/r2inference/Makefile.am +++ b/gst-libs/gst/r2inference/Makefile.am @@ -9,6 +9,7 @@ libgstinference_@GST_API_VERSION@_la_SOURCES= \ gstbackend.cc \ gstncsdk.cc \ gsttensorflow.cc \ + gsttflite.cc \ gstinferencepreprocess.c \ gstinferencepostprocess.c \ gstinferencedebug.c \ @@ -45,6 +46,7 @@ gstinferenceinclude_HEADERS= \ gstbackendsubclass.h \ gstncsdk.h \ gsttensorflow.h \ + gsttflite.h \ gstinferencepreprocess.h \ gstinferencepostprocess.h \ gstinferencedebug.h \ diff --git a/gst-libs/gst/r2inference/gstinferencebackends.cc b/gst-libs/gst/r2inference/gstinferencebackends.cc index d171f172..a1f1de8c 100644 --- a/gst-libs/gst/r2inference/gstinferencebackends.cc +++ b/gst-libs/gst/r2inference/gstinferencebackends.cc @@ -23,6 +23,7 @@ #include "gstchildinspector.h" #include "gstncsdk.h" #include "gsttensorflow.h" +#include "gsttflite.h" #include "gstbackend.h" #include #include @@ -34,6 +35,7 @@ static std::unordered_map backend_types ({ {r2i::FrameworkCode::NCSDK, GST_TYPE_NCSDK}, {r2i::FrameworkCode::TENSORFLOW, GST_TYPE_TENSORFLOW}, + {r2i::FrameworkCode::TFLITE, GST_TYPE_TFLITE}, {r2i::FrameworkCode::MAX_FRAMEWORK, G_TYPE_INVALID} }); @@ -51,6 +53,8 @@ gst_inference_backends_get_type (void) {r2i::FrameworkCode::NCSDK, "Intel Movidius Neural Compute SDK", "ncsdk"}, {r2i::FrameworkCode::TENSORFLOW, "TensorFlow Machine Learning Framework", "tensorflow"}, + {r2i::FrameworkCode::TFLITE, "Tensorflow Lite Machine Learning Framework", + "tflite"}, {0, NULL, NULL} }; if (!backend_type) { diff --git a/gst-libs/gst/r2inference/gsttflite.cc b/gst-libs/gst/r2inference/gsttflite.cc new file mode 100644 index 00000000..d2286ac0 --- /dev/null +++ b/gst-libs/gst/r2inference/gsttflite.cc @@ -0,0 +1,54 @@ +/* + * GStreamer + * Copyright (C) 2018-2020 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + */ + +#include "gsttflite.h" + +#include + +GST_DEBUG_CATEGORY_STATIC (gst_tflite_debug_category); +#define GST_CAT_DEFAULT gst_tflite_debug_category + +struct _GstTflite +{ + GstBackend parent; +}; + +G_DEFINE_TYPE_WITH_CODE (GstTflite, gst_tflite, GST_TYPE_BACKEND, + GST_DEBUG_CATEGORY_INIT (gst_tflite_debug_category, "tflite", 0, + "debug category for tflite parameters")); + +static void +gst_tflite_class_init (GstTfliteClass * klass) +{ + GstBackendClass *bclass = GST_BACKEND_CLASS (klass); + GObjectClass *oclass = G_OBJECT_CLASS (klass); + + oclass->set_property = gst_backend_set_property; + oclass->get_property = gst_backend_get_property; + gst_backend_install_properties (bclass, r2i::FrameworkCode::TFLITE); +} + +static void +gst_tflite_init (GstTflite * self) +{ + gst_backend_set_framework_code (GST_BACKEND (self), + r2i::FrameworkCode::TFLITE); +} diff --git a/gst-libs/gst/r2inference/gsttflite.h b/gst-libs/gst/r2inference/gsttflite.h new file mode 100644 index 00000000..0260114f --- /dev/null +++ b/gst-libs/gst/r2inference/gsttflite.h @@ -0,0 +1,35 @@ +/* + * GStreamer + * Copyright (C) 2018-2020 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + */ + +#ifndef __GST_TFLITE_H__ +#define __GST_TFLITE_H__ + +#include +#include + +G_BEGIN_DECLS + +#define GST_TYPE_TFLITE gst_tflite_get_type () +G_DECLARE_FINAL_TYPE(GstTflite, gst_tflite, GST, TFLITE, GstBackend); + +G_END_DECLS + +#endif //__GST_TFLITE_H__ From 4209767ca980671c7167f8d2506b6277db0c19a5 Mon Sep 17 00:00:00 2001 From: Carlos Rodriguez Date: Tue, 4 Feb 2020 14:29:46 -0600 Subject: [PATCH 176/188] Add stream-id field to inferencemeta --- gst-libs/gst/r2inference/gstinferencemeta.c | 15 ++++++++++++--- gst-libs/gst/r2inference/gstinferencemeta.h | 2 ++ 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/gst-libs/gst/r2inference/gstinferencemeta.c b/gst-libs/gst/r2inference/gstinferencemeta.c index 326b28a9..b4b44b1e 100644 --- a/gst-libs/gst/r2inference/gstinferencemeta.c +++ b/gst-libs/gst/r2inference/gstinferencemeta.c @@ -226,14 +226,17 @@ gst_inference_meta_transform_existing_meta (GstBuffer * dest, GstMeta * meta, needs_scale = gst_inference_prediction_merge (smeta->prediction, pred); + /* Transfer Stream ID */ + g_free (dmeta->stream_id); + dmeta->stream_id = g_strdup (smeta->stream_id); + if (GST_META_TRANSFORM_IS_COPY (type)) { - GST_LOG ("Copy detection metadata"); + GST_LOG ("Copy inference metadata"); /* The merge already handled the copy */ goto out; } - if (GST_VIDEO_META_TRANSFORM_IS_SCALE (type)) { if (needs_scale) { GstVideoMetaTransform *trans = (GstVideoMetaTransform *) data; @@ -273,8 +276,12 @@ gst_inference_meta_transform_new_meta (GstBuffer * dest, GstMeta * meta, gst_inference_prediction_unref (dmeta->prediction); + /* Transfer Stream ID */ + g_free (dmeta->stream_id); + dmeta->stream_id = g_strdup (smeta->stream_id); + if (GST_META_TRANSFORM_IS_COPY (type)) { - GST_LOG ("Copy detection metadata"); + GST_LOG ("Copy inference metadata"); dmeta->prediction = gst_inference_prediction_copy (smeta->prediction); return TRUE; @@ -493,6 +500,7 @@ gst_inference_meta_init (GstMeta * meta, gpointer params, GstBuffer * buffer) root = gst_inference_prediction_new (); imeta->prediction = root; + imeta->stream_id = NULL; return TRUE; } @@ -507,4 +515,5 @@ gst_inference_meta_free (GstMeta * meta, GstBuffer * buffer) imeta = (GstInferenceMeta *) meta; gst_inference_prediction_unref (imeta->prediction); + g_free (imeta->stream_id); } diff --git a/gst-libs/gst/r2inference/gstinferencemeta.h b/gst-libs/gst/r2inference/gstinferencemeta.h index 7d23f707..59902943 100644 --- a/gst-libs/gst/r2inference/gstinferencemeta.h +++ b/gst-libs/gst/r2inference/gstinferencemeta.h @@ -59,6 +59,8 @@ struct _GstInferenceMeta GstMeta meta; GstInferencePrediction *prediction; + + gchar *stream_id; }; /** From 99cd4b24f2d2af86c4adcd0a81816f12b31ec75f Mon Sep 17 00:00:00 2001 From: Carlos Rodriguez Date: Tue, 4 Feb 2020 14:30:45 -0600 Subject: [PATCH 177/188] Update stream-id for each forwarded buffer Since we are storing buffers, that is causing a mismatch with the stream ids in each pad. In order to handle this we save the actual stream id when the buffer arrives and when the buffer is going to be forwarded a new stream-start event is sent. --- gst-libs/gst/r2inference/gstvideoinference.c | 28 ++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/gst-libs/gst/r2inference/gstvideoinference.c b/gst-libs/gst/r2inference/gstvideoinference.c index fb0e871e..5b21d2da 100644 --- a/gst-libs/gst/r2inference/gstvideoinference.c +++ b/gst-libs/gst/r2inference/gstvideoinference.c @@ -645,6 +645,7 @@ gst_video_inference_forward_buffer (GstVideoInference * self, { GstFlowReturn ret = GST_FLOW_OK; GstDebugLevel level = GST_LEVEL_LOG; + GstInferenceMeta *imeta = NULL; g_return_val_if_fail (self, GST_FLOW_ERROR); @@ -660,6 +661,27 @@ gst_video_inference_forward_buffer (GstVideoInference * self, return ret; } + /* Get Stream ID */ + imeta = + (GstInferenceMeta *) gst_buffer_get_meta (buffer, + gst_inference_meta_api_get_type ()); + if (imeta) { + if (imeta->stream_id) { + /* Check current stream ID in pad */ + gchar *stream_id = gst_pad_get_stream_id (pad); + if (0 != g_strcmp0 (stream_id, imeta->stream_id)) { + /* Different Stream IDs, forward new stream-start event */ + GstEvent *stream_start = gst_event_new_stream_start (imeta->stream_id); + GST_LOG_OBJECT (self, + "stream-start mismatch, resetting: Current Stream ID: %s, Buffer Stream ID: %s", + stream_id, imeta->stream_id); + if (!gst_pad_push_event (pad, stream_start)) + GST_WARNING_OBJECT (self, "Failed to push stream start event"); + } + g_free (stream_id); + } + } + GST_LOG_OBJECT (self, "Forwarding buffer %" GST_PTR_FORMAT " to %" GST_PTR_FORMAT, buffer, pad); ret = gst_pad_push (pad, buffer); @@ -936,11 +958,17 @@ gst_video_inference_process_model (GstVideoInference * self, GstBuffer * buffer, "There is no sinkpad for bypass, forwarding model buffer..."); goto forward_buffer; } else { + GstInferenceMeta *imeta = (GstInferenceMeta *) meta_model[1]; /* Queue buffer */ GST_LOG_OBJECT (self, "Queue model buffer"); g_mutex_lock (&priv->mtx_model_queue); g_queue_push_head (priv->model_queue, (gpointer) buffer_model); g_mutex_unlock (&priv->mtx_model_queue); + /* Keep current Stream ID */ + if (imeta) { + g_free (imeta->stream_id); + imeta->stream_id = gst_pad_get_stream_id (pad->data.pad); + } goto out; } From 16ec19c38462c94b6c4c4311fe2a53f4e01da296 Mon Sep 17 00:00:00 2001 From: tvlenin Date: Thu, 6 Feb 2020 10:05:52 -0600 Subject: [PATCH 178/188] Use the correct size for predictions --- gst-libs/gst/r2inference/gstbackend.cc | 4 ++-- gst-libs/gst/r2inference/gstinferencepostprocess.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/gst-libs/gst/r2inference/gstbackend.cc b/gst-libs/gst/r2inference/gstbackend.cc index 7c84517c..b0e723b7 100644 --- a/gst-libs/gst/r2inference/gstbackend.cc +++ b/gst-libs/gst/r2inference/gstbackend.cc @@ -432,8 +432,8 @@ gst_backend_process_frame (GstBackend *self, GstVideoFrame *input_frame, *prediction_size = prediction->GetResultSize (); /*could we avoid memory copy ?*/ - *prediction_data = g_malloc(*prediction_size); - memcpy(*prediction_data, prediction->GetResultData(), *prediction_size); + *prediction_data = g_malloc(*prediction_size * sizeof(gfloat)); + memcpy(*prediction_data, prediction->GetResultData(), *prediction_size * sizeof(gfloat)); GST_LOG_OBJECT (self, "Size of prediction %p is %lu", *prediction_data, *prediction_size); diff --git a/gst-libs/gst/r2inference/gstinferencepostprocess.c b/gst-libs/gst/r2inference/gstinferencepostprocess.c index 328456cf..50f676e1 100644 --- a/gst-libs/gst/r2inference/gstinferencepostprocess.c +++ b/gst-libs/gst/r2inference/gstinferencepostprocess.c @@ -51,7 +51,7 @@ gst_fill_classification_meta (GstClassificationMeta * class_meta, g_return_val_if_fail (class_meta != NULL, FALSE); g_return_val_if_fail (prediction != NULL, FALSE); - class_meta->num_labels = predsize / sizeof (gfloat); + class_meta->num_labels = predsize; class_meta->label_probs = g_malloc (class_meta->num_labels * sizeof (gdouble)); for (gint i = 0; i < class_meta->num_labels; ++i) { From a578721cec5d9f00f922f8288f22d879b0da8f7b Mon Sep 17 00:00:00 2001 From: tvlenin Date: Thu, 6 Feb 2020 10:50:14 -0600 Subject: [PATCH 179/188] Update size of new meta predictions --- gst-libs/gst/r2inference/gstinferencepostprocess.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gst-libs/gst/r2inference/gstinferencepostprocess.c b/gst-libs/gst/r2inference/gstinferencepostprocess.c index 50f676e1..fa3cc0aa 100644 --- a/gst-libs/gst/r2inference/gstinferencepostprocess.c +++ b/gst-libs/gst/r2inference/gstinferencepostprocess.c @@ -300,7 +300,7 @@ gst_create_class_from_prediction (GstVideoInference * vi, g_return_val_if_fail (vi != NULL, NULL); - num_classes = predsize / sizeof (gfloat); + num_classes = predsize; /* FIXME: This is just dumb */ if (sizeof (gfloat) != sizeof (gdouble)) { From 6bb63e843af85901599e3bd08d9d1fdbf4d0f974 Mon Sep 17 00:00:00 2001 From: tvlenin Date: Wed, 13 Nov 2019 12:52:51 -0600 Subject: [PATCH 180/188] Refactor pre process to use different formats --- gst-libs/gst/r2inference/gstinferencepreprocess.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/gst-libs/gst/r2inference/gstinferencepreprocess.c b/gst-libs/gst/r2inference/gstinferencepreprocess.c index 0312f3d5..e79cf719 100644 --- a/gst-libs/gst/r2inference/gstinferencepreprocess.c +++ b/gst-libs/gst/r2inference/gstinferencepreprocess.c @@ -22,7 +22,7 @@ #include "gstinferencepreprocess.h" #include -static gboolean gst_check_format_RGB (GstVideoFrame * inframe, +static gboolean gst_configure_format_values (GstVideoFrame * inframe, gint * first_index, gint * last_index, gint * offset, gint * channels); static void gst_apply_means_std (GstVideoFrame * inframe, GstVideoFrame * outframe, gint first_index, gint last_index, @@ -65,7 +65,7 @@ gst_apply_means_std (GstVideoFrame * inframe, GstVideoFrame * outframe, } static gboolean -gst_check_format_RGB (GstVideoFrame * inframe, gint * first_index, +gst_configure_format_values (GstVideoFrame * inframe, gint * first_index, gint * last_index, gint * offset, gint * channels) { g_return_val_if_fail (inframe != NULL, FALSE); @@ -116,7 +116,7 @@ gst_normalize (GstVideoFrame * inframe, GstVideoFrame * outframe, gdouble mean, gint first_index = 0, last_index = 0, offset = 0, channels = 0; g_return_val_if_fail (inframe != NULL, FALSE); g_return_val_if_fail (outframe != NULL, FALSE); - if (gst_check_format_RGB (inframe, &first_index, &last_index, &offset, + if (gst_configure_format_values (inframe, &first_index, &last_index, &offset, &channels) == FALSE) { return FALSE; } @@ -193,7 +193,7 @@ gst_subtract_mean (GstVideoFrame * inframe, GstVideoFrame * outframe, const gdouble std = 1; g_return_val_if_fail (inframe != NULL, FALSE); g_return_val_if_fail (outframe != NULL, FALSE); - if (gst_check_format_RGB (inframe, &first_index, &last_index, &offset, + if (gst_configure_format_values (inframe, &first_index, &last_index, &offset, &channels) == FALSE) { return FALSE; } @@ -211,7 +211,7 @@ gst_pixel_to_float (GstVideoFrame * inframe, GstVideoFrame * outframe, const gdouble std = 1, mean = 0; g_return_val_if_fail (inframe != NULL, FALSE); g_return_val_if_fail (outframe != NULL, FALSE); - if (gst_check_format_RGB (inframe, &first_index, &last_index, &offset, + if (gst_configure_format_values (inframe, &first_index, &last_index, &offset, &channels) == FALSE) { return FALSE; } From 2ae3c25235ec7f38e235acfdc863564ae4fe8a18 Mon Sep 17 00:00:00 2001 From: tvlenin Date: Wed, 13 Nov 2019 12:56:17 -0600 Subject: [PATCH 181/188] Add GRAY8 case to configure format values --- gst-libs/gst/r2inference/gstinferencepreprocess.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/gst-libs/gst/r2inference/gstinferencepreprocess.c b/gst-libs/gst/r2inference/gstinferencepreprocess.c index e79cf719..5e52feb4 100644 --- a/gst-libs/gst/r2inference/gstinferencepreprocess.c +++ b/gst-libs/gst/r2inference/gstinferencepreprocess.c @@ -102,6 +102,11 @@ gst_configure_format_values (GstVideoFrame * inframe, gint * first_index, *last_index = 0; *offset = 1; break; + case GST_VIDEO_FORMAT_GRAY8: + *first_index = 0; + *last_index = 0; + *offset = 0; + break; default: return FALSE; break; From d781dacbd5557e8b6cdfe129b16ca64ac6b80c73 Mon Sep 17 00:00:00 2001 From: tvlenin Date: Wed, 13 Nov 2019 13:04:15 -0600 Subject: [PATCH 182/188] Add cast from GstVideoFormat to r2i::ImageFormat::Id --- gst-libs/gst/r2inference/gstbackend.cc | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/gst-libs/gst/r2inference/gstbackend.cc b/gst-libs/gst/r2inference/gstbackend.cc index b0e723b7..403db957 100644 --- a/gst-libs/gst/r2inference/gstbackend.cc +++ b/gst-libs/gst/r2inference/gstbackend.cc @@ -395,6 +395,28 @@ gst_backend_stop (GstBackend *self, GError **err) { return FALSE; } +static r2i::ImageFormat::Id +gst_backend_cast_format (GstVideoFormat format) +{ + r2i::ImageFormat::Id image_format; + + switch (format) { + case GST_VIDEO_FORMAT_RGB: + image_format = r2i::ImageFormat::Id::RGB; + break; + case GST_VIDEO_FORMAT_BGR: + image_format = r2i::ImageFormat::Id::BGR; + break; + case GST_VIDEO_FORMAT_GRAY8: + image_format = r2i::ImageFormat::Id::GRAY8; + break; + default: + image_format = r2i::ImageFormat::Id::RGB; + break; + } + return image_format; +} + gboolean gst_backend_process_frame (GstBackend *self, GstVideoFrame *input_frame, gpointer *prediction_data, gsize *prediction_size, GError **err) { From 2e8b7ab0f741ca24fca6f05904fdb973cd7dcfce Mon Sep 17 00:00:00 2001 From: tvlenin Date: Wed, 13 Nov 2019 13:10:13 -0600 Subject: [PATCH 183/188] Remove the fixed r2i::ImageFormat::Id::RGB value --- gst-libs/gst/r2inference/gstbackend.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/gst-libs/gst/r2inference/gstbackend.cc b/gst-libs/gst/r2inference/gstbackend.cc index 403db957..018acb3b 100644 --- a/gst-libs/gst/r2inference/gstbackend.cc +++ b/gst-libs/gst/r2inference/gstbackend.cc @@ -396,8 +396,7 @@ gst_backend_stop (GstBackend *self, GError **err) { } static r2i::ImageFormat::Id -gst_backend_cast_format (GstVideoFormat format) -{ +gst_backend_cast_format (GstVideoFormat format) { r2i::ImageFormat::Id image_format; switch (format) { @@ -441,7 +440,8 @@ gst_backend_process_frame (GstBackend *self, GstVideoFrame *input_frame, error = frame->Configure (input_frame->data[0], input_frame->info.width, - input_frame->info.height, r2i::ImageFormat::Id::RGB); + input_frame->info.height, + gst_backend_cast_format(input_frame->info.finfo->format)); if (error.IsError ()) { goto error; } From 9b15f10c4d7eec756eec14c1a88b5a8d8a685d6f Mon Sep 17 00:00:00 2001 From: tvlenin Date: Thu, 6 Feb 2020 17:15:15 -0600 Subject: [PATCH 184/188] Set unknown frame format when not supported --- gst-libs/gst/r2inference/gstbackend.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gst-libs/gst/r2inference/gstbackend.cc b/gst-libs/gst/r2inference/gstbackend.cc index 018acb3b..beb4e415 100644 --- a/gst-libs/gst/r2inference/gstbackend.cc +++ b/gst-libs/gst/r2inference/gstbackend.cc @@ -410,7 +410,7 @@ gst_backend_cast_format (GstVideoFormat format) { image_format = r2i::ImageFormat::Id::GRAY8; break; default: - image_format = r2i::ImageFormat::Id::RGB; + image_format = r2i::ImageFormat::Id::UNKNOWN_FORMAT; break; } return image_format; From 59c90697678f13668c97fc4e2ae1f8c824e5bfdf Mon Sep 17 00:00:00 2001 From: ctrejos Date: Thu, 6 Feb 2020 17:34:35 -0500 Subject: [PATCH 185/188] Remove division on predsize --- .../process/test_gst_fill_classification_meta_function.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/check/process/test_gst_fill_classification_meta_function.c b/tests/check/process/test_gst_fill_classification_meta_function.c index 8b245f45..8c221803 100644 --- a/tests/check/process/test_gst_fill_classification_meta_function.c +++ b/tests/check/process/test_gst_fill_classification_meta_function.c @@ -36,9 +36,7 @@ GST_START_TEST (test_gst_fill_classification_meta) (GstClassificationMeta *) gst_classification_meta_api_get_type (); gst_fill_classification_meta (class_meta, prediction, predsize); - - fail_if (class_meta->num_labels != predsize / sizeof (gfloat)); - + fail_if (class_meta->num_labels != predsize); for (gint i = 0; i < class_meta->num_labels; i++) { fail_if (class_meta->label_probs[i] != values[i]); } From 9a88fb4d36733481d92bb71b1d2aa62aa19d859a Mon Sep 17 00:00:00 2001 From: Carlos Rodriguez Date: Thu, 6 Feb 2020 19:01:09 -0600 Subject: [PATCH 186/188] Use class label in overlay --- ext/opencv/gstinferenceoverlay.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ext/opencv/gstinferenceoverlay.cc b/ext/opencv/gstinferenceoverlay.cc index 76676906..81e5cd2d 100644 --- a/ext/opencv/gstinferenceoverlay.cc +++ b/ext/opencv/gstinferenceoverlay.cc @@ -233,13 +233,13 @@ gst_get_meta (GstInferencePrediction * pred, cv::Mat & cv_mat, classes++; if (classification->num_classes > classification->class_id - && classification->labels != NULL) { + && classification->class_label != NULL) { label = - cv::format ("%s Prob: %f", - classification->labels[classification->class_id], + cv::format ("%s : %0.3f", + classification->class_label, classification->class_prob); } else { - label = cv::format ("Label #%d Prob: %f", classification->class_id, + label = cv::format ("Label #%d : %0.3f", classification->class_id, classification->class_prob); } cv::putText (cv_mat, label, cv::Point (box.x + box.width, From a3382d945b1fb12314bd77c4f3eaa30d4d302784 Mon Sep 17 00:00:00 2001 From: Carlos Rodriguez Date: Thu, 6 Feb 2020 19:01:59 -0600 Subject: [PATCH 187/188] Use RGB as default format --- gst-libs/gst/r2inference/gstbackend.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gst-libs/gst/r2inference/gstbackend.cc b/gst-libs/gst/r2inference/gstbackend.cc index beb4e415..018acb3b 100644 --- a/gst-libs/gst/r2inference/gstbackend.cc +++ b/gst-libs/gst/r2inference/gstbackend.cc @@ -410,7 +410,7 @@ gst_backend_cast_format (GstVideoFormat format) { image_format = r2i::ImageFormat::Id::GRAY8; break; default: - image_format = r2i::ImageFormat::Id::UNKNOWN_FORMAT; + image_format = r2i::ImageFormat::Id::RGB; break; } return image_format; From 4f4808621c7623ebe4bf80610b4a240d3fd6831f Mon Sep 17 00:00:00 2001 From: Carlos Rodriguez Date: Thu, 6 Feb 2020 19:47:30 -0600 Subject: [PATCH 188/188] Update r2inference version requirement --- configure.ac | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index f763d3c5..87a0819f 100644 --- a/configure.ac +++ b/configure.ac @@ -288,8 +288,9 @@ if test "$USE_OPENCV" = "yes"; then fi dnl *** r2inference *** +R2INFERENCE_REQ=0.4.0 AG_GST_CHECK_FEATURE(R2INFERENCE, [RidgeRun\'s Inference Framework], r2inference, [ - AG_GST_PKG_CHECK_MODULES(R2INFERENCE, r2inference-0.0) + AG_GST_PKG_CHECK_MODULES(R2INFERENCE, r2inference-0.0 >= $R2INFERENCE_REQ) ],[],[],[ AC_MSG_ERROR([Please install R2Inference from https://github.com/RidgeRun/r2inference.git]) ])