From 38b20f764e21716d38e264163dfaadfe9f1fc491 Mon Sep 17 00:00:00 2001 From: Carlos Rodriguez Date: Mon, 24 Feb 2020 09:09:14 -0600 Subject: [PATCH 01/16] Bump version number --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 28bf4014..b4821eb6 100644 --- a/configure.ac +++ b/configure.ac @@ -5,7 +5,7 @@ dnl please read gstreamer/docs/random/autotools before changing this file dnl initialize autoconf dnl releases only do -Wall, git and prerelease does -Werror too dnl use a three digit version number for releases, and four for git/pre -AC_INIT([GStreamer Inference],[0.7.1.1],[https://github.com/RidgeRun/gst-inference/issues],[gst-inference]) +AC_INIT([GStreamer Inference],[0.8.0.1],[https://github.com/RidgeRun/gst-inference/issues],[gst-inference]) AG_GST_INIT From 026d89ea3402d511dee13c81fe65e5fc774f487b Mon Sep 17 00:00:00 2001 From: Carlos Rodriguez Date: Mon, 24 Feb 2020 10:40:10 -0600 Subject: [PATCH 02/16] Rename detectioncrop element to inferencecrop --- gst/inferencecrop/Makefile.am | 4 +- ...stdetectioncrop.cc => gstinferencecrop.cc} | 124 +++++++++--------- ...{gstdetectioncrop.h => gstinferencecrop.h} | 8 +- 3 files changed, 68 insertions(+), 68 deletions(-) rename gst/inferencecrop/{gstdetectioncrop.cc => gstinferencecrop.cc} (77%) rename gst/inferencecrop/{gstdetectioncrop.h => gstinferencecrop.h} (82%) diff --git a/gst/inferencecrop/Makefile.am b/gst/inferencecrop/Makefile.am index 0b96c0a0..cdda14d3 100644 --- a/gst/inferencecrop/Makefile.am +++ b/gst/inferencecrop/Makefile.am @@ -1,7 +1,7 @@ plugin_LTLIBRARIES = libgstinferencecrop.la libgstinferencecrop_la_SOURCES = \ - gstdetectioncrop.cc \ + gstinferencecrop.cc \ cropelement.cc \ videocrop.cc @@ -31,6 +31,6 @@ libgstinferencecrop_la_LIBTOOLFLAGS = \ $(GST_PLUGIN_LIBTOOLFLAGS) noinst_HEADERS = \ - gstdetectioncrop.h \ + gstinferencecrop.h \ cropelement.h \ videocrop.h diff --git a/gst/inferencecrop/gstdetectioncrop.cc b/gst/inferencecrop/gstinferencecrop.cc similarity index 77% rename from gst/inferencecrop/gstdetectioncrop.cc rename to gst/inferencecrop/gstinferencecrop.cc index 3b2a7592..097b103b 100644 --- a/gst/inferencecrop/gstdetectioncrop.cc +++ b/gst/inferencecrop/gstinferencecrop.cc @@ -20,9 +20,9 @@ */ /** - * SECTION:element-gstdetectioncrop + * SECTION:element-gstinferencecrop * - * The detectioncrop element parses a detectionmeta and crops the incomming + * The inferencecrop element parses a inferencemeta and crops the incomming * image to the bounding box. * * @@ -31,9 +31,9 @@ * gst-launch-1.0 v4l2src device=$CAMERA ! videoconvert ! tee name=t t. ! videoscale ! queue ! net.sink_model t. ! queue ! net.sink_bypass tinyyolov2 name=net model-location=$MODEL_LOCATION ! \ backend=tensorflow backend::input-layer=$INPUT_LAYER backend::output-layer=OUTPUT_LAYER net.src_bypass \ - detectioncrop aspect-ratio=1/1 ! videoscale ! ximagesink sync=false + inferencecrop aspect-ratio=1/1 ! videoscale ! ximagesink sync=false * ]| - * Process video frames from the camera using a detectioncrop model. + * Crops video frames based on inferencemeta information. * */ @@ -42,7 +42,7 @@ #endif -#include "gstdetectioncrop.h" +#include "gstinferencecrop.h" #include "gst/r2inference/gstinferencemeta.h" #include "videocrop.h" @@ -59,27 +59,27 @@ static GstStaticPadTemplate src_template = GST_STATIC_PAD_TEMPLATE ("src", GST_PAD_ALWAYS, GST_STATIC_CAPS_ANY); -GST_DEBUG_CATEGORY_STATIC (gst_detection_crop_debug_category); -#define GST_CAT_DEFAULT gst_detection_crop_debug_category +GST_DEBUG_CATEGORY_STATIC (gst_inference_crop_debug_category); +#define GST_CAT_DEFAULT gst_inference_crop_debug_category -static void gst_detection_crop_finalize (GObject *object); -static void gst_detection_crop_set_property (GObject *object, +static void gst_inference_crop_finalize (GObject *object); +static void gst_inference_crop_set_property (GObject *object, guint property_id, const GValue *value, GParamSpec *pspec); -static void gst_detection_crop_get_property (GObject *object, +static void gst_inference_crop_get_property (GObject *object, guint property_id, GValue *value, GParamSpec *pspec); -static GstStateChangeReturn gst_detection_crop_change_state (GstElement * +static GstStateChangeReturn gst_inference_crop_change_state (GstElement * element, GstStateChange transition); -static gboolean gst_detection_crop_start (GstDetectionCrop *self); -static void gst_detection_crop_set_caps (GstPad *pad, GParamSpec *unused, - GstDetectionCrop *self); -static void gst_detection_crop_new_buffer_size (GstDetectionCrop *self, gint x, +static gboolean gst_inference_crop_start (GstInferenceCrop *self); +static void gst_inference_crop_set_caps (GstPad *pad, GParamSpec *unused, + GstInferenceCrop *self); +static void gst_inference_crop_new_buffer_size (GstInferenceCrop *self, gint x, gint y, gint width, gint height, gint width_ratio, gint height_ratio, gint *top, gint *bottom, gint *right, gint *left); -static GstPadProbeReturn gst_detection_crop_new_buffer (GstPad *pad, - GstPadProbeInfo *info, GstDetectionCrop *self); -static void gst_detection_crop_find_predictions (GstDetectionCrop *self, - gint *num_detections, GstInferenceMeta *meta, GList **list, +static GstPadProbeReturn gst_inference_crop_new_buffer (GstPad *pad, + GstPadProbeInfo *info, GstInferenceCrop *self); +static void gst_inference_crop_find_predictions (GstInferenceCrop *self, + gint *num_inferences, GstInferenceMeta *meta, GList **list, GstInferencePrediction *pred); @@ -91,7 +91,7 @@ enum { PROP_CROP_ASPECT_RATIO, }; -struct _GstDetectionCrop { +struct _GstInferenceCrop { GstBin parent; GstPad *sinkpad; GstPad *srcpad; @@ -102,19 +102,19 @@ struct _GstDetectionCrop { gint height; }; -struct _GstDetectionCropClass { +struct _GstInferenceCropClass { GstBinClass parent; }; /* class initialization */ -G_DEFINE_TYPE_WITH_CODE (GstDetectionCrop, gst_detection_crop, +G_DEFINE_TYPE_WITH_CODE (GstInferenceCrop, gst_inference_crop, GST_TYPE_BIN, - GST_DEBUG_CATEGORY_INIT (gst_detection_crop_debug_category, "detectioncrop", - 0, "debug category for detectioncrop element")); + GST_DEBUG_CATEGORY_INIT (gst_inference_crop_debug_category, "inferencecrop", + 0, "debug category for inferencecrop element")); static void -gst_detection_crop_class_init (GstDetectionCropClass *klass) { +gst_inference_crop_class_init (GstInferenceCropClass *klass) { GstElementClass *element_class = GST_ELEMENT_CLASS (klass); GObjectClass *object_class = G_OBJECT_CLASS (klass); @@ -124,21 +124,21 @@ gst_detection_crop_class_init (GstDetectionCropClass *klass) { gst_static_pad_template_get (&src_template)); gst_element_class_set_static_metadata (GST_ELEMENT_CLASS (klass), - "detectioncrop", "Filter", + "inferencecrop", "Filter", "Crops an incoming image based on an inference prediction bounding box", "Michael Gruner \n\t\t\t" " Lenin Torres "); element_class->change_state = - GST_DEBUG_FUNCPTR (gst_detection_crop_change_state); + GST_DEBUG_FUNCPTR (gst_inference_crop_change_state); - object_class->finalize = gst_detection_crop_finalize; - object_class->set_property = gst_detection_crop_set_property; - object_class->get_property = gst_detection_crop_get_property; + object_class->finalize = gst_inference_crop_finalize; + object_class->set_property = gst_inference_crop_set_property; + object_class->get_property = gst_inference_crop_get_property; g_object_class_install_property (object_class, PROP_CROP_ASPECT_RATIO, gst_param_spec_fraction ("aspect-ratio", "Aspect Ratio", - "Aspect ratio to crop the detections, width and height separated by '/'. " + "Aspect ratio to crop the predictions, width and height separated by '/'. " "If set to 0/1 it maintains the aspect ratio of each bounding box.", 0, 1, G_MAXINT, 1, PROP_CROP_RATIO_DEFAULT_WIDTH, PROP_CROP_RATIO_DEFAULT_HEIGHT, @@ -146,7 +146,7 @@ gst_detection_crop_class_init (GstDetectionCropClass *klass) { } static void -gst_detection_crop_init (GstDetectionCrop *self) { +gst_inference_crop_init (GstInferenceCrop *self) { GstElement *element; GstPad *sinkpad, *sinkgpad, *srcpad, *srcgpad; @@ -173,9 +173,9 @@ gst_detection_crop_init (GstDetectionCrop *self) { gst_element_add_pad (GST_ELEMENT (self), sinkgpad); g_signal_connect (sinkgpad, "notify::caps", - G_CALLBACK (gst_detection_crop_set_caps), self); + G_CALLBACK (gst_inference_crop_set_caps), self); gst_pad_add_probe (sinkgpad, GST_PAD_PROBE_TYPE_BUFFER, - (GstPadProbeCallback) gst_detection_crop_new_buffer, self, NULL); + (GstPadProbeCallback) gst_inference_crop_new_buffer, self, NULL); srcpad = self->element->GetSrcPad (); g_return_if_fail (srcpad); @@ -191,8 +191,8 @@ gst_detection_crop_init (GstDetectionCrop *self) { } static void -gst_detection_crop_finalize (GObject *object) { - GstDetectionCrop *self = GST_DETECTION_CROP (object); +gst_inference_crop_finalize (GObject *object) { + GstInferenceCrop *self = GST_INFERENCE_CROP (object); g_return_if_fail(self); @@ -202,13 +202,13 @@ gst_detection_crop_finalize (GObject *object) { gst_object_unref (self->srcpad); self->srcpad = NULL; - G_OBJECT_CLASS (gst_detection_crop_parent_class)->finalize (object); + G_OBJECT_CLASS (gst_inference_crop_parent_class)->finalize (object); } static void -gst_detection_crop_set_property (GObject *object, guint property_id, +gst_inference_crop_set_property (GObject *object, guint property_id, const GValue *value, GParamSpec *pspec) { - GstDetectionCrop *self = GST_DETECTION_CROP (object); + GstInferenceCrop *self = GST_INFERENCE_CROP (object); GST_DEBUG_OBJECT (self, "set_property"); @@ -228,9 +228,9 @@ gst_detection_crop_set_property (GObject *object, guint property_id, } static void -gst_detection_crop_get_property (GObject *object, guint property_id, +gst_inference_crop_get_property (GObject *object, guint property_id, GValue *value, GParamSpec *pspec) { - GstDetectionCrop *self = GST_DETECTION_CROP (object); + GstInferenceCrop *self = GST_INFERENCE_CROP (object); GST_DEBUG_OBJECT (self, "get_property"); @@ -247,21 +247,21 @@ gst_detection_crop_get_property (GObject *object, guint property_id, } static gboolean -gst_detection_crop_start (GstDetectionCrop *self) { +gst_inference_crop_start (GstInferenceCrop *self) { g_return_val_if_fail (self, FALSE); return self->element->Validate (); } static GstStateChangeReturn -gst_detection_crop_change_state (GstElement *element, +gst_inference_crop_change_state (GstElement *element, GstStateChange transition) { GstStateChangeReturn ret; - GstDetectionCrop *self = GST_DETECTION_CROP (element); + GstInferenceCrop *self = GST_INFERENCE_CROP (element); switch (transition) { case GST_STATE_CHANGE_READY_TO_PAUSED: - if (FALSE == gst_detection_crop_start (self)) { + if (FALSE == gst_inference_crop_start (self)) { GST_ERROR_OBJECT (self, "Failed to start"); ret = GST_STATE_CHANGE_FAILURE; goto out; @@ -270,7 +270,7 @@ gst_detection_crop_change_state (GstElement *element, break; } ret = - GST_ELEMENT_CLASS (gst_detection_crop_parent_class)->change_state + GST_ELEMENT_CLASS (gst_inference_crop_parent_class)->change_state (element, transition); if (GST_STATE_CHANGE_FAILURE == ret) { GST_ERROR_OBJECT (self, "Parent failed to change state"); @@ -282,8 +282,8 @@ gst_detection_crop_change_state (GstElement *element, } static void -gst_detection_crop_set_caps (GstPad *pad, GParamSpec *unused, - GstDetectionCrop *self) { +gst_inference_crop_set_caps (GstPad *pad, GParamSpec *unused, + GstInferenceCrop *self) { GstCaps *caps; GstStructure *st; gint width; @@ -305,7 +305,7 @@ gst_detection_crop_set_caps (GstPad *pad, GParamSpec *unused, } static void -gst_detection_crop_new_buffer_size (GstDetectionCrop *self, gint x, gint y, +gst_inference_crop_new_buffer_size (GstInferenceCrop *self, gint x, gint y, gint width, gint height, gint width_ratio, gint height_ratio, gint *top, gint *bottom, gint *right, gint *left) { @@ -357,14 +357,14 @@ gst_detection_crop_new_buffer_size (GstDetectionCrop *self, gint x, gint y, } static void -gst_detection_crop_find_predictions (GstDetectionCrop *self, - gint *num_detections, +gst_inference_crop_find_predictions (GstInferenceCrop *self, + gint *num_inferences, GstInferenceMeta *meta, GList **list, GstInferencePrediction *pred) { GSList *children_list = NULL; GSList *iter = NULL; g_return_if_fail (self); - g_return_if_fail (num_detections); + g_return_if_fail (num_inferences); g_return_if_fail (meta); g_return_if_fail (list); g_return_if_fail (pred); @@ -374,22 +374,22 @@ gst_detection_crop_find_predictions (GstDetectionCrop *self, for (iter = children_list; iter != NULL; iter = g_slist_next(iter)) { GstInferencePrediction *predict = (GstInferencePrediction *)iter->data; - gst_detection_crop_find_predictions (self, num_detections, meta, list, + gst_inference_crop_find_predictions (self, num_inferences, meta, list, predict ); } if (FALSE == G_NODE_IS_ROOT(pred->predictions) && TRUE == pred->enabled ) { *list = g_list_append (*list, pred); - *num_detections = *num_detections + 1; + *num_inferences = *num_inferences + 1; } } static GstPadProbeReturn -gst_detection_crop_new_buffer (GstPad *pad, GstPadProbeInfo *info, - GstDetectionCrop *self) { +gst_inference_crop_new_buffer (GstPad *pad, GstPadProbeInfo *info, + GstInferenceCrop *self) { GstBuffer *buffer; GstInferenceMeta *inference_meta; - gint num_detections; + gint num_inferences; gint crop_width_ratio; gint crop_height_ratio; BoundingBox box; @@ -414,8 +414,8 @@ gst_detection_crop_new_buffer (GstPad *pad, GstPadProbeInfo *info, goto out; } - num_detections = 0; - gst_detection_crop_find_predictions (self, &num_detections, + num_inferences = 0; + gst_inference_crop_find_predictions (self, &num_inferences, inference_meta, &list, inference_meta->prediction); for (iter = list; iter != NULL; iter = g_list_next(iter)) { @@ -427,7 +427,7 @@ gst_detection_crop_new_buffer (GstPad *pad, GstPadProbeInfo *info, GST_LOG_OBJECT (self, "BBox: %dx%dx%dx%d", box.x, box.y, box.width, box.height); - gst_detection_crop_new_buffer_size (self, box.x, box.y, box.width, box.height, + gst_inference_crop_new_buffer_size (self, box.x, box.y, box.width, box.height, crop_width_ratio, crop_height_ratio, &top, &bottom, &right, &left); self->element->SetCroppingSize ((gint) top, (gint) bottom, (gint) right, @@ -470,8 +470,8 @@ plugin_init (GstPlugin *plugin) { gboolean ret = TRUE; ret = - gst_element_register (plugin, "detectioncrop", GST_RANK_NONE, - GST_TYPE_DETECTION_CROP); + gst_element_register (plugin, "inferencecrop", GST_RANK_NONE, + GST_TYPE_INFERENCE_CROP); return ret; } diff --git a/gst/inferencecrop/gstdetectioncrop.h b/gst/inferencecrop/gstinferencecrop.h similarity index 82% rename from gst/inferencecrop/gstdetectioncrop.h rename to gst/inferencecrop/gstinferencecrop.h index cb577819..7e26d3bd 100644 --- a/gst/inferencecrop/gstdetectioncrop.h +++ b/gst/inferencecrop/gstinferencecrop.h @@ -19,15 +19,15 @@ * */ -#ifndef _GST_DETECTIONCROP_H_ -#define _GST_DETECTIONCROP_H_ +#ifndef _GST_INFERENCECROP_H_ +#define _GST_INFERENCECROP_H_ #include G_BEGIN_DECLS -#define GST_TYPE_DETECTION_CROP gst_detection_crop_get_type () -G_DECLARE_FINAL_TYPE (GstDetectionCrop, gst_detection_crop, GST, DETECTION_CROP, +#define GST_TYPE_INFERENCE_CROP gst_inference_crop_get_type () +G_DECLARE_FINAL_TYPE (GstInferenceCrop, gst_inference_crop, GST, INFERENCE_CROP, GstBin) G_END_DECLS From 23dfd014352442afc388f6c1132ea9348b2af352 Mon Sep 17 00:00:00 2001 From: Carlos Rodriguez Date: Mon, 24 Feb 2020 11:38:17 -0600 Subject: [PATCH 03/16] Collapse inference utils elements in one plugin --- configure.ac | 4 +- gst/Makefile.am | 4 +- gst/inferencedebug/Makefile.am | 27 -------- gst/inferencefilter/Makefile.am | 27 -------- .../Makefile.am | 19 ++++-- .../cropelement.cc | 0 .../cropelement.h | 0 .../gstinferencecrop.cc | 17 ----- .../gstinferencecrop.h | 0 .../gstinferencedebug.c | 13 ---- .../gstinferencedebug.h | 0 .../gstinferencefilter.c | 14 ---- .../gstinferencefilter.h | 0 gst/inferenceutils/gstinferenceutils.c | 64 +++++++++++++++++++ .../videocrop.cc | 0 .../videocrop.h | 0 16 files changed, 78 insertions(+), 111 deletions(-) delete mode 100644 gst/inferencedebug/Makefile.am delete mode 100644 gst/inferencefilter/Makefile.am rename gst/{inferencecrop => inferenceutils}/Makefile.am (58%) rename gst/{inferencecrop => inferenceutils}/cropelement.cc (100%) rename gst/{inferencecrop => inferenceutils}/cropelement.h (100%) rename gst/{inferencecrop => inferenceutils}/gstinferencecrop.cc (96%) rename gst/{inferencecrop => inferenceutils}/gstinferencecrop.h (100%) rename gst/{inferencedebug => inferenceutils}/gstinferencedebug.c (92%) rename gst/{inferencedebug => inferenceutils}/gstinferencedebug.h (100%) rename gst/{inferencefilter => inferenceutils}/gstinferencefilter.c (95%) rename gst/{inferencefilter => inferenceutils}/gstinferencefilter.h (100%) create mode 100644 gst/inferenceutils/gstinferenceutils.c rename gst/{inferencecrop => inferenceutils}/videocrop.cc (100%) rename gst/{inferencecrop => inferenceutils}/videocrop.h (100%) diff --git a/configure.ac b/configure.ac index b4821eb6..1ef6a129 100644 --- a/configure.ac +++ b/configure.ac @@ -382,9 +382,7 @@ ext/Makefile ext/opencv/Makefile ext/r2inference/Makefile gst/Makefile -gst/inferencecrop/Makefile -gst/inferencefilter/Makefile -gst/inferencedebug/Makefile +gst/inferenceutils/Makefile gst-libs/Makefile gst-libs/gst/Makefile gst-libs/gst/opencv/Makefile diff --git a/gst/Makefile.am b/gst/Makefile.am index 1c7fd945..8131eb85 100644 --- a/gst/Makefile.am +++ b/gst/Makefile.am @@ -1,4 +1,2 @@ SUBDIRS = \ - inferencecrop \ - inferencefilter \ - inferencedebug + inferenceutils diff --git a/gst/inferencedebug/Makefile.am b/gst/inferencedebug/Makefile.am deleted file mode 100644 index 3f9c28a3..00000000 --- a/gst/inferencedebug/Makefile.am +++ /dev/null @@ -1,27 +0,0 @@ -plugin_LTLIBRARIES = libgstinferencedebug.la - -libgstinferencedebug_la_SOURCES = \ - gstinferencedebug.c - -libgstinferencedebug_la_CFLAGS = \ - $(GST_CFLAGS) \ - $(GST_BASE_CFLAGS) \ - $(GST_PLUGINS_BASE_CFLAGS) \ - -I$(top_srcdir)/gst-libs \ - $(R2INFERENCE_CFLAGS) - -libgstinferencedebug_la_LIBADD = \ - $(GST_LIBS) \ - $(GST_BASE_LIBS) \ - $(GST_PLUGINS_BASE_LIBS) \ - $(R2INFERENCE_LIBS) \ - $(top_builddir)/gst-libs/gst/r2inference/libgstinference-@GST_API_VERSION@.la - -libgstinferencedebug_la_LDFLAGS = \ -$(GST_PLUGIN_LDFLAGS) - -libgstinferencedebug_la_LIBTOOLFLAGS = \ - $(GST_PLUGIN_LIBTOOLFLAGS) - -noinst_HEADERS = \ - gstinferencedebug.h diff --git a/gst/inferencefilter/Makefile.am b/gst/inferencefilter/Makefile.am deleted file mode 100644 index 84b68a81..00000000 --- a/gst/inferencefilter/Makefile.am +++ /dev/null @@ -1,27 +0,0 @@ -plugin_LTLIBRARIES = libgstinferencefilter.la - -libgstinferencefilter_la_SOURCES = \ - gstinferencefilter.c - -libgstinferencefilter_la_CFLAGS = \ - $(GST_CFLAGS) \ - $(GST_BASE_CFLAGS) \ - $(GST_PLUGINS_BASE_CFLAGS) \ - -I$(top_srcdir)/gst-libs \ - $(R2INFERENCE_CFLAGS) - -libgstinferencefilter_la_LIBADD = \ - $(GST_LIBS) \ - $(GST_BASE_LIBS) \ - $(GST_PLUGINS_BASE_LIBS) \ - $(R2INFERENCE_LIBS) \ - $(top_builddir)/gst-libs/gst/r2inference/libgstinference-@GST_API_VERSION@.la - -libgstinferencefilter_la_LDFLAGS = \ -$(GST_PLUGIN_LDFLAGS) - -libgstinferencefilter_la_LIBTOOLFLAGS = \ - $(GST_PLUGIN_LIBTOOLFLAGS) - -noinst_HEADERS = \ - gstinferencefilter.h diff --git a/gst/inferencecrop/Makefile.am b/gst/inferenceutils/Makefile.am similarity index 58% rename from gst/inferencecrop/Makefile.am rename to gst/inferenceutils/Makefile.am index cdda14d3..d64a9a3b 100644 --- a/gst/inferencecrop/Makefile.am +++ b/gst/inferenceutils/Makefile.am @@ -1,36 +1,41 @@ -plugin_LTLIBRARIES = libgstinferencecrop.la +plugin_LTLIBRARIES = libgstinferenceutils.la -libgstinferencecrop_la_SOURCES = \ +libgstinferenceutils_la_SOURCES = \ + gstinferenceutils.c \ + gstinferencefilter.c \ + gstinferencedebug.c \ gstinferencecrop.cc \ cropelement.cc \ videocrop.cc -libgstinferencecrop_la_CFLAGS = \ +libgstinferenceutils_la_CFLAGS = \ $(GST_CFLAGS) \ $(GST_BASE_CFLAGS) \ $(GST_PLUGINS_BASE_CFLAGS) \ -I$(top_srcdir)/gst-libs -libgstinferencecrop_la_CXXFLAGS = \ +libgstinferenceutils_la_CXXFLAGS = \ $(GST_CFLAGS) \ $(GST_BASE_CFLAGS) \ $(GST_PLUGINS_BASE_CFLAGS) \ -I$(top_srcdir)/gst-libs \ -std=c++11 -libgstinferencecrop_la_LIBADD = \ +libgstinferenceutils_la_LIBADD = \ $(GST_LIBS) \ $(GST_BASE_LIBS) \ $(GST_PLUGINS_BASE_LIBS) \ $(top_builddir)/gst-libs/gst/r2inference/libgstinference-@GST_API_VERSION@.la -libgstinferencecrop_la_LDFLAGS = \ +libgstinferenceutils_la_LDFLAGS = \ $(GST_PLUGIN_LDFLAGS) -libgstinferencecrop_la_LIBTOOLFLAGS = \ +libgstinferenceutils_la_LIBTOOLFLAGS = \ $(GST_PLUGIN_LIBTOOLFLAGS) noinst_HEADERS = \ + gstinferencefilter.h \ + gstinferencedebug.h \ gstinferencecrop.h \ cropelement.h \ videocrop.h diff --git a/gst/inferencecrop/cropelement.cc b/gst/inferenceutils/cropelement.cc similarity index 100% rename from gst/inferencecrop/cropelement.cc rename to gst/inferenceutils/cropelement.cc diff --git a/gst/inferencecrop/cropelement.h b/gst/inferenceutils/cropelement.h similarity index 100% rename from gst/inferencecrop/cropelement.h rename to gst/inferenceutils/cropelement.h diff --git a/gst/inferencecrop/gstinferencecrop.cc b/gst/inferenceutils/gstinferencecrop.cc similarity index 96% rename from gst/inferencecrop/gstinferencecrop.cc rename to gst/inferenceutils/gstinferencecrop.cc index 097b103b..f8180f27 100644 --- a/gst/inferencecrop/gstinferencecrop.cc +++ b/gst/inferenceutils/gstinferencecrop.cc @@ -464,20 +464,3 @@ gst_inference_crop_new_buffer (GstPad *pad, GstPadProbeInfo *info, return ret; } - -static gboolean -plugin_init (GstPlugin *plugin) { - gboolean ret = TRUE; - - ret = - gst_element_register (plugin, "inferencecrop", GST_RANK_NONE, - GST_TYPE_INFERENCE_CROP); - - return ret; -} - -GST_PLUGIN_DEFINE (GST_VERSION_MAJOR, - GST_VERSION_MINOR, - inferencecrop, - "Crops an incoming image based on an inference prediction bounding box", - plugin_init, VERSION, "LGPL", PACKAGE_NAME, GST_PACKAGE_ORIGIN) diff --git a/gst/inferencecrop/gstinferencecrop.h b/gst/inferenceutils/gstinferencecrop.h similarity index 100% rename from gst/inferencecrop/gstinferencecrop.h rename to gst/inferenceutils/gstinferencecrop.h diff --git a/gst/inferencedebug/gstinferencedebug.c b/gst/inferenceutils/gstinferencedebug.c similarity index 92% rename from gst/inferencedebug/gstinferencedebug.c rename to gst/inferenceutils/gstinferencedebug.c index 60559d5a..b8697af6 100644 --- a/gst/inferencedebug/gstinferencedebug.c +++ b/gst/inferenceutils/gstinferencedebug.c @@ -145,16 +145,3 @@ gst_inference_debug_transform_ip (GstBaseTransform * trans, GstBuffer * buf) return GST_FLOW_OK; } - -static gboolean -plugin_init (GstPlugin * plugin) -{ - return gst_element_register (plugin, "inferencedebug", GST_RANK_NONE, - GST_TYPE_INFERENCE_DEBUG); -} - -GST_PLUGIN_DEFINE (GST_VERSION_MAJOR, - GST_VERSION_MINOR, - inferencedebug, - "Print InferenceMeta Predictions", - plugin_init, VERSION, "LGPL", PACKAGE_NAME, GST_PACKAGE_ORIGIN) diff --git a/gst/inferencedebug/gstinferencedebug.h b/gst/inferenceutils/gstinferencedebug.h similarity index 100% rename from gst/inferencedebug/gstinferencedebug.h rename to gst/inferenceutils/gstinferencedebug.h diff --git a/gst/inferencefilter/gstinferencefilter.c b/gst/inferenceutils/gstinferencefilter.c similarity index 95% rename from gst/inferencefilter/gstinferencefilter.c rename to gst/inferenceutils/gstinferencefilter.c index 5b821088..3653248b 100644 --- a/gst/inferencefilter/gstinferencefilter.c +++ b/gst/inferenceutils/gstinferencefilter.c @@ -271,17 +271,3 @@ gst_inferencefilter_transform_ip (GstBaseTransform * trans, GstBuffer * buf) filter, reset); return GST_FLOW_OK; } - -static gboolean -plugin_init (GstPlugin * plugin) -{ - return gst_element_register (plugin, "inferencefilter", GST_RANK_NONE, - GST_TYPE_INFERENCEFILTER); -} - - -GST_PLUGIN_DEFINE (GST_VERSION_MAJOR, - GST_VERSION_MINOR, - inferencefilter, - "Enables/disables selected classes on inference meta to be processed", - plugin_init, VERSION, "LGPL", PACKAGE_NAME, GST_PACKAGE_ORIGIN) diff --git a/gst/inferencefilter/gstinferencefilter.h b/gst/inferenceutils/gstinferencefilter.h similarity index 100% rename from gst/inferencefilter/gstinferencefilter.h rename to gst/inferenceutils/gstinferencefilter.h diff --git a/gst/inferenceutils/gstinferenceutils.c b/gst/inferenceutils/gstinferenceutils.c new file mode 100644 index 00000000..9809a2f0 --- /dev/null +++ b/gst/inferenceutils/gstinferenceutils.c @@ -0,0 +1,64 @@ +/* + * GStreamer + * Copyright (C) 2018-2020 RidgeRun + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License as published by the Free Software Foundation; either + * version 2 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 02111-1307, USA. + * + */ + +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + +#include "gstinferencecrop.h" +#include "gstinferencedebug.h" +#include "gstinferencefilter.h" + +static gboolean +plugin_init (GstPlugin * plugin) +{ + gboolean ret = TRUE; + + ret = + gst_element_register (plugin, "inferencecrop", GST_RANK_NONE, + GST_TYPE_INFERENCE_CROP); + if (!ret) { + goto out; + } + + ret = + gst_element_register (plugin, "inferencedebug", GST_RANK_NONE, + GST_TYPE_INFERENCE_DEBUG); + if (!ret) { + goto out; + } + + ret = + gst_element_register (plugin, "inferencefilter", GST_RANK_NONE, + GST_TYPE_INFERENCEFILTER); + if (!ret) { + goto out; + } + +out: + return ret; +} + +GST_PLUGIN_DEFINE (GST_VERSION_MAJOR, + GST_VERSION_MINOR, + inferenceutils, + "Utils to process inferencemeta information", + plugin_init, VERSION, "LGPL", PACKAGE_NAME, GST_PACKAGE_ORIGIN) diff --git a/gst/inferencecrop/videocrop.cc b/gst/inferenceutils/videocrop.cc similarity index 100% rename from gst/inferencecrop/videocrop.cc rename to gst/inferenceutils/videocrop.cc diff --git a/gst/inferencecrop/videocrop.h b/gst/inferenceutils/videocrop.h similarity index 100% rename from gst/inferencecrop/videocrop.h rename to gst/inferenceutils/videocrop.h From 7f3ef9dee557e43d6f0e1c391475f35e61c290b3 Mon Sep 17 00:00:00 2001 From: Carlos Rodriguez Date: Mon, 24 Feb 2020 11:44:12 -0600 Subject: [PATCH 04/16] Change InferenceFilter namespace --- gst/inferenceutils/gstinferencefilter.c | 63 +++++++++++++------------ gst/inferenceutils/gstinferencefilter.h | 10 ++-- gst/inferenceutils/gstinferenceutils.c | 2 +- 3 files changed, 38 insertions(+), 37 deletions(-) diff --git a/gst/inferenceutils/gstinferencefilter.c b/gst/inferenceutils/gstinferencefilter.c index 3653248b..1f90dd26 100644 --- a/gst/inferenceutils/gstinferencefilter.c +++ b/gst/inferenceutils/gstinferencefilter.c @@ -47,25 +47,25 @@ #include -GST_DEBUG_CATEGORY_STATIC (gst_inferencefilter_debug_category); -#define GST_CAT_DEFAULT gst_inferencefilter_debug_category +GST_DEBUG_CATEGORY_STATIC (gst_inference_filter_debug_category); +#define GST_CAT_DEFAULT gst_inference_filter_debug_category -#define GST_INFERENCEFILTER_PROPERTY_FLAGS (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS) +#define GST_INFERENCE_FILTER_PROPERTY_FLAGS (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS) #define PROP_FILTER_CLASS_LABEL_DEFAULT -1 #define PROP_FILTER_CLASS_LABEL_MIN -1 #define PROP_RESET_ENABLE_DEFAULT FALSE /* prototypes */ -static void gst_inferencefilter_set_property (GObject * object, +static void gst_inference_filter_set_property (GObject * object, guint property_id, const GValue * value, GParamSpec * pspec); -static void gst_inferencefilter_get_property (GObject * object, +static void gst_inference_filter_get_property (GObject * object, guint property_id, GValue * value, GParamSpec * pspec); -static void gst_inferencefilter_filter_enable (GstInferencefilter * +static void gst_inference_filter_filter_enable (GstInferenceFilter * inferencefilter, GstInferencePrediction * rot, gint class_id, gboolean reset); -static GstFlowReturn gst_inferencefilter_transform_ip (GstBaseTransform * trans, - GstBuffer * buf); +static GstFlowReturn gst_inference_filter_transform_ip (GstBaseTransform * + trans, GstBuffer * buf); enum { @@ -74,7 +74,7 @@ enum PROP_RESET_ENABLE, }; -struct _GstInferencefilter +struct _GstInferenceFilter { GstBaseTransform base_inferencefilter; gint filter_class; @@ -84,13 +84,13 @@ struct _GstInferencefilter /* pad templates */ -static GstStaticPadTemplate gst_inferencefilter_src_template = +static GstStaticPadTemplate gst_inference_filter_src_template = GST_STATIC_PAD_TEMPLATE ("src", GST_PAD_SRC, GST_PAD_ALWAYS, GST_STATIC_CAPS_ANY); -static GstStaticPadTemplate gst_inferencefilter_sink_template = +static GstStaticPadTemplate gst_inference_filter_sink_template = GST_STATIC_PAD_TEMPLATE ("sink", GST_PAD_SINK, GST_PAD_ALWAYS, @@ -99,13 +99,13 @@ GST_STATIC_PAD_TEMPLATE ("sink", /* class initialization */ -G_DEFINE_TYPE_WITH_CODE (GstInferencefilter, gst_inferencefilter, +G_DEFINE_TYPE_WITH_CODE (GstInferenceFilter, gst_inference_filter, GST_TYPE_BASE_TRANSFORM, - GST_DEBUG_CATEGORY_INIT (gst_inferencefilter_debug_category, + GST_DEBUG_CATEGORY_INIT (gst_inference_filter_debug_category, "inferencefilter", 0, "debug category for inferencefilter element")); static void -gst_inferencefilter_class_init (GstInferencefilterClass * klass) +gst_inference_filter_class_init (GstInferenceFilterClass * klass) { GObjectClass *gobject_class = G_OBJECT_CLASS (klass); GstBaseTransformClass *base_transform_class = @@ -114,42 +114,43 @@ gst_inferencefilter_class_init (GstInferencefilterClass * klass) /* Setting up pads and setting metadata should be moved to base_class_init if you intend to subclass this class. */ gst_element_class_add_static_pad_template (GST_ELEMENT_CLASS (klass), - &gst_inferencefilter_src_template); + &gst_inference_filter_src_template); gst_element_class_add_static_pad_template (GST_ELEMENT_CLASS (klass), - &gst_inferencefilter_sink_template); + &gst_inference_filter_sink_template); gst_element_class_set_static_metadata (GST_ELEMENT_CLASS (klass), "Inference Filter", "Generic", "Enables/disables specific classes contained on the inference metadata to be processed", ""); - gobject_class->set_property = gst_inferencefilter_set_property; - gobject_class->get_property = gst_inferencefilter_get_property; + gobject_class->set_property = gst_inference_filter_set_property; + gobject_class->get_property = gst_inference_filter_get_property; g_object_class_install_property (gobject_class, PROP_FILTER_CLASS_LABEL, g_param_spec_int ("filter-class", "filter-class", "Filter class (-1 = disabled)", PROP_FILTER_CLASS_LABEL_MIN, G_MAXINT, - PROP_FILTER_CLASS_LABEL_DEFAULT, GST_INFERENCEFILTER_PROPERTY_FLAGS)); + PROP_FILTER_CLASS_LABEL_DEFAULT, + GST_INFERENCE_FILTER_PROPERTY_FLAGS)); g_object_class_install_property (gobject_class, PROP_RESET_ENABLE, g_param_spec_boolean ("reset-enable", "Reset enable", "Enables all inference meta to be processed", - PROP_RESET_ENABLE_DEFAULT, GST_INFERENCEFILTER_PROPERTY_FLAGS)); + PROP_RESET_ENABLE_DEFAULT, GST_INFERENCE_FILTER_PROPERTY_FLAGS)); base_transform_class->transform_ip = - GST_DEBUG_FUNCPTR (gst_inferencefilter_transform_ip); + GST_DEBUG_FUNCPTR (gst_inference_filter_transform_ip); } static void -gst_inferencefilter_init (GstInferencefilter * inferencefilter) +gst_inference_filter_init (GstInferenceFilter * inferencefilter) { inferencefilter->filter_class = -1; inferencefilter->reset_enable = FALSE; } void -gst_inferencefilter_set_property (GObject * object, guint property_id, +gst_inference_filter_set_property (GObject * object, guint property_id, const GValue * value, GParamSpec * pspec) { - GstInferencefilter *inferencefilter = GST_INFERENCEFILTER (object); + GstInferenceFilter *inferencefilter = GST_INFERENCE_FILTER (object); GST_DEBUG_OBJECT (inferencefilter, "set_property"); @@ -171,10 +172,10 @@ gst_inferencefilter_set_property (GObject * object, guint property_id, } void -gst_inferencefilter_get_property (GObject * object, guint property_id, +gst_inference_filter_get_property (GObject * object, guint property_id, GValue * value, GParamSpec * pspec) { - GstInferencefilter *inferencefilter = GST_INFERENCEFILTER (object); + GstInferenceFilter *inferencefilter = GST_INFERENCE_FILTER (object); GST_DEBUG_OBJECT (inferencefilter, "get_property"); @@ -196,7 +197,7 @@ gst_inferencefilter_get_property (GObject * object, guint property_id, } static void -gst_inferencefilter_filter_enable (GstInferencefilter * inferencefilter, +gst_inference_filter_filter_enable (GstInferenceFilter * inferencefilter, GstInferencePrediction * root, gint class_id, gboolean reset) { GList *iter = NULL; @@ -228,7 +229,7 @@ gst_inferencefilter_filter_enable (GstInferencefilter * inferencefilter, iter_child != NULL; iter_child = g_slist_next (iter_child)) { GstInferencePrediction *predict = (GstInferencePrediction *) iter_child->data; - gst_inferencefilter_filter_enable (inferencefilter, predict, class_id, + gst_inference_filter_filter_enable (inferencefilter, predict, class_id, reset); } @@ -238,9 +239,9 @@ gst_inferencefilter_filter_enable (GstInferencefilter * inferencefilter, } static GstFlowReturn -gst_inferencefilter_transform_ip (GstBaseTransform * trans, GstBuffer * buf) +gst_inference_filter_transform_ip (GstBaseTransform * trans, GstBuffer * buf) { - GstInferencefilter *inferencefilter = GST_INFERENCEFILTER (trans); + GstInferenceFilter *inferencefilter = GST_INFERENCE_FILTER (trans); gboolean reset = FALSE; gint filter = -1; GstInferenceMeta *meta; @@ -267,7 +268,7 @@ gst_inferencefilter_transform_ip (GstBaseTransform * trans, GstBuffer * buf) return GST_FLOW_OK; } - gst_inferencefilter_filter_enable (inferencefilter, meta->prediction, + gst_inference_filter_filter_enable (inferencefilter, meta->prediction, filter, reset); return GST_FLOW_OK; } diff --git a/gst/inferenceutils/gstinferencefilter.h b/gst/inferenceutils/gstinferencefilter.h index 85005487..c8d42bf5 100644 --- a/gst/inferenceutils/gstinferencefilter.h +++ b/gst/inferenceutils/gstinferencefilter.h @@ -19,15 +19,15 @@ * */ -#ifndef _GST_INFERENCEFILTER_H_ -#define _GST_INFERENCEFILTER_H_ +#ifndef _GST_INFERENCE_FILTER_H_ +#define _GST_INFERENCE_FILTER_H_ #include G_BEGIN_DECLS -#define GST_TYPE_INFERENCEFILTER (gst_inferencefilter_get_type()) -G_DECLARE_FINAL_TYPE (GstInferencefilter, gst_inferencefilter, GST, - INFERENCEFILTER, GstBaseTransform) +#define GST_TYPE_INFERENCE_FILTER (gst_inference_filter_get_type()) +G_DECLARE_FINAL_TYPE (GstInferenceFilter, gst_inference_filter, GST, + INFERENCE_FILTER, GstBaseTransform) G_END_DECLS #endif diff --git a/gst/inferenceutils/gstinferenceutils.c b/gst/inferenceutils/gstinferenceutils.c index 9809a2f0..460a0c92 100644 --- a/gst/inferenceutils/gstinferenceutils.c +++ b/gst/inferenceutils/gstinferenceutils.c @@ -48,7 +48,7 @@ plugin_init (GstPlugin * plugin) ret = gst_element_register (plugin, "inferencefilter", GST_RANK_NONE, - GST_TYPE_INFERENCEFILTER); + GST_TYPE_INFERENCE_FILTER); if (!ret) { goto out; } From 665fc1a407e9bb4eb6187db523555068e12bdadd Mon Sep 17 00:00:00 2001 From: tvlenin Date: Tue, 25 Feb 2020 12:31:22 -0600 Subject: [PATCH 05/16] Adapt postprocess to use new meta --- ext/r2inference/gsttinyyolov3.c | 4 ++-- .../gst/r2inference/gstinferencepostprocess.c | 23 +++++++++++-------- .../gst/r2inference/gstinferencepostprocess.h | 2 +- 3 files changed, 17 insertions(+), 12 deletions(-) diff --git a/ext/r2inference/gsttinyyolov3.c b/ext/r2inference/gsttinyyolov3.c index e7ff510d..7e68998f 100644 --- a/ext/r2inference/gsttinyyolov3.c +++ b/ext/r2inference/gsttinyyolov3.c @@ -292,7 +292,7 @@ gst_tinyyolov3_postprocess_old (GstVideoInference * vi, detect_meta->num_boxes = 0; tinyyolov3 = GST_TINYYOLOV3 (vi); - gst_create_boxes (vi, prediction, valid_prediction, + gst_create_boxes_float (vi, prediction, valid_prediction, &detect_meta->boxes, &detect_meta->num_boxes, tinyyolov3->obj_thresh, tinyyolov3->prob_thresh, tinyyolov3->iou_thresh, &probabilities); @@ -325,7 +325,7 @@ gst_tinyyolov3_postprocess_new (GstVideoInference * vi, GST_LOG_OBJECT (tinyyolov3, "Postprocess Meta"); /* Create boxes from prediction data */ - gst_create_boxes (vi, prediction, valid_prediction, + gst_create_boxes_float (vi, prediction, valid_prediction, &boxes, &num_boxes, tinyyolov3->obj_thresh, tinyyolov3->prob_thresh, tinyyolov3->iou_thresh, &probabilities); diff --git a/gst-libs/gst/r2inference/gstinferencepostprocess.c b/gst-libs/gst/r2inference/gstinferencepostprocess.c index 328456cf..24b8e09b 100644 --- a/gst-libs/gst/r2inference/gstinferencepostprocess.c +++ b/gst-libs/gst/r2inference/gstinferencepostprocess.c @@ -26,7 +26,8 @@ #define TOTAL_BOXES_5 845 #define TOTAL_BOXES_15 2535 -#define TOTAL_CLASSES 20 +#define TOTAL_CLASSES_TINYYOLOV2 20 +#define TOTAL_CLASSES_TINYYOLOV3 80 /* Functions declaration*/ @@ -42,7 +43,7 @@ static void gst_get_boxes_from_prediction (gfloat obj_thresh, gint grid_h, gint grid_w, gint boxes_size, gdouble * probabilities); static void gst_get_boxes_from_prediction_float (gfloat obj_thresh, gfloat prob_thresh, gpointer prediction, BBox * boxes, gint * elements, - gint total_boxes); + gint total_boxes, gdouble * probabilities); gboolean gst_fill_classification_meta (GstClassificationMeta * class_meta, @@ -180,7 +181,7 @@ gst_get_boxes_from_prediction (gfloat obj_thresh, gfloat prob_thresh, gint max_class_prob_index; gint counter = 0; gint box_dim = 5; - gint classes = TOTAL_CLASSES; + gint classes = TOTAL_CLASSES_TINYYOLOV2; g_return_if_fail (boxes != NULL); g_return_if_fail (elements != NULL); @@ -245,7 +246,7 @@ gst_create_boxes (GstVideoInference * vi, const gpointer prediction, g_return_val_if_fail (probabilities != NULL, FALSE); *elements = 0; - *probabilities = g_malloc (TOTAL_CLASSES * sizeof (gdouble)); + *probabilities = g_malloc (TOTAL_CLASSES_TINYYOLOV2 * sizeof (gdouble)); gst_get_boxes_from_prediction (obj_thresh, prob_thresh, prediction, boxes, elements, grid_h, grid_w, boxes_size, *probabilities); @@ -331,7 +332,8 @@ gst_create_class_from_prediction (GstVideoInference * vi, static void gst_get_boxes_from_prediction_float (gfloat obj_thresh, gfloat prob_thresh, - gpointer prediction, BBox * boxes, gint * elements, gint total_boxes) + gpointer prediction, BBox * boxes, gint * elements, gint total_boxes, + gdouble * probabilities) { gint i, c; gint index; @@ -341,7 +343,7 @@ gst_get_boxes_from_prediction_float (gfloat obj_thresh, gfloat prob_thresh, gint counter = 0; gint box_class_base; gint box_dim = 5; - gint classes = 80; + gint classes = TOTAL_CLASSES_TINYYOLOV3; gint dimensions_per_box = box_dim + classes; g_return_if_fail (boxes != NULL); @@ -361,6 +363,7 @@ gst_get_boxes_from_prediction_float (gfloat obj_thresh, gfloat prob_thresh, /* Iterate each class probability */ for (c = 0; c < classes; c++) { cur_class_prob = ((gfloat *) prediction)[box_class_base + c]; + probabilities[c] = cur_class_prob; if (cur_class_prob > max_class_prob) { max_class_prob = cur_class_prob; max_class_prob_index = c; @@ -387,11 +390,10 @@ gboolean gst_create_boxes_float (GstVideoInference * vi, const gpointer prediction, gboolean * valid_prediction, BBox ** resulting_boxes, gint * elements, gdouble obj_thresh, gdouble prob_thresh, - gdouble iou_thresh) + gdouble iou_thresh, gdouble ** probabilities) { gint total_boxes = 2535; BBox boxes[TOTAL_BOXES_15]; - *elements = 0; g_return_val_if_fail (vi != NULL, FALSE); g_return_val_if_fail (prediction != NULL, FALSE); @@ -399,8 +401,11 @@ gst_create_boxes_float (GstVideoInference * vi, const gpointer prediction, g_return_val_if_fail (resulting_boxes != NULL, FALSE); g_return_val_if_fail (elements != NULL, FALSE); + *elements = 0; + *probabilities = g_malloc (TOTAL_CLASSES_TINYYOLOV3 * sizeof (gdouble)); + gst_get_boxes_from_prediction_float (obj_thresh, prob_thresh, prediction, - boxes, elements, total_boxes); + boxes, elements, total_boxes, *probabilities); gst_remove_duplicated_boxes (iou_thresh, boxes, elements); *resulting_boxes = g_malloc (*elements * sizeof (BBox)); diff --git a/gst-libs/gst/r2inference/gstinferencepostprocess.h b/gst-libs/gst/r2inference/gstinferencepostprocess.h index 779f8286..0589820c 100644 --- a/gst-libs/gst/r2inference/gstinferencepostprocess.h +++ b/gst-libs/gst/r2inference/gstinferencepostprocess.h @@ -67,7 +67,7 @@ gboolean gst_create_boxes (GstVideoInference * vi, const gpointer prediction, gboolean gst_create_boxes_float (GstVideoInference * vi, const gpointer prediction, gboolean * valid_prediction, BBox ** resulting_boxes, gint * elements, gdouble obj_thresh, - gdouble prob_thresh, gdouble iou_thresh); + gdouble prob_thresh, gdouble iou_thresh, gdouble ** probabilities); /** * \brief Create Prediction from box From f2592db9edf46bf45eb5f300595dae73cbc47eec Mon Sep 17 00:00:00 2001 From: tvlenin Date: Tue, 25 Feb 2020 13:35:31 -0600 Subject: [PATCH 06/16] Add pointer guards --- gst-libs/gst/r2inference/gstinferencepostprocess.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gst-libs/gst/r2inference/gstinferencepostprocess.c b/gst-libs/gst/r2inference/gstinferencepostprocess.c index 24b8e09b..89dd541b 100644 --- a/gst-libs/gst/r2inference/gstinferencepostprocess.c +++ b/gst-libs/gst/r2inference/gstinferencepostprocess.c @@ -185,6 +185,7 @@ gst_get_boxes_from_prediction (gfloat obj_thresh, gfloat prob_thresh, g_return_if_fail (boxes != NULL); g_return_if_fail (elements != NULL); + g_return_if_fail (probabilities != NULL); /* Iterate rows */ for (i = 0; i < grid_h; i++) { @@ -348,6 +349,7 @@ gst_get_boxes_from_prediction_float (gfloat obj_thresh, gfloat prob_thresh, g_return_if_fail (boxes != NULL); g_return_if_fail (elements != NULL); + g_return_if_fail (probabilities != NULL); /* Iterate boxes */ for (i = 0; i < total_boxes; i++) { From be57bbecf1b5dbf5ca5fbf643da16d125775e15b Mon Sep 17 00:00:00 2001 From: tvlenin Date: Tue, 25 Feb 2020 14:01:58 -0600 Subject: [PATCH 07/16] Refactor number of classes to each detection model --- ext/r2inference/gsttinyyolov2.c | 7 ++-- ext/r2inference/gsttinyyolov3.c | 8 +++-- .../gst/r2inference/gstinferencepostprocess.c | 34 ++++++++----------- .../gst/r2inference/gstinferencepostprocess.h | 6 ++-- 4 files changed, 30 insertions(+), 25 deletions(-) diff --git a/ext/r2inference/gsttinyyolov2.c b/ext/r2inference/gsttinyyolov2.c index 33aace84..7edba1ce 100644 --- a/ext/r2inference/gsttinyyolov2.c +++ b/ext/r2inference/gsttinyyolov2.c @@ -66,6 +66,7 @@ GST_DEBUG_CATEGORY_STATIC (gst_tinyyolov2_debug_category); #define MIN_IOU_THRESH 0 #define DEFAULT_IOU_THRESH 0.30 +#define TOTAL_CLASSES 20 const gfloat box_anchors[] = { 1.08, 1.19, 3.42, 4.41, 6.63, 11.38, 9.42, 5.11, 16.62, 10.52 }; @@ -328,7 +329,8 @@ gst_tinyyolov2_postprocess_old (GstVideoInference * vi, gst_create_boxes (vi, prediction, valid_prediction, &detect_meta->boxes, &detect_meta->num_boxes, tinyyolov2->obj_thresh, - tinyyolov2->prob_thresh, tinyyolov2->iou_thresh, &probabilities); + tinyyolov2->prob_thresh, tinyyolov2->iou_thresh, &probabilities, + TOTAL_CLASSES); gst_inference_print_boxes (vi, gst_tinyyolov2_debug_category, detect_meta); @@ -361,7 +363,8 @@ gst_tinyyolov2_postprocess_new (GstVideoInference * vi, /* Create boxes from prediction data */ gst_create_boxes (vi, prediction, valid_prediction, &boxes, &num_boxes, tinyyolov2->obj_thresh, - tinyyolov2->prob_thresh, tinyyolov2->iou_thresh, &probabilities); + tinyyolov2->prob_thresh, tinyyolov2->iou_thresh, &probabilities, + TOTAL_CLASSES); GST_LOG_OBJECT (tinyyolov2, "Number of predictions: %d", num_boxes); diff --git a/ext/r2inference/gsttinyyolov3.c b/ext/r2inference/gsttinyyolov3.c index 7e68998f..c2e8bbf3 100644 --- a/ext/r2inference/gsttinyyolov3.c +++ b/ext/r2inference/gsttinyyolov3.c @@ -63,6 +63,8 @@ GST_DEBUG_CATEGORY_STATIC (gst_tinyyolov3_debug_category); #define MIN_IOU_THRESH 0 #define DEFAULT_IOU_THRESH 0.40 +#define TOTAL_CLASSES 80 + /* prototypes */ static void gst_tinyyolov3_set_property (GObject * object, guint property_id, const GValue * value, GParamSpec * pspec); @@ -294,7 +296,8 @@ gst_tinyyolov3_postprocess_old (GstVideoInference * vi, gst_create_boxes_float (vi, prediction, valid_prediction, &detect_meta->boxes, &detect_meta->num_boxes, tinyyolov3->obj_thresh, - tinyyolov3->prob_thresh, tinyyolov3->iou_thresh, &probabilities); + tinyyolov3->prob_thresh, tinyyolov3->iou_thresh, &probabilities, + TOTAL_CLASSES); gst_inference_print_boxes (vi, gst_tinyyolov3_debug_category, detect_meta); @@ -327,7 +330,8 @@ gst_tinyyolov3_postprocess_new (GstVideoInference * vi, /* Create boxes from prediction data */ gst_create_boxes_float (vi, prediction, valid_prediction, &boxes, &num_boxes, tinyyolov3->obj_thresh, - tinyyolov3->prob_thresh, tinyyolov3->iou_thresh, &probabilities); + tinyyolov3->prob_thresh, tinyyolov3->iou_thresh, &probabilities, + TOTAL_CLASSES); GST_LOG_OBJECT (tinyyolov3, "Number of predictions: %d", num_boxes); diff --git a/gst-libs/gst/r2inference/gstinferencepostprocess.c b/gst-libs/gst/r2inference/gstinferencepostprocess.c index 89dd541b..dcd129bb 100644 --- a/gst-libs/gst/r2inference/gstinferencepostprocess.c +++ b/gst-libs/gst/r2inference/gstinferencepostprocess.c @@ -26,9 +26,6 @@ #define TOTAL_BOXES_5 845 #define TOTAL_BOXES_15 2535 -#define TOTAL_CLASSES_TINYYOLOV2 20 -#define TOTAL_CLASSES_TINYYOLOV3 80 - /* Functions declaration*/ static gdouble gst_intersection_over_union (BBox box_1, BBox box_2); @@ -40,10 +37,11 @@ static void gst_box_to_pixels (BBox * normalized_box, gint row, gint col, static gdouble gst_sigmoid (gdouble x); static void gst_get_boxes_from_prediction (gfloat obj_thresh, gfloat prob_thresh, gpointer prediction, BBox * boxes, gint * elements, - gint grid_h, gint grid_w, gint boxes_size, gdouble * probabilities); + gint grid_h, gint grid_w, gint boxes_size, gdouble * probabilities, + gint num_classes); static void gst_get_boxes_from_prediction_float (gfloat obj_thresh, gfloat prob_thresh, gpointer prediction, BBox * boxes, gint * elements, - gint total_boxes, gdouble * probabilities); + gint total_boxes, gdouble * probabilities, gint num_classes); gboolean gst_fill_classification_meta (GstClassificationMeta * class_meta, @@ -172,7 +170,7 @@ gst_box_to_pixels (BBox * normalized_box, gint row, gint col, gint box) static void gst_get_boxes_from_prediction (gfloat obj_thresh, gfloat prob_thresh, gpointer prediction, BBox * boxes, gint * elements, gint grid_h, - gint grid_w, gint boxes_size, gdouble * probabilities) + gint grid_w, gint boxes_size, gdouble * probabilities, gint num_classes) { gint i, j, c, b; gint index; @@ -181,7 +179,6 @@ gst_get_boxes_from_prediction (gfloat obj_thresh, gfloat prob_thresh, gint max_class_prob_index; gint counter = 0; gint box_dim = 5; - gint classes = TOTAL_CLASSES_TINYYOLOV2; g_return_if_fail (boxes != NULL); g_return_if_fail (elements != NULL); @@ -193,13 +190,13 @@ gst_get_boxes_from_prediction (gfloat obj_thresh, gfloat prob_thresh, for (j = 0; j < grid_w; j++) { /* Iterate boxes */ for (b = 0; b < boxes_size; b++) { - index = ((i * grid_w + j) * boxes_size + b) * (box_dim + classes); + index = ((i * grid_w + j) * boxes_size + b) * (box_dim + num_classes); obj_prob = ((gfloat *) prediction)[index + 4]; /* If the Objectness score is over the threshold add it to the boxes list */ if (obj_prob > obj_thresh) { max_class_prob = 0; max_class_prob_index = 0; - for (c = 0; c < classes; c++) { + for (c = 0; c < num_classes; c++) { cur_class_prob = ((gfloat *) prediction)[index + box_dim + c]; probabilities[c] = cur_class_prob; if (cur_class_prob > max_class_prob) { @@ -232,7 +229,7 @@ gboolean gst_create_boxes (GstVideoInference * vi, const gpointer prediction, gboolean * valid_prediction, BBox ** resulting_boxes, gint * elements, gfloat obj_thresh, gfloat prob_thresh, gfloat iou_thresh, - gdouble ** probabilities) + gdouble ** probabilities, gint num_classes) { gint grid_h = 13; gint grid_w = 13; @@ -247,10 +244,10 @@ gst_create_boxes (GstVideoInference * vi, const gpointer prediction, g_return_val_if_fail (probabilities != NULL, FALSE); *elements = 0; - *probabilities = g_malloc (TOTAL_CLASSES_TINYYOLOV2 * sizeof (gdouble)); + *probabilities = g_malloc (num_classes * sizeof (gdouble)); gst_get_boxes_from_prediction (obj_thresh, prob_thresh, prediction, boxes, - elements, grid_h, grid_w, boxes_size, *probabilities); + elements, grid_h, grid_w, boxes_size, *probabilities, num_classes); gst_remove_duplicated_boxes (iou_thresh, boxes, elements); *resulting_boxes = g_malloc (*elements * sizeof (BBox)); @@ -334,7 +331,7 @@ gst_create_class_from_prediction (GstVideoInference * vi, static void gst_get_boxes_from_prediction_float (gfloat obj_thresh, gfloat prob_thresh, gpointer prediction, BBox * boxes, gint * elements, gint total_boxes, - gdouble * probabilities) + gdouble * probabilities, gint num_classes) { gint i, c; gint index; @@ -344,8 +341,7 @@ gst_get_boxes_from_prediction_float (gfloat obj_thresh, gfloat prob_thresh, gint counter = 0; gint box_class_base; gint box_dim = 5; - gint classes = TOTAL_CLASSES_TINYYOLOV3; - gint dimensions_per_box = box_dim + classes; + gint dimensions_per_box = box_dim + num_classes; g_return_if_fail (boxes != NULL); g_return_if_fail (elements != NULL); @@ -363,7 +359,7 @@ gst_get_boxes_from_prediction_float (gfloat obj_thresh, gfloat prob_thresh, box_class_base = index + box_dim; /* Iterate each class probability */ - for (c = 0; c < classes; c++) { + for (c = 0; c < num_classes; c++) { cur_class_prob = ((gfloat *) prediction)[box_class_base + c]; probabilities[c] = cur_class_prob; if (cur_class_prob > max_class_prob) { @@ -392,7 +388,7 @@ gboolean gst_create_boxes_float (GstVideoInference * vi, const gpointer prediction, gboolean * valid_prediction, BBox ** resulting_boxes, gint * elements, gdouble obj_thresh, gdouble prob_thresh, - gdouble iou_thresh, gdouble ** probabilities) + gdouble iou_thresh, gdouble ** probabilities, gint num_classes) { gint total_boxes = 2535; BBox boxes[TOTAL_BOXES_15]; @@ -404,10 +400,10 @@ gst_create_boxes_float (GstVideoInference * vi, const gpointer prediction, g_return_val_if_fail (elements != NULL, FALSE); *elements = 0; - *probabilities = g_malloc (TOTAL_CLASSES_TINYYOLOV3 * sizeof (gdouble)); + *probabilities = g_malloc (num_classes * sizeof (gdouble)); gst_get_boxes_from_prediction_float (obj_thresh, prob_thresh, prediction, - boxes, elements, total_boxes, *probabilities); + boxes, elements, total_boxes, *probabilities, num_classes); gst_remove_duplicated_boxes (iou_thresh, boxes, elements); *resulting_boxes = g_malloc (*elements * sizeof (BBox)); diff --git a/gst-libs/gst/r2inference/gstinferencepostprocess.h b/gst-libs/gst/r2inference/gstinferencepostprocess.h index 0589820c..1dbcef11 100644 --- a/gst-libs/gst/r2inference/gstinferencepostprocess.h +++ b/gst-libs/gst/r2inference/gstinferencepostprocess.h @@ -50,7 +50,8 @@ G_BEGIN_DECLS */ gboolean gst_create_boxes (GstVideoInference * vi, const gpointer prediction, gboolean * valid_prediction, BBox ** resulting_boxes, - gint * elements, gfloat obj_thresh, gfloat prob_thresh, gfloat iou_thresh, gdouble ** probabilities); + gint * elements, gfloat obj_thresh, gfloat prob_thresh, gfloat iou_thresh, + gdouble ** probabilities, gint num_classes); /** * \brief Fill all the detection meta with the boxes @@ -67,7 +68,8 @@ gboolean gst_create_boxes (GstVideoInference * vi, const gpointer prediction, gboolean gst_create_boxes_float (GstVideoInference * vi, const gpointer prediction, gboolean * valid_prediction, BBox ** resulting_boxes, gint * elements, gdouble obj_thresh, - gdouble prob_thresh, gdouble iou_thresh, gdouble ** probabilities); + gdouble prob_thresh, gdouble iou_thresh, gdouble ** probabilities, + gint num_classes); /** * \brief Create Prediction from box From f176a7d18238881648747966836d24ed13045988 Mon Sep 17 00:00:00 2001 From: tvlenin Date: Wed, 26 Feb 2020 10:14:06 -0600 Subject: [PATCH 08/16] Add documentation of new parameters --- gst-libs/gst/r2inference/gstinferencepostprocess.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/gst-libs/gst/r2inference/gstinferencepostprocess.h b/gst-libs/gst/r2inference/gstinferencepostprocess.h index 1dbcef11..228d6fad 100644 --- a/gst-libs/gst/r2inference/gstinferencepostprocess.h +++ b/gst-libs/gst/r2inference/gstinferencepostprocess.h @@ -47,6 +47,8 @@ G_BEGIN_DECLS * \param obj_thresh Objectness threshold * \param prob_thresh Class probability threshold * \param iou_thresh Intersection over union threshold + * \param probabilities Probabilities of each classes + * \param num_classes The number of classes */ gboolean gst_create_boxes (GstVideoInference * vi, const gpointer prediction, gboolean * valid_prediction, BBox ** resulting_boxes, @@ -64,6 +66,8 @@ gboolean gst_create_boxes (GstVideoInference * vi, const gpointer prediction, * \param obj_thresh Objectness threshold * \param prob_thresh Class probability threshold * \param iou_thresh Intersection over union threshold + * \param probabilities Probabilities of each classes + * \param num_classes The number of classes */ gboolean gst_create_boxes_float (GstVideoInference * vi, const gpointer prediction, gboolean * valid_prediction, @@ -78,6 +82,7 @@ gboolean gst_create_boxes_float (GstVideoInference * vi, * \param box Box used to fill Prediction * \param labels_list List with all possible lables * \param num_labels The number of posibble labels + * \param probabilities Probabilities of each classes */ GstInferencePrediction *gst_create_prediction_from_box (GstVideoInference * vi, BBox * box, gchar **labels_list, gint num_labels, const gdouble * probabilities); From cd53a27d620345957aa47773af7f41ca86916788 Mon Sep 17 00:00:00 2001 From: tvlenin Date: Mon, 24 Feb 2020 16:57:29 -0600 Subject: [PATCH 09/16] Include root prediction to the overlay --- ext/opencv/gstinferenceoverlay.cc | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/ext/opencv/gstinferenceoverlay.cc b/ext/opencv/gstinferenceoverlay.cc index 81e5cd2d..f055cbee 100644 --- a/ext/opencv/gstinferenceoverlay.cc +++ b/ext/opencv/gstinferenceoverlay.cc @@ -206,6 +206,8 @@ gst_get_meta (GstInferencePrediction * pred, cv::Mat & cv_mat, gdouble alpha = 0.5; cv::Mat alpha_overlay; gint width, height, x, y = 0; + cv::Size text = cv::Size(0,0); + cv::Size tmp_text_size = cv::Size(0,0); g_return_if_fail (pred != NULL); @@ -227,6 +229,10 @@ gst_get_meta (GstInferencePrediction * pred, cv::Mat & cv_mat, box = pred->bbox; + if (TRUE == G_NODE_IS_ROOT (pred->predictions)) { + box.width = 0; + } + for (iter = pred->classifications; iter != NULL; iter = g_list_next (iter)) { GstInferenceClassification *classification = (GstInferenceClassification *) iter->data; @@ -245,10 +251,12 @@ gst_get_meta (GstInferencePrediction * pred, cv::Mat & cv_mat, cv::putText (cv_mat, label, cv::Point (box.x + box.width, box.y + classes * OVERLAY_WIDTH), cv::FONT_HERSHEY_PLAIN, font_scale, cv::Scalar::all (0), thickness); - } - - cv::Size text = cv::getTextSize (label, cv::FONT_HERSHEY_PLAIN, font_scale, + tmp_text_size = cv::getTextSize (label, cv::FONT_HERSHEY_PLAIN, font_scale, thickness, 0); + if(tmp_text_size.width > text.width){ + text = tmp_text_size; + } + } if ((box.x + box.width) < 0) { x = 0; From a1e60c783290f006206f89160e57b9b039a85ad4 Mon Sep 17 00:00:00 2001 From: tvlenin Date: Mon, 24 Feb 2020 17:14:36 -0600 Subject: [PATCH 10/16] Change classification overlay position --- ext/opencv/gstinferenceoverlay.cc | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/ext/opencv/gstinferenceoverlay.cc b/ext/opencv/gstinferenceoverlay.cc index f055cbee..a81d3c74 100644 --- a/ext/opencv/gstinferenceoverlay.cc +++ b/ext/opencv/gstinferenceoverlay.cc @@ -54,6 +54,8 @@ static const cv::Scalar colors[] = { #define CHOSEN_COLOR 14 #define OVERLAY_HEIGHT 50 #define OVERLAY_WIDTH 30 +#define OVERLAY_X_POSITION 0 +#define ALPHA_OVERLAY 0.98 #define LINES_GAP 20 GST_DEBUG_CATEGORY_STATIC (gst_inference_overlay_debug_category); @@ -203,7 +205,7 @@ gst_get_meta (GstInferencePrediction * pred, cv::Mat & cv_mat, cv::String prob; BoundingBox box; gint classes = 0; - gdouble alpha = 0.5; + gdouble alpha = ALPHA_OVERLAY; cv::Mat alpha_overlay; gint width, height, x, y = 0; cv::Size text = cv::Size(0,0); @@ -248,7 +250,7 @@ gst_get_meta (GstInferencePrediction * pred, cv::Mat & cv_mat, label = cv::format ("Label #%d : %0.3f", classification->class_id, classification->class_prob); } - cv::putText (cv_mat, label, cv::Point (box.x + box.width, + cv::putText (cv_mat, label, cv::Point (box.x + OVERLAY_X_POSITION, box.y + classes * OVERLAY_WIDTH), cv::FONT_HERSHEY_PLAIN, font_scale, cv::Scalar::all (0), thickness); tmp_text_size = cv::getTextSize (label, cv::FONT_HERSHEY_PLAIN, font_scale, @@ -258,15 +260,15 @@ gst_get_meta (GstInferencePrediction * pred, cv::Mat & cv_mat, } } - if ((box.x + box.width) < 0) { + if ((box.x + OVERLAY_X_POSITION) < 0) { x = 0; - } else if ((int) (box.x + box.width) >= cv_mat.cols) { + } else if ((int) (box.x + OVERLAY_X_POSITION) >= cv_mat.cols) { x = cv_mat.cols - 1; } else { - x = box.x + box.width; + x = box.x + OVERLAY_X_POSITION; } - if ((int) (x + box.width + text.width) >= cv_mat.cols) { + if ((int) (x + OVERLAY_X_POSITION + text.width) >= cv_mat.cols) { width = cv_mat.cols - x - 1; } else { width = text.width; From 5686e1aad7afb9805dd9a2b2c081f7d2a6fb0783 Mon Sep 17 00:00:00 2001 From: tvlenin Date: Wed, 26 Feb 2020 11:33:15 -0600 Subject: [PATCH 11/16] Add property to set the overlay transparency --- ext/opencv/gstclassificationoverlay.cc | 4 ++-- ext/opencv/gstdetectionoverlay.cc | 4 ++-- ext/opencv/gstembeddingoverlay.cc | 4 ++-- ext/opencv/gstinferenceoverlay.cc | 14 +++++------- .../gst/opencv/gstinferencebaseoverlay.cc | 22 +++++++++++++++++-- gst-libs/gst/opencv/gstinferencebaseoverlay.h | 3 ++- 6 files changed, 34 insertions(+), 17 deletions(-) diff --git a/ext/opencv/gstclassificationoverlay.cc b/ext/opencv/gstclassificationoverlay.cc index 8b54c2f6..234ff555 100644 --- a/ext/opencv/gstclassificationoverlay.cc +++ b/ext/opencv/gstclassificationoverlay.cc @@ -39,7 +39,7 @@ static GstFlowReturn gst_classification_overlay_process_meta (GstInferenceBaseOverlay * inference_overlay, cv::Mat & cv_mat, GstVideoFrame * frame, GstMeta * meta, gdouble font_scale, gint thickness, gchar ** labels_list, gint num_labels, - LineStyleBoundingBox style); + LineStyleBoundingBox style, gdouble alpha_overlay); enum { @@ -95,7 +95,7 @@ static GstFlowReturn gst_classification_overlay_process_meta (GstInferenceBaseOverlay * inference_overlay, cv::Mat & cv_mat, GstVideoFrame * frame, GstMeta * meta, gdouble font_scale, gint thickness, gchar ** labels_list, gint num_labels, - LineStyleBoundingBox style) + LineStyleBoundingBox style, gdouble alpha_overlay) { GstClassificationMeta *class_meta; gint index, i; diff --git a/ext/opencv/gstdetectionoverlay.cc b/ext/opencv/gstdetectionoverlay.cc index 1ee31c5b..97920b64 100644 --- a/ext/opencv/gstdetectionoverlay.cc +++ b/ext/opencv/gstdetectionoverlay.cc @@ -50,7 +50,7 @@ GST_DEBUG_CATEGORY_STATIC (gst_detection_overlay_debug_category); static GstFlowReturn gst_detection_overlay_process_meta (GstInferenceBaseOverlay * inference_overlay, cv::Mat & cv_mat, GstVideoFrame * frame, GstMeta * meta, gdouble font_scale, gint thickness, gchar ** labels_list, - gint num_labels, LineStyleBoundingBox style); + gint num_labels, LineStyleBoundingBox style, gdouble alpha_overlay); enum { @@ -104,7 +104,7 @@ static GstFlowReturn gst_detection_overlay_process_meta (GstInferenceBaseOverlay * inference_overlay, cv::Mat & cv_mat, GstVideoFrame * frame, GstMeta * meta, gdouble font_scale, gint thickness, gchar ** labels_list, gint num_labels, - LineStyleBoundingBox style) + LineStyleBoundingBox style, gdouble alpha_overlay) { GstDetectionMeta *detect_meta; gint i; diff --git a/ext/opencv/gstembeddingoverlay.cc b/ext/opencv/gstembeddingoverlay.cc index 5d7638c9..ab5c1291 100644 --- a/ext/opencv/gstembeddingoverlay.cc +++ b/ext/opencv/gstembeddingoverlay.cc @@ -51,7 +51,7 @@ static void gst_embedding_overlay_finalize (GObject * object); static GstFlowReturn gst_embedding_overlay_process_meta (GstInferenceBaseOverlay * inference_overlay, cv::Mat & cv_mat, GstVideoFrame * frame, GstMeta * meta, gdouble font_scale, gint thickness, gchar ** labels_list, - gint num_labels, LineStyleBoundingBox style); + gint num_labels, LineStyleBoundingBox style, gdouble alpha_overlay); static gboolean gst_embedding_overlay_set_embeddings (GstEmbeddingOverlay * embedding_overlay, const GValue * value); @@ -209,7 +209,7 @@ static GstFlowReturn gst_embedding_overlay_process_meta (GstInferenceBaseOverlay * inference_overlay, cv::Mat & cv_mat, GstVideoFrame * frame, GstMeta * meta, gdouble font_scale, gint thickness, gchar ** labels_list, gint num_labels, - LineStyleBoundingBox style) + LineStyleBoundingBox, gdouble alpha_overlay ) { GstEmbeddingOverlay *embedding_overlay = GST_EMBEDDING_OVERLAY (inference_overlay); diff --git a/ext/opencv/gstinferenceoverlay.cc b/ext/opencv/gstinferenceoverlay.cc index a81d3c74..db658ed6 100644 --- a/ext/opencv/gstinferenceoverlay.cc +++ b/ext/opencv/gstinferenceoverlay.cc @@ -55,7 +55,6 @@ static const cv::Scalar colors[] = { #define OVERLAY_HEIGHT 50 #define OVERLAY_WIDTH 30 #define OVERLAY_X_POSITION 0 -#define ALPHA_OVERLAY 0.98 #define LINES_GAP 20 GST_DEBUG_CATEGORY_STATIC (gst_inference_overlay_debug_category); @@ -69,7 +68,7 @@ static void gst_inference_overlay_get_property (GObject * object, static GstFlowReturn gst_inference_overlay_process_meta (GstInferenceBaseOverlay * inference_overlay, cv::Mat & cv_mat, GstVideoFrame * frame, GstMeta * meta, gdouble font_scale, gint thickness, gchar ** labels_list, - gint num_labels, LineStyleBoundingBox style); + gint num_labels, LineStyleBoundingBox style, gdouble alpha_overlay); static void draw_line (cv::Mat & img, cv::Point pt1, cv::Point pt2, cv::Scalar color, gint thickness, LineStyleBoundingBox style, gint gap); @@ -195,7 +194,7 @@ draw_line (cv::Mat & img, cv::Point pt1, cv::Point pt2, cv::Scalar color, static void gst_get_meta (GstInferencePrediction * pred, cv::Mat & cv_mat, gdouble font_scale, gint thickness, gchar ** labels_list, gint num_labels, - LineStyleBoundingBox style) + LineStyleBoundingBox style, gdouble alpha_overlay) { cv::Size size; cv::String label; @@ -205,8 +204,7 @@ gst_get_meta (GstInferencePrediction * pred, cv::Mat & cv_mat, cv::String prob; BoundingBox box; gint classes = 0; - gdouble alpha = ALPHA_OVERLAY; - cv::Mat alpha_overlay; + gdouble alpha = alpha_overlay; gint width, height, x, y = 0; cv::Size text = cv::Size(0,0); cv::Size tmp_text_size = cv::Size(0,0); @@ -221,7 +219,7 @@ gst_get_meta (GstInferencePrediction * pred, cv::Mat & cv_mat, (GstInferencePrediction *) tree_iter->data; gst_get_meta (predict, cv_mat, font_scale, thickness, - labels_list, num_labels, style); + labels_list, num_labels, style, alpha_overlay); } if (!pred->enabled) { @@ -325,7 +323,7 @@ static GstFlowReturn gst_inference_overlay_process_meta (GstInferenceBaseOverlay * inference_overlay, cv::Mat & cv_mat, GstVideoFrame * frame, GstMeta * meta, gdouble font_scale, gint thickness, gchar ** labels_list, gint num_labels, - LineStyleBoundingBox style) + LineStyleBoundingBox style, gdouble alpha_overlay) { GstInferenceMeta *detect_meta; @@ -336,7 +334,7 @@ gst_inference_overlay_process_meta (GstInferenceBaseOverlay * inference_overlay, detect_meta = (GstInferenceMeta *) meta; gst_get_meta (detect_meta->prediction, cv_mat, font_scale, thickness, - labels_list, num_labels, style); + labels_list, num_labels, style, alpha_overlay); return GST_FLOW_OK; } diff --git a/gst-libs/gst/opencv/gstinferencebaseoverlay.cc b/gst-libs/gst/opencv/gstinferencebaseoverlay.cc index 3bde2118..82697811 100644 --- a/gst-libs/gst/opencv/gstinferencebaseoverlay.cc +++ b/gst-libs/gst/opencv/gstinferencebaseoverlay.cc @@ -45,13 +45,18 @@ GST_DEBUG_CATEGORY_STATIC (gst_inference_base_overlay_debug_category); #define DEFAULT_STYLE CLASSIC #define MAX_STYLE DASHED +#define DEFAULT_ALPHA_OVERLAY 1.0 +#define MIN_ALPHA_OVERLAY 0 +#define MAX_ALPHA_OVERLAY 1.0 + enum { PROP_0, PROP_FONT_SCALE, PROP_THICKNESS, PROP_LABELS, - PROP_STYLE + PROP_STYLE, + PROP_ALPHA_OVERLAY }; GType @@ -79,6 +84,7 @@ struct _GstInferenceBaseOverlayPrivate gchar **labels_list; gint num_labels; LineStyleBoundingBox style; + gdouble alpha_overlay; }; /* prototypes */ static void gst_inference_base_overlay_set_property (GObject * object, @@ -142,6 +148,9 @@ gst_inference_base_overlay_class_init (GstInferenceBaseOverlayClass * klass) g_param_spec_enum ("style", "style", "Line style to draw the bounding box", LINE_STYLE_BOUNDING_BOX, DEFAULT_STYLE, G_PARAM_READWRITE)); + g_object_class_install_property (gobject_class, PROP_ALPHA_OVERLAY, + g_param_spec_double ("alpha_overlay", "alpha", "Overlay transparency", MIN_ALPHA_OVERLAY, + MAX_ALPHA_OVERLAY, DEFAULT_ALPHA_OVERLAY, G_PARAM_READWRITE)); base_transform_class->start = GST_DEBUG_FUNCPTR (gst_inference_base_overlay_start); @@ -163,6 +172,7 @@ gst_inference_base_overlay_init (GstInferenceBaseOverlay * inference_overlay) priv->labels_list = DEFAULT_LABELS; priv->num_labels = DEFAULT_NUM_LABELS; priv->style = DEFAULT_STYLE; + priv->alpha_overlay = DEFAULT_ALPHA_OVERLAY; } void @@ -205,6 +215,11 @@ gst_inference_base_overlay_set_property (GObject * object, guint property_id, GST_DEBUG_OBJECT (inference_overlay, "Changed box style to %d", priv->style); break; + case PROP_ALPHA_OVERLAY: + priv->alpha_overlay = g_value_get_double (value); + GST_DEBUG_OBJECT (inference_overlay, "Changed overlay transparency to %lf", + priv->alpha_overlay); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); break; @@ -235,6 +250,9 @@ gst_inference_base_overlay_get_property (GObject * object, guint property_id, case PROP_STYLE: g_value_set_enum (value, priv->style); break; + case PROP_ALPHA_OVERLAY: + g_value_set_double (value, priv->alpha_overlay); + break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID (object, property_id, pspec); break; @@ -351,7 +369,7 @@ gst_inference_base_overlay_transform_frame_ip (GstVideoFilter * trans, ret = io_class->process_meta (inference_overlay, cv_mat, frame, meta, priv->font_scale, priv->thickness, priv->labels_list, priv->num_labels, - priv->style); + priv->style, priv->alpha_overlay); out: return ret; diff --git a/gst-libs/gst/opencv/gstinferencebaseoverlay.h b/gst-libs/gst/opencv/gstinferencebaseoverlay.h index 32a6effe..2df4e7e0 100644 --- a/gst-libs/gst/opencv/gstinferencebaseoverlay.h +++ b/gst-libs/gst/opencv/gstinferencebaseoverlay.h @@ -59,7 +59,8 @@ struct _GstInferenceBaseOverlayClass GstFlowReturn (* process_meta) (GstInferenceBaseOverlay * inference_base_overlay, cv::Mat &mat, GstVideoFrame * frame, GstMeta* meta, gdouble font_scale, - gint thickness, gchar **labels_list, gint num_labels, LineStyleBoundingBox style); + gint thickness, gchar **labels_list, gint num_labels, LineStyleBoundingBox style, + gdouble alpha_overlay); GType meta_type; }; From 71824480fbdbb85b3919622b7f86993439b3f3c9 Mon Sep 17 00:00:00 2001 From: tvlenin Date: Thu, 27 Feb 2020 15:38:11 -0600 Subject: [PATCH 12/16] Refactor variable names --- ext/opencv/gstinferenceoverlay.cc | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/ext/opencv/gstinferenceoverlay.cc b/ext/opencv/gstinferenceoverlay.cc index db658ed6..ec73f7c0 100644 --- a/ext/opencv/gstinferenceoverlay.cc +++ b/ext/opencv/gstinferenceoverlay.cc @@ -206,8 +206,8 @@ gst_get_meta (GstInferencePrediction * pred, cv::Mat & cv_mat, gint classes = 0; gdouble alpha = alpha_overlay; gint width, height, x, y = 0; - cv::Size text = cv::Size(0,0); - cv::Size tmp_text_size = cv::Size(0,0); + cv::Size max_size = cv::Size(0,0); + cv::Size current_size = cv::Size(0,0); g_return_if_fail (pred != NULL); @@ -251,10 +251,10 @@ gst_get_meta (GstInferencePrediction * pred, cv::Mat & cv_mat, cv::putText (cv_mat, label, cv::Point (box.x + OVERLAY_X_POSITION, box.y + classes * OVERLAY_WIDTH), cv::FONT_HERSHEY_PLAIN, font_scale, cv::Scalar::all (0), thickness); - tmp_text_size = cv::getTextSize (label, cv::FONT_HERSHEY_PLAIN, font_scale, + current_size = cv::getTextSize (label, cv::FONT_HERSHEY_PLAIN, font_scale, thickness, 0); - if(tmp_text_size.width > text.width){ - text = tmp_text_size; + if(current_size.width > max_size.width){ + max_size = current_size; } } @@ -266,10 +266,10 @@ gst_get_meta (GstInferencePrediction * pred, cv::Mat & cv_mat, x = box.x + OVERLAY_X_POSITION; } - if ((int) (x + OVERLAY_X_POSITION + text.width) >= cv_mat.cols) { + if ((int) (x + OVERLAY_X_POSITION + max_size.width) >= cv_mat.cols) { width = cv_mat.cols - x - 1; } else { - width = text.width; + width = max_size.width; } if ((int) (box.y + OVERLAY_HEIGHT * classes) >= cv_mat.rows) { From 47b8cde301641faa96dfe37083c95d5be7af3bc4 Mon Sep 17 00:00:00 2001 From: tvlenin Date: Fri, 28 Feb 2020 09:41:33 -0600 Subject: [PATCH 13/16] Add division on predsize --- .../check/process/test_gst_fill_classification_meta_function.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/check/process/test_gst_fill_classification_meta_function.c b/tests/check/process/test_gst_fill_classification_meta_function.c index 8c221803..66377177 100644 --- a/tests/check/process/test_gst_fill_classification_meta_function.c +++ b/tests/check/process/test_gst_fill_classification_meta_function.c @@ -36,7 +36,7 @@ GST_START_TEST (test_gst_fill_classification_meta) (GstClassificationMeta *) gst_classification_meta_api_get_type (); gst_fill_classification_meta (class_meta, prediction, predsize); - fail_if (class_meta->num_labels != predsize); + fail_if (class_meta->num_labels != predsize / sizeof (gfloat)); for (gint i = 0; i < class_meta->num_labels; i++) { fail_if (class_meta->label_probs[i] != values[i]); } From 255846ef1d1878f06e7bf9f7295a557e9c24036f Mon Sep 17 00:00:00 2001 From: ctrejos Date: Thu, 27 Feb 2020 16:33:33 -0600 Subject: [PATCH 14/16] Add double type to backend properties --- gst-libs/gst/r2inference/gstbackend.cc | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/gst-libs/gst/r2inference/gstbackend.cc b/gst-libs/gst/r2inference/gstbackend.cc index 51c8d381..a828d282 100644 --- a/gst-libs/gst/r2inference/gstbackend.cc +++ b/gst-libs/gst/r2inference/gstbackend.cc @@ -31,6 +31,8 @@ GST_DEBUG_CATEGORY_STATIC (gst_backend_debug_category); #define GST_CAT_DEFAULT gst_backend_debug_category +#define DOUBLE_PROPERTY_DEFAULT_VALUE 0.5 + class InferenceProperty { private: @@ -69,6 +71,11 @@ class InferenceProperty { g_value_get_int(avalue)); error = params->Set(apspec->name, g_value_get_int(avalue)); break; + case G_TYPE_DOUBLE: + GST_INFO_OBJECT (self, "Setting property: %s=%f\n", apspec->name, + g_value_get_double(avalue)); + error = params->Set(apspec->name, g_value_get_double(avalue)); + break; default: GST_WARNING_OBJECT (self, "Invalid property type"); break; @@ -197,10 +204,18 @@ gst_backend_param_to_spec (r2i::ParameterMeta *param) { NULL, (GParamFlags) gst_backend_param_flags (param->flags)); break; } + case (r2i::ParameterMeta::Type::DOUBLE): { + spec = g_param_spec_double (param->name.c_str (), + param->name.c_str (), + param->description.c_str (), + G_MINDOUBLE, + G_MAXDOUBLE, DOUBLE_PROPERTY_DEFAULT_VALUE, + (GParamFlags) gst_backend_param_flags (param->flags)); + break; + } default: g_return_val_if_reached (NULL); } - return spec; } @@ -222,6 +237,9 @@ gst_backend_set_property (GObject *object, guint property_id, case G_TYPE_INT: priv->params->Set(pspec->name, g_value_get_int(value)); break; + case G_TYPE_DOUBLE: + priv->params->Set(pspec->name, g_value_get_double(value)); + break; default: GST_WARNING_OBJECT (self, "Invalid property type"); break; @@ -241,6 +259,7 @@ gst_backend_get_property (GObject *object, guint property_id, GstBackend *self = GST_BACKEND (object); GstBackendPrivate *priv = GST_BACKEND_PRIVATE (self); int int_buffer; + double double_buffer; std::string string_buffer; GST_DEBUG_OBJECT (self, "get_property"); @@ -254,6 +273,10 @@ gst_backend_get_property (GObject *object, guint property_id, priv->params->Get (pspec->name, int_buffer); g_value_set_int (value, int_buffer); break; + case G_TYPE_DOUBLE: + priv->params->Get (pspec->name, double_buffer); + g_value_set_double (value, double_buffer); + break; } } } From ea30c18ed3b5f58226cddc325aa4d7ec49f9ee5a Mon Sep 17 00:00:00 2001 From: ctrejos Date: Fri, 28 Feb 2020 13:09:57 -0600 Subject: [PATCH 15/16] Change inferencecrop description to fit standard --- gst/inferenceutils/gstinferencecrop.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gst/inferenceutils/gstinferencecrop.cc b/gst/inferenceutils/gstinferencecrop.cc index f8180f27..3b8f14f8 100644 --- a/gst/inferenceutils/gstinferencecrop.cc +++ b/gst/inferenceutils/gstinferencecrop.cc @@ -124,7 +124,7 @@ gst_inference_crop_class_init (GstInferenceCropClass *klass) { gst_static_pad_template_get (&src_template)); gst_element_class_set_static_metadata (GST_ELEMENT_CLASS (klass), - "inferencecrop", "Filter", + "Inference Crop", "Filter", "Crops an incoming image based on an inference prediction bounding box", "Michael Gruner \n\t\t\t" " Lenin Torres "); From 3ad5b88a990295123cf2213f1d171a40a074ff40 Mon Sep 17 00:00:00 2001 From: ctrejos Date: Fri, 28 Feb 2020 15:31:33 -0600 Subject: [PATCH 16/16] Update R2Inference version requirement --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 1ef6a129..4c252549 100644 --- a/configure.ac +++ b/configure.ac @@ -288,7 +288,7 @@ if test "$USE_OPENCV" = "yes"; then fi dnl *** r2inference *** -R2INFERENCE_REQ=0.4.1 +R2INFERENCE_REQ=0.5.0 AG_GST_CHECK_FEATURE(R2INFERENCE, [RidgeRun\'s Inference Framework], r2inference, [ AG_GST_PKG_CHECK_MODULES(R2INFERENCE, r2inference-0.0 >= $R2INFERENCE_REQ) ],[],[],[