diff --git a/.config b/.config index 0c96810..4cf980b 100644 --- a/.config +++ b/.config @@ -1 +1 @@ -hailort_version=4.13 +hailort_version=4.14 diff --git a/README.rst b/README.rst index 9b8e968..b3819a6 100644 --- a/README.rst +++ b/README.rst @@ -7,7 +7,7 @@ Hailo TAPPAS - Optimized Execution of Video-Processing Pipelines :width: 150 :height: 20 -.. |hailort| image:: https://img.shields.io/badge/HailoRT-4.13.0-green +.. |hailort| image:: https://img.shields.io/badge/HailoRT-4.14.0-green :target: https://github.com/hailo-ai/hailort :alt: HailoRT :height: 20 @@ -22,7 +22,7 @@ Hailo TAPPAS - Optimized Execution of Video-Processing Pipelines :width: 20 :align: middle -.. image:: ./resources/TAPPAS.png +.. image:: ./resources/github_TAPPAS.jpg :height: 300 :width: 600 :align: center @@ -51,7 +51,7 @@ Demonstrating Hailo's system integration scenario of specific use cases on prede * Simplifying integration with Hailo’s runtime SW stack * Providing a starting point for customers to fine-tune their applications -.. image:: ./resources/HAILO_TAPPAS_SW_STACK.jpg +.. image:: ./resources/HAILO_TAPPAS_SW_STACK.svg ---- @@ -420,18 +420,22 @@ Our VMS reference application demonstrates the use of 5 networks over multiple s Changelog ---------- +**v3.25.0 (July 2023)** + +* Improved Yolov5seg post-process performance +* Updated Yolo networks to use the HailoRT native post-process (selected models) +* Added "non-blocking mode" and "wait-time" properties to hailoroundrobin element + **v3.24.0 (March 2023)** * Added support for `Rockchip RK3588 `_ (validated on Firefly ITX-3588J platform) * `Video Management System `_ now supports multi-device (Ubuntu 22.04 only) * `Video Management System `_ (single device) now works on Ubuntu 20.04 -* Added a new model to `Instance Segmentation Pipeline `_ -- `yolov5seg`, which has improved - performance compared to `yolact` +* Added a new model to `Instance Segmentation Pipeline `_: + * `yolov5seg` - which has improved performance compared to `yolact` * New applications for `i.MX8 `_: - * Object Detection and Pose Estimation (cascaded) * Multi-Stream Detection - * Added a TAPPAS Graphic User Interface to easily run selected general example applications (preview) on the TAPPAS Docker - to activate it, run `tappas-gui` * Added back `yolox_l_leaky` to the `Century general application `_ * Reduced docker size diff --git a/apps/h8/gstreamer/general/cascading_networks/object_detection_and_pose_estimation.sh b/apps/h8/gstreamer/general/cascading_networks/object_detection_and_pose_estimation.sh index da34231..8acaec4 100755 --- a/apps/h8/gstreamer/general/cascading_networks/object_detection_and_pose_estimation.sh +++ b/apps/h8/gstreamer/general/cascading_networks/object_detection_and_pose_estimation.sh @@ -17,13 +17,14 @@ function init_variables() { readonly DEFAULT_DETECTION_HEF_PATH="$RESOURCES_DIR/yolov5m_wo_spp_60p.hef" readonly DEFAULT_POST_ESTIMATION_HEF_PATH="$RESOURCES_DIR/mspn_regnetx_800mf.hef" - readonly DEFAULT_DETECTION_POSTPROCESS_SO="$POSTPROCESS_DIR/libyolo_post.so" + readonly DEFAULT_DETECTION_POSTPROCESS_SO="$POSTPROCESS_DIR/libyolo_hailortpp_post.so" readonly DEFAULT_LANDMARKS_POSTPROCESS_SO="$POSTPROCESS_DIR/libmspn_post.so" readonly DEFAULT_CROP_SO="$CROPING_ALGORITHMS_DIR/libmspn.so" readonly DEFAULT_JSON_CONFIG_PATH="$RESOURCES_DIR/configs/mspn.json" readonly DEFAULT_VDEVICE_KEY="1" + readonly DEFAULT_NETWORK_NAME="yolov5" video_sink_element=$([ "$XV_SUPPORTED" = "true" ] && echo "xvimagesink" || echo "ximagesink") video_sink="fpsdisplaysink video-sink=$video_sink_element text-overlay=false" @@ -34,6 +35,7 @@ function init_variables() { input_source=$DEFAULT_DETECTION_VIDEO_SOURCE crop_so=$DEFAULT_CROP_SO json_config_path=$DEFAULT_JSON_CONFIG_PATH + network_name=$DEFAULT_NETWORK_NAME internal_offset=false print_gst_launch_only=false @@ -51,7 +53,7 @@ function print_usage() { echo " --show-fps Print fps" echo " --max-camera-resolution WIDTHxHEIGHT The maximun input resolution from camera as an input (default ${DEFAULT_MAX_CAMERA_WIDTH}x${DEFAULT_MAX_CAMERA_HEGIHT})" echo " --print-gst-launch Print the ready gst-launch command without running it" - echo " --tcp-address If specified, set the sink to a TCP client (expected format is 'host:port')" + echo " --tcp-address Used for TAPPAS GUI, switchs the sink to TCP client" exit 0 } @@ -84,8 +86,8 @@ function parse_args() { tcp_host=$(echo $2 | awk -F':' '{print $1}') tcp_port=$(echo $2 | awk -F':' '{print $2}') video_sink="queue name=queue_before_sink leaky=no max-size-buffers=30 max-size-bytes=0 max-size-time=0 ! \ - x264enc tune=zerolatency ! \ - queue max-size-bytes=0 max-size-time=0 ! matroskamux ! tcpclientsink host=$tcp_host port=$tcp_port" + videoscale ! video/x-raw,width=836,height=546,format=RGB ! \ + queue max-size-bytes=0 max-size-time=0 ! tcpclientsink host=$tcp_host port=$tcp_port" shift else @@ -140,7 +142,7 @@ OBJECT_DETECTION_PIPELINE="videoscale qos=false ! \ hailonet hef-path=$detection_hef_path scheduling-algorithm=1 scheduler-threshold=5 \ scheduler-timeout-ms=100 vdevice-key=$DEFAULT_VDEVICE_KEY ! \ queue leaky=no max-size-buffers=3 max-size-bytes=0 max-size-time=0 ! \ - hailofilter name=detection so-path=$detection_postprocess_so qos=false function-name=yolov5 ! \ + hailofilter name=detection so-path=$detection_postprocess_so qos=false function-name=$network_name ! \ queue leaky=no max-size-buffers=3 max-size-bytes=0 max-size-time=0" LANDMARKS_PIPELINE="videoscale qos=false ! \ diff --git a/apps/h8/gstreamer/general/cascading_networks/readme_resources/object_detection_and_pose_estimation.gif b/apps/h8/gstreamer/general/cascading_networks/readme_resources/object_detection_and_pose_estimation.gif index 98298a7..3f6437f 100644 Binary files a/apps/h8/gstreamer/general/cascading_networks/readme_resources/object_detection_and_pose_estimation.gif and b/apps/h8/gstreamer/general/cascading_networks/readme_resources/object_detection_and_pose_estimation.gif differ diff --git a/apps/h8/gstreamer/general/century/README.rst b/apps/h8/gstreamer/general/century/README.rst index e32fce4..c3f3b16 100644 --- a/apps/h8/gstreamer/general/century/README.rst +++ b/apps/h8/gstreamer/general/century/README.rst @@ -30,6 +30,7 @@ Supported Networks ------------------ * 'yolov5m_wo_spp_60p' - https://github.com/hailo-ai/hailo_model_zoo/blob/master/hailo_model_zoo/cfg/networks/yolov5m_wo_spp_60p.yaml +* 'yolox_l_leaky' - https://github.com/hailo-ai/hailo_model_zoo/blob/master/hailo_model_zoo/cfg/networks/yolov5m_wo_spp_60p.yaml Run --- @@ -70,4 +71,14 @@ that are trained on your own dataset: - TAPPAS changes to replace model: - Update HEF_PATH on the .sh file - - Update ``resources/configs/yolov5.json`` with your new post-processing parameters (NMS) \ No newline at end of file + - Update ``resources/configs/yolov5.json`` with your new post-processing parameters (NMS) + +- ``yolox_l_leaky`` + + - `Retraining docker `_ + + - For best compatibility and performance with TAPPAS, use for compilation the corresponsing YAML file from above. + - TAPPAS changes to replace model: + + - Update HEF_PATH on the .sh file + - Update ``resources/configs/yolox.json`` with your new post-processing parameters (NMS) diff --git a/apps/h8/gstreamer/general/century/century.sh b/apps/h8/gstreamer/general/century/century.sh index fad3251..113d8d6 100755 --- a/apps/h8/gstreamer/general/century/century.sh +++ b/apps/h8/gstreamer/general/century/century.sh @@ -9,12 +9,11 @@ function init_variables() { readonly POSTPROCESS_DIR="$TAPPAS_WORKSPACE/apps/h8/gstreamer/libs/post_processes/" readonly RESOURCES_DIR="$TAPPAS_WORKSPACE/apps/h8/gstreamer/general/century/resources" - readonly DEFAULT_POSTPROCESS_SO="$POSTPROCESS_DIR/libyolo_post.so" + readonly DEFAULT_POSTPROCESS_SO="$POSTPROCESS_DIR/libyolo_hailortpp_post.so" readonly DEFAULT_VIDEO_SOURCE="$RESOURCES_DIR/detection_5m.mp4" readonly DEFAULT_HEF_PATH="$RESOURCES_DIR/yolov5m_wo_spp_60p.hef" readonly DEVICE_COUNT=4 readonly DEVICE_PREFIX="[-]" - readonly DEFAULT_JSON_CONFIG_PATH="$RESOURCES_DIR/configs/yolov5m.json" readonly DEFAULT_NETWORK_NAME="yolov5" network_name=$DEFAULT_NETWORK_NAME @@ -24,7 +23,6 @@ function init_variables() { device_count=$DEVICE_COUNT input_source=$DEFAULT_VIDEO_SOURCE hef_path=$DEFAULT_HEF_PATH - json_config_path=$DEFAULT_JSON_CONFIG_PATH print_gst_launch_only=false additional_parameters="" @@ -73,7 +71,6 @@ function parse_args() { if [ $2 == "yolox" ]; then network_name="yolox" hef_path="$RESOURCES_DIR/yolovx_l_leaky.hef" - json_config_path="$RESOURCES_DIR/configs/yolox.json" elif [ $2 != "yolov5m" ]; then echo "Received invalid network: $2. See expected arguments below:" print_usage @@ -115,16 +112,16 @@ else fi PIPELINE="gst-launch-1.0 \ - $source_element ! videoconvert ! \ - videoscale ! video/x-raw, pixel-aspect-ratio=1/1 ! \ + $source_element ! videoconvert qos=false ! \ + videoscale qos=false ! video/x-raw, pixel-aspect-ratio=1/1 ! \ queue leaky=no max-size-buffers=30 max-size-bytes=0 max-size-time=0 ! \ hailonet hef-path=$hef_path device-count=$device_count scheduling-algorithm=0 is-active=true ! \ queue leaky=no max-size-buffers=30 max-size-bytes=0 max-size-time=0 ! \ - hailofilter function-name=$network_name so-path=$postprocess_so config-path=$json_config_path qos=false ! \ + hailofilter function-name=$network_name so-path=$postprocess_so qos=false ! \ queue leaky=no max-size-buffers=30 max-size-bytes=0 max-size-time=0 ! \ hailooverlay qos=false ! \ queue leaky=no max-size-buffers=30 max-size-bytes=0 max-size-time=0 ! \ - videoconvert ! \ + videoconvert qos=false n-threads=4 ! \ fpsdisplaysink video-sink=$video_sink_element name=hailo_display sync=$sync_pipeline text-overlay=false ${additional_parameters}" echo "Running $network_name" diff --git a/apps/h8/gstreamer/general/classification/classification.sh b/apps/h8/gstreamer/general/classification/classification.sh index 4d8b581..c3527b2 100755 --- a/apps/h8/gstreamer/general/classification/classification.sh +++ b/apps/h8/gstreamer/general/classification/classification.sh @@ -9,14 +9,12 @@ function init_variables() { readonly RESOURCES_DIR="$TAPPAS_WORKSPACE/apps/h8/gstreamer/general/classification/resources" readonly POSTPROCESS_DIR="$TAPPAS_WORKSPACE/apps/h8/gstreamer/libs/post_processes/" readonly DEFAULT_POSTPROCESS_SO="$POSTPROCESS_DIR/libclassification.so" - readonly DEFAULT_DRAW_SO="$POSTPROCESS_DIR/libdetection_draw.so" readonly DEFAULT_VIDEO_SOURCE="$RESOURCES_DIR/classification_movie.mp4" readonly DEFAULT_HEF_PATH="$RESOURCES_DIR/resnet_v1_50.hef" hef_path=$DEFAULT_HEF_PATH input_source=$DEFAULT_VIDEO_SOURCE postprocess_so=$DEFAULT_POSTPROCESS_SO - draw_so=$DEFAULT_DRAW_SO print_gst_launch_only=false additional_parameters="" @@ -32,7 +30,7 @@ function print_usage() { echo " -i INPUT --input INPUT Set the video source (only videos allowed) (default $input_source)" echo " --show-fps Print fps" echo " --print-gst-launch Print the ready gst-launch command without running it" - echo " --tcp-address If specified, set the sink to a TCP client (expected format is 'host:port')" + echo " --tcp-address Used for TAPPAS GUI, switchs the sink to TCP client" exit 0 } @@ -60,8 +58,8 @@ function parse_args() { tcp_host=$(echo $2 | awk -F':' '{print $1}') tcp_port=$(echo $2 | awk -F':' '{print $2}') video_sink="queue name=queue_before_sink leaky=no max-size-buffers=30 max-size-bytes=0 max-size-time=0 ! \ - x264enc tune=zerolatency ! \ - queue max-size-bytes=0 max-size-time=0 ! matroskamux ! tcpclientsink host=$tcp_host port=$tcp_port" + videoscale ! video/x-raw,width=836,height=546,format=RGB ! \ + queue max-size-bytes=0 max-size-time=0 ! tcpclientsink host=$tcp_host port=$tcp_port" shift else diff --git a/apps/h8/gstreamer/general/detection/detection.sh b/apps/h8/gstreamer/general/detection/detection.sh index 4df0ab0..f5e4fdd 100755 --- a/apps/h8/gstreamer/general/detection/detection.sh +++ b/apps/h8/gstreamer/general/detection/detection.sh @@ -10,12 +10,11 @@ function init_variables() { readonly POSTPROCESS_DIR="$TAPPAS_WORKSPACE/apps/h8/gstreamer/libs/post_processes/" readonly RESOURCES_DIR="$TAPPAS_WORKSPACE/apps/h8/gstreamer/general/detection/resources" - readonly DEFAULT_POSTPROCESS_SO="$POSTPROCESS_DIR/libyolo_post.so" + readonly DEFAULT_POSTPROCESS_SO="$POSTPROCESS_DIR/libyolo_hailortpp_post.so" readonly DEFAULT_NETWORK_NAME="yolov5" readonly DEFAULT_BATCH_SIZE="1" readonly DEFAULT_VIDEO_SOURCE="$RESOURCES_DIR/detection.mp4" readonly DEFAULT_HEF_PATH="$RESOURCES_DIR/yolov5m_wo_spp_60p.hef" - readonly DEFAULT_JSON_CONFIG_PATH="$RESOURCES_DIR/configs/yolov5.json" video_sink_element=$([ "$XV_SUPPORTED" = "true" ] && echo "xvimagesink" || echo "ximagesink") @@ -25,7 +24,7 @@ function init_variables() { input_source=$DEFAULT_VIDEO_SOURCE batch_size=$DEFAULT_BATCH_SIZE hef_path=$DEFAULT_HEF_PATH - json_config_path=$DEFAULT_JSON_CONFIG_PATH + json_config_path="null" print_gst_launch_only=false additional_parameters="" @@ -33,8 +32,7 @@ function init_variables() { debug_stats_export="" sync_pipeline=false device_id_prop="" - tcp_host="" - tcp_port="" + tappas_gui_mode=false } function print_help_if_needed() { @@ -57,7 +55,7 @@ function print_usage() { echo " --show-fps Print fps" echo " --print-gst-launch Print the ready gst-launch command without running it" echo " --print-device-stats Print the power and temperature measured" - echo " --tcp-address If specified, set the sink to a TCP client (expected format is 'host:port')" + echo " --tcp-address Used for TAPPAS GUI, switchs the sink to TCP client" exit 0 } @@ -66,10 +64,12 @@ function parse_args() { if [ $1 == "--network" ]; then if [ $2 == "yolov4" ]; then network_name="yolov4" + postprocess_so="$POSTPROCESS_DIR/libyolo_post.so" hef_path="$RESOURCES_DIR/yolov4_leaky.hef" batch_size="4" json_config_path="$RESOURCES_DIR/configs/yolov4.json" elif [ $2 == "yolov3" ]; then + postprocess_so="$POSTPROCESS_DIR/libyolo_post.so" network_name="yolov3" hef_path="$RESOURCES_DIR/yolov3.hef" batch_size="4" @@ -108,9 +108,8 @@ function parse_args() { tcp_host=$(echo $2 | awk -F':' '{print $1}') tcp_port=$(echo $2 | awk -F':' '{print $2}') video_sink="queue name=queue_before_sink leaky=no max-size-buffers=30 max-size-bytes=0 max-size-time=0 ! \ - x264enc tune=zerolatency ! \ - queue max-size-bytes=0 max-size-time=0 ! matroskamux ! tcpclientsink host=$tcp_host port=$tcp_port" - + videoscale ! video/x-raw,width=836,height=546,format=RGB ! \ + tcpclientsink host=$tcp_host port=$tcp_port" shift else echo "Received invalid argument: $1. See expected arguments below:" diff --git a/apps/h8/gstreamer/general/detection/detection_every_x_frames.sh b/apps/h8/gstreamer/general/detection/detection_every_x_frames.sh index a88d322..48ea063 100755 --- a/apps/h8/gstreamer/general/detection/detection_every_x_frames.sh +++ b/apps/h8/gstreamer/general/detection/detection_every_x_frames.sh @@ -11,12 +11,11 @@ function init_variables() { readonly RESOURCES_DIR="$TAPPAS_WORKSPACE/apps/h8/gstreamer/general/detection/resources" readonly CROPPING_ALGORITHMS_DIR="$POSTPROCESS_DIR/cropping_algorithms" - readonly DEFAULT_POSTPROCESS_SO="$POSTPROCESS_DIR/libyolo_post.so" + readonly DEFAULT_POSTPROCESS_SO="$POSTPROCESS_DIR/libyolo_hailortpp_post.so" readonly DEFAULT_NETWORK_NAME="yolov5" readonly DEFAULT_BATCH_SIZE="1" readonly DEFAULT_VIDEO_SOURCE="$RESOURCES_DIR/detection.mp4" readonly DEFAULT_HEF_PATH="$RESOURCES_DIR/yolov5m_wo_spp_60p.hef" - readonly DEFAULT_JSON_CONFIG_PATH="$RESOURCES_DIR/configs/yolov5.json" # Cropping Algorithm Macros readonly WHOLE_BUFFER_CROP_SO="$CROPPING_ALGORITHMS_DIR/libwhole_buffer.so" @@ -28,7 +27,7 @@ function init_variables() { batch_size=$DEFAULT_BATCH_SIZE hef_path=$DEFAULT_HEF_PATH crop_so=$WHOLE_BUFFER_CROP_SO - json_config_path=$DEFAULT_JSON_CONFIG_PATH + json_config_path="null" print_gst_launch_only=false additional_parameters="" @@ -73,11 +72,13 @@ function parse_args() { hef_path="$RESOURCES_DIR/yolov4_leaky.hef" batch_size="4" json_config_path="$RESOURCES_DIR/configs/yolov4.json" + postprocess_so="$POSTPROCESS_DIR/libyolo_post.so" elif [ $2 == "yolov3" ]; then network_name="yolov3" hef_path="$RESOURCES_DIR/yolov3.hef" batch_size="4" json_config_path="$RESOURCES_DIR/configs/yolov3.json" + postprocess_so="$POSTPROCESS_DIR/libyolo_post.so" elif [ $2 == "mobilenet_ssd" ]; then network_name="mobilenet_ssd" hef_path="$RESOURCES_DIR/ssd_mobilenet_v1.hef" diff --git a/apps/h8/gstreamer/general/face_detection/face_detection.sh b/apps/h8/gstreamer/general/face_detection/face_detection.sh index 14d65a6..ce67daf 100755 --- a/apps/h8/gstreamer/general/face_detection/face_detection.sh +++ b/apps/h8/gstreamer/general/face_detection/face_detection.sh @@ -10,7 +10,6 @@ function init_variables() { readonly RESOURCES_DIR="$TAPPAS_WORKSPACE/apps/h8/gstreamer/general/face_detection/resources" readonly DEFAULT_POSTPROCESS_SO="$POSTPROCESS_DIR/libface_detection_post.so" readonly SCRFD_POSTPROCESS_SO="$POSTPROCESS_DIR/libscrfd_post.so" - readonly DEFAULT_DRAW_SO="$POSTPROCESS_DIR/libdetection_draw.so" readonly DEFAULT_VIDEO_SOURCE="$RESOURCES_DIR/face_detection.mp4" readonly DEFAULT_HEF_PATH="$RESOURCES_DIR/lightface_slim.hef" readonly DEFAULT_NETWORK_NAME="lightface" @@ -21,7 +20,6 @@ function init_variables() { input_source=$DEFAULT_VIDEO_SOURCE hef_path=$DEFAULT_HEF_PATH postprocess_so=$DEFAULT_POSTPROCESS_SO - draw_so=$DEFAULT_DRAW_SO sync_pipeline=false json_config_path=$DEFAULT_JSON_CONFIG_PATH diff --git a/apps/h8/gstreamer/general/facial_landmarks/facial_landmarks.sh b/apps/h8/gstreamer/general/facial_landmarks/facial_landmarks.sh index d6571ff..ba3d5b5 100755 --- a/apps/h8/gstreamer/general/facial_landmarks/facial_landmarks.sh +++ b/apps/h8/gstreamer/general/facial_landmarks/facial_landmarks.sh @@ -9,14 +9,12 @@ function init_variables() { readonly POSTPROCESS_DIR="$TAPPAS_WORKSPACE/apps/h8/gstreamer/libs/post_processes/" readonly RESOURCES_DIR="$TAPPAS_WORKSPACE/apps/h8/gstreamer/general/facial_landmarks/resources" readonly DEFAULT_POSTPROCESS_SO="$POSTPROCESS_DIR/libfacial_landmarks_post.so" - readonly DEFAULT_DRAW_SO="$POSTPROCESS_DIR/libfacial_landmarks_draw.so" readonly DEFAULT_VIDEO_SOURCE="$RESOURCES_DIR/faces_120_120.mp4" readonly DEFAULT_HEF_PATH="$RESOURCES_DIR/tddfa_mobilenet_v1.hef" hef_path=$DEFAULT_HEF_PATH input_source=$DEFAULT_VIDEO_SOURCE postprocess_so=$DEFAULT_POSTPROCESS_SO - draw_so=$DEFAULT_DRAW_SO print_gst_launch_only=false additional_parameters="" diff --git a/apps/h8/gstreamer/general/instance_segmentation/instance_segmentation.sh b/apps/h8/gstreamer/general/instance_segmentation/instance_segmentation.sh index 8cd6a10..1785c1a 100755 --- a/apps/h8/gstreamer/general/instance_segmentation/instance_segmentation.sh +++ b/apps/h8/gstreamer/general/instance_segmentation/instance_segmentation.sh @@ -13,6 +13,7 @@ function init_variables() { readonly DEFAULT_VIDEO_SOURCE="$RESOURCES_DIR/instance_segmentation.mp4" readonly DEFAULT_HEF_PATH="$RESOURCES_DIR/yolov5n_seg.hef" readonly DEFAULT_NETWORK_NAME="yolov5seg" + readonly json_config_path="$RESOURCES_DIR/configs/yolov5seg.json" postprocess_so=$DEFAULT_POSTPROCESS_SO network_name=$DEFAULT_NETWORK_NAME @@ -107,7 +108,7 @@ PIPELINE="gst-launch-1.0 \ queue leaky=no max-size-buffers=30 max-size-bytes=0 max-size-time=0 ! \ hailonet hef-path=$hef_path ! \ queue leaky=no max-size-buffers=30 max-size-bytes=0 max-size-time=0 ! \ - hailofilter function-name=$network_name so-path=$postprocess_so qos=false ! \ + hailofilter function-name=$network_name so-path=$postprocess_so config-path=$json_config_path qos=false ! \ queue leaky=no max-size-buffers=30 max-size-bytes=0 max-size-time=0 ! \ hailooverlay qos=false ! \ queue leaky=no max-size-buffers=30 max-size-bytes=0 max-size-time=0 ! \ diff --git a/apps/h8/gstreamer/general/instance_segmentation/resources/configs/yolov5seg.json b/apps/h8/gstreamer/general/instance_segmentation/resources/configs/yolov5seg.json new file mode 100644 index 0000000..dbdbd20 --- /dev/null +++ b/apps/h8/gstreamer/general/instance_segmentation/resources/configs/yolov5seg.json @@ -0,0 +1,34 @@ +{ + "iou_threshold": 0.6, + "score_threshold": 0.25, + "outputs_size": [20, 40, 80], + "outputs_name": ["yolov5n_seg/conv63", "yolov5n_seg/conv48", "yolov5n_seg/conv55", "yolov5n_seg/conv61"], + "anchors": [ + [ + 116, + 90, + 156, + 198, + 373, + 326 + ], + [ + 30, + 61, + 62, + 45, + 59, + 119 + ], + [ + 10, + 13, + 16, + 30, + 33, + 23 + ] + ], + "input_shape": [640, 640], + "strides": [32, 16, 8] +} \ No newline at end of file diff --git a/apps/h8/gstreamer/general/license_plate_recognition/license_plate_recognition.sh b/apps/h8/gstreamer/general/license_plate_recognition/license_plate_recognition.sh index 88ec962..801c806 100755 --- a/apps/h8/gstreamer/general/license_plate_recognition/license_plate_recognition.sh +++ b/apps/h8/gstreamer/general/license_plate_recognition/license_plate_recognition.sh @@ -20,8 +20,8 @@ function init_variables() { # Vehicle Detection Macros readonly VEHICLE_DETECTION_HEF="$RESOURCES_DIR/yolov5m_vehicles.hef" - readonly VEHICLE_DETECTION_POST_SO="$POSTPROCESS_DIR/libyolo_post.so" - readonly VEHICLE_DETECTION_POST_FUNC="yolov5_vehicles_only" + readonly VEHICLE_DETECTION_POST_SO="$POSTPROCESS_DIR/libyolo_hailortpp_post.so" + readonly VEHICLE_DETECTION_POST_FUNC="yolov5m_vehicles" # License Plate Detection Macros readonly LICENSE_PLATE_DETECTION_HEF="$RESOURCES_DIR/tiny_yolov4_license_plates.hef" @@ -78,7 +78,7 @@ function print_usage() { echo " --show-fps Print fps" echo " --print-gst-launch Print the ready gst-launch command without running it" echo " --print-device-stats Print the power and temperature measured" - echo " --tcp-address If specified, set the sink to a TCP client (expected format is 'host:port')" + echo " --tcp-address Used for TAPPAS GUI, switchs the sink to TCP client" exit 0 } @@ -95,8 +95,8 @@ function parse_args() { tcp_host=$(echo $2 | awk -F':' '{print $1}') tcp_port=$(echo $2 | awk -F':' '{print $2}') video_sink="queue name=queue_before_sink leaky=no max-size-buffers=30 max-size-bytes=0 max-size-time=0 ! \ - x264enc tune=zerolatency ! \ - queue max-size-bytes=0 max-size-time=0 ! matroskamux ! tcpclientsink host=$tcp_host port=$tcp_port" + videoscale ! video/x-raw,width=836,height=546,format=RGB ! \ + queue max-size-bytes=0 max-size-time=0 ! tcpclientsink host=$tcp_host port=$tcp_port" shift elif [ "$1" = "--show-fps" ]; then echo "Printing fps" diff --git a/apps/h8/gstreamer/general/multi_person_multi_camera_tracking/multi_person_multi_camera_tracking.sh b/apps/h8/gstreamer/general/multi_person_multi_camera_tracking/multi_person_multi_camera_tracking.sh index d1cecab..3d73115 100755 --- a/apps/h8/gstreamer/general/multi_person_multi_camera_tracking/multi_person_multi_camera_tracking.sh +++ b/apps/h8/gstreamer/general/multi_person_multi_camera_tracking/multi_person_multi_camera_tracking.sh @@ -43,7 +43,7 @@ function print_usage() { echo " --num-of-sources NUM Setting number of sources to given input (default value is 4)" echo " --print-gst-launch Print the ready gst-launch command without running it" echo " --online-dewarp Perform online dewarping" - echo " --tcp-address If specified, set the sink to a TCP client (expected format is 'host:port')" + echo " --tcp-address Used for TAPPAS GUI, switchs the sink to TCP client" exit 0 } @@ -79,8 +79,8 @@ function parse_args() { tcp_host=$(echo $2 | awk -F':' '{print $1}') tcp_port=$(echo $2 | awk -F':' '{print $2}') video_sink="queue name=queue_before_sink leaky=no max-size-buffers=30 max-size-bytes=0 max-size-time=0 ! \ - x264enc tune=zerolatency ! \ - queue max-size-bytes=0 max-size-time=0 ! matroskamux ! tcpclientsink host=$tcp_host port=$tcp_port" + videoscale ! video/x-raw,width=836,height=546,format=RGB ! \ + queue max-size-bytes=0 max-size-time=0 ! tcpclientsink host=$tcp_host port=$tcp_port" shift else @@ -134,7 +134,7 @@ function main() { agg1. agg1." pipeline="gst-launch-1.0 \ - hailoroundrobin name=fun ! \ + hailoroundrobin mode=1 name=fun ! \ queue name=hailo_pre_convert_0 leaky=no max-size-buffers=30 max-size-bytes=0 max-size-time=0 ! \ videoconvert n-threads=1 qos=false ! video/x-raw,format=RGB ! \ $dewarp_element \ diff --git a/apps/h8/gstreamer/general/multistream_detection/multi_stream_detection.sh b/apps/h8/gstreamer/general/multistream_detection/multi_stream_detection.sh index 91dbbea..52491df 100755 --- a/apps/h8/gstreamer/general/multistream_detection/multi_stream_detection.sh +++ b/apps/h8/gstreamer/general/multistream_detection/multi_stream_detection.sh @@ -8,9 +8,8 @@ function init_variables() { readonly RESOURCES_DIR="$TAPPAS_WORKSPACE/apps/h8/gstreamer/general/multistream_detection/resources" readonly POSTPROCESS_DIR="$TAPPAS_WORKSPACE/apps/h8/gstreamer/libs/post_processes/" - readonly POSTPROCESS_SO="$POSTPROCESS_DIR/libyolo_post.so" + readonly POSTPROCESS_SO="$POSTPROCESS_DIR/libyolo_hailortpp_post.so" readonly HEF_PATH="$RESOURCES_DIR/yolov5m_wo_spp_60p.hef" - readonly DEFAULT_JSON_CONFIG_PATH="$RESOURCES_DIR/configs/yolov5.json" num_of_src=12 live_src="" @@ -19,7 +18,6 @@ function init_variables() { compositor_locations="sink_0::xpos=0 sink_0::ypos=0 sink_1::xpos=640 sink_1::ypos=0 sink_2::xpos=1280 sink_2::ypos=0 sink_3::xpos=1920 sink_3::ypos=0 sink_4::xpos=0 sink_4::ypos=640 sink_5::xpos=640 sink_5::ypos=640 sink_6::xpos=1280 sink_6::ypos=640 sink_7::xpos=1920 sink_7::ypos=640 sink_8::xpos=0 sink_8::ypos=1280 sink_9::xpos=640 sink_9::ypos=1280 sink_10::xpos=1280 sink_10::ypos=1280 sink_11::xpos=1920 sink_11::ypos=1280 sink_12::xpos=0 sink_12::ypos=1920 sink_13::xpos=640 sink_13::ypos=1920 sink_14::xpos=1280 sink_14::ypos=1920 sink_15::xpos=1920 sink_15::ypos=1920" print_gst_launch_only=false video_sink_element=$([ "$XV_SUPPORTED" = "true" ] && echo "xvimagesink" || echo "ximagesink") - json_config_path=$DEFAULT_JSON_CONFIG_PATH } function print_usage() { @@ -81,6 +79,7 @@ function create_sources() { videoconvert ! videobox autocrop=true ! video/x-raw,width=640,height=640,pixel-aspect-ratio=1/1 ! \ $identity fun.sink_0 sid.src_0 ! queue name=comp_q_0 \ leaky=downstream max-size-buffers=30 max-size-bytes=0 max-size-time=0 ! comp.sink_0 " + streamrouter_input_streams+=" src_0::input-streams=\"\"" start_index=1 if [ $num_of_src -gt 4 ]; then num_of_src=4 @@ -95,6 +94,7 @@ function create_sources() { video/x-raw,width=640,height=640,pixel-aspect-ratio=1/1 ! $identity \ fun.sink_$n sid.src_$n ! queue name=comp_q_$n leaky=downstream max-size-buffers=30 \ max-size-bytes=0 max-size-time=0 ! comp.sink_$n " + streamrouter_input_streams+=" src_$n::input-streams=\"\"" done } @@ -104,14 +104,14 @@ function main() { create_sources pipeline="gst-launch-1.0 \ - funnel name=fun ! \ + hailoroundrobin mode=0 name=fun ! \ queue name=hailo_pre_infer_q_0 leaky=no max-size-buffers=30 max-size-bytes=0 max-size-time=0 ! \ hailonet hef-path=$HEF_PATH ! \ queue name=hailo_postprocess0 leaky=no max-size-buffers=30 max-size-bytes=0 max-size-time=0 ! \ - hailofilter so-path=$POSTPROCESS_SO config-path=$json_config_path qos=false ! \ + hailofilter so-path=$POSTPROCESS_SO qos=false ! \ queue name=hailo_draw0 leaky=no max-size-buffers=30 max-size-bytes=0 max-size-time=0 ! \ hailooverlay ! \ - streamiddemux name=sid compositor name=comp start-time-selection=0 $compositor_locations ! \ + hailostreamrouter name=sid $streamrouter_input_streams compositor name=comp start-time-selection=0 $compositor_locations ! \ queue name=hailo_video_q_0 leaky=no max-size-buffers=30 max-size-bytes=0 max-size-time=0 ! \ videoconvert ! queue name=hailo_display_q_0 leaky=no max-size-buffers=30 max-size-bytes=0 max-size-time=0 ! \ fpsdisplaysink video-sink=$video_sink_element name=hailo_display sync=false text-overlay=false \ diff --git a/apps/h8/gstreamer/general/multistream_detection/multi_stream_detection_rtsp.sh b/apps/h8/gstreamer/general/multistream_detection/multi_stream_detection_rtsp.sh index 9251c79..f065d0c 100755 --- a/apps/h8/gstreamer/general/multistream_detection/multi_stream_detection_rtsp.sh +++ b/apps/h8/gstreamer/general/multistream_detection/multi_stream_detection_rtsp.sh @@ -17,10 +17,9 @@ function init_variables() { readonly RESOURCES_DIR="$TAPPAS_WORKSPACE/apps/h8/gstreamer/general/multistream_detection/resources" readonly POSTPROCESS_DIR="$TAPPAS_WORKSPACE/apps/h8/gstreamer/libs/post_processes/" - readonly POSTPROCESS_SO="$POSTPROCESS_DIR/libyolo_post.so" + readonly POSTPROCESS_SO="$POSTPROCESS_DIR/libyolo_hailortpp_post.so" readonly STREAM_DISPLAY_SIZE=640 readonly HEF_PATH="$RESOURCES_DIR/yolov5m_wo_spp_60p.hef" - readonly DEFAULT_JSON_CONFIG_PATH="$RESOURCES_DIR/configs/yolov5.json" num_of_src=8 debug=false @@ -33,8 +32,6 @@ function init_variables() { compositor_locations="sink_0::xpos=0 sink_0::ypos=0 sink_1::xpos=640 sink_1::ypos=0 sink_2::xpos=1280 sink_2::ypos=0 sink_3::xpos=1920 sink_3::ypos=0 sink_4::xpos=0 sink_4::ypos=640 sink_5::xpos=640 sink_5::ypos=640 sink_6::xpos=1280 sink_6::ypos=640 sink_7::xpos=1920 sink_7::ypos=640" print_gst_launch_only=false decode_scale_elements="decodebin ! queue leaky=downstream max-size-buffers=5 max-size-bytes=0 max-size-time=0 ! videoscale n-threads=8 ! video/x-raw,pixel-aspect-ratio=1/1" - json_config_path=$DEFAULT_JSON_CONFIG_PATH - video_sink_element=$([ "$XV_SUPPORTED" = "true" ] && echo "xvimagesink" || echo "ximagesink") } @@ -119,9 +116,9 @@ function main() { pipeline="$gst_top_command gst-launch-1.0 \ funnel name=fun ! \ queue name=hailo_pre_infer_q_0 leaky=downstream max-size-buffers=5 max-size-bytes=0 max-size-time=0 ! \ - hailonet hef-path=$HEF_PATH ! \ + hailonet hef-path=$HEF_PATH ! \ queue name=hailo_postprocess0 leaky=no max-size-buffers=30 max-size-bytes=0 max-size-time=0 ! \ - hailofilter so-path=$POSTPROCESS_SO config-path=$json_config_path qos=false ! \ + hailofilter so-path=$POSTPROCESS_SO qos=false ! \ queue name=hailo_draw0 leaky=no max-size-buffers=30 max-size-bytes=0 max-size-time=0 ! \ hailooverlay ! streamiddemux name=sid \ compositor name=comp start-time-selection=0 $compositor_locations ! videoscale n-threads=8 name=disp_scale ! video/x-raw,width=$screen_width,height=$screen_height ! \ diff --git a/apps/h8/gstreamer/general/multistream_detection/multi_stream_detection_stream_multiplexer.sh b/apps/h8/gstreamer/general/multistream_detection/multi_stream_detection_stream_multiplexer.sh index 6309d78..3990e65 100755 --- a/apps/h8/gstreamer/general/multistream_detection/multi_stream_detection_stream_multiplexer.sh +++ b/apps/h8/gstreamer/general/multistream_detection/multi_stream_detection_stream_multiplexer.sh @@ -8,9 +8,8 @@ function init_variables() { readonly RESOURCES_DIR="$TAPPAS_WORKSPACE/apps/h8/gstreamer/general/multistream_detection/resources" readonly POSTPROCESS_DIR="$TAPPAS_WORKSPACE/apps/h8/gstreamer/libs/post_processes/" - readonly POSTPROCESS_SO="$POSTPROCESS_DIR/libyolo_post.so" + readonly POSTPROCESS_SO="$POSTPROCESS_DIR/libyolo_hailortpp_post.so" readonly HEF_PATH="$RESOURCES_DIR/yolov5m_wo_spp_60p.hef" - readonly DEFAULT_JSON_CONFIG_PATH="$RESOURCES_DIR/configs/yolov5.json" num_of_src=12 additional_parameters="" @@ -18,7 +17,6 @@ function init_variables() { compositor_locations="sink_0::xpos=0 sink_0::ypos=0 sink_1::xpos=640 sink_1::ypos=0 sink_2::xpos=1280 sink_2::ypos=0 sink_3::xpos=1920 sink_3::ypos=0 sink_4::xpos=0 sink_4::ypos=640 sink_5::xpos=640 sink_5::ypos=640 sink_6::xpos=1280 sink_6::ypos=640 sink_7::xpos=1920 sink_7::ypos=640 sink_8::xpos=0 sink_8::ypos=1280 sink_9::xpos=640 sink_9::ypos=1280 sink_10::xpos=1280 sink_10::ypos=1280 sink_11::xpos=1920 sink_11::ypos=1280 sink_12::xpos=0 sink_12::ypos=1920 sink_13::xpos=640 sink_13::ypos=1920 sink_14::xpos=1280 sink_14::ypos=1920 sink_15::xpos=1920 sink_15::ypos=1920" print_gst_launch_only=false video_sink_element=$([ "$XV_SUPPORTED" = "true" ] && echo "xvimagesink" || echo "ximagesink") - json_config_path=$DEFAULT_JSON_CONFIG_PATH } function print_usage() { @@ -74,7 +72,7 @@ function create_sources() { queue name=hailo_pre_infer_q_$n leaky=no max-size-buffers=30 max-size-bytes=0 max-size-time=0 ! \ hailonet hef-path=$HEF_PATH scheduling-algorithm=1 batch-size=1 vdevice-key=1 ! \ queue name=hailo_postprocess0_$n leaky=no max-size-buffers=30 max-size-bytes=0 max-size-time=0 ! \ - hailofilter so-path=$POSTPROCESS_SO config-path=$json_config_path qos=false ! \ + hailofilter so-path=$POSTPROCESS_SO qos=false ! \ queue name=hailo_draw0_$n leaky=no max-size-buffers=30 max-size-bytes=0 max-size-time=0 ! \ hailooverlay ! \ queue name=comp_q_$n leaky=downstream max-size-buffers=30 \ diff --git a/apps/h8/gstreamer/general/multistream_multidevice/rtsp_detection_and_pose_estimation.sh b/apps/h8/gstreamer/general/multistream_multidevice/rtsp_detection_and_pose_estimation.sh index 8a83971..1800485 100755 --- a/apps/h8/gstreamer/general/multistream_multidevice/rtsp_detection_and_pose_estimation.sh +++ b/apps/h8/gstreamer/general/multistream_multidevice/rtsp_detection_and_pose_estimation.sh @@ -20,12 +20,12 @@ function init_variables() { readonly POSE_ESTIMATION_HEF_PATH="$RESOURCES_DIR/joined_centerpose_repvgg_a0_center_nms_joint_nms.hef" readonly POSTPROCESS_DIR="$TAPPAS_WORKSPACE/apps/h8/gstreamer/libs/post_processes/" - readonly DETECTION_POSTPROCESS_SO="$POSTPROCESS_DIR/libyolo_post.so" + readonly DETECTION_POSTPROCESS_SO="$POSTPROCESS_DIR/libyolo_hailortpp_post.so" readonly POSE_ESTIMATION_POSTPROCESS_SO="$POSTPROCESS_DIR/libcenterpose_post.so" readonly POSE_ESTIMATION_POSTPROCESS_FUNCTION_NAME="centerpose_416" readonly DETECTION_POSTPROCESS_FUNCTION_NAME="yolov5_no_persons" readonly STREAM_DISPLAY_SIZE=300 - readonly DEFAULT_JSON_CONFIG_PATH="$RESOURCES_DIR/configs/yolov5.json" + readonly DEFAULT_JSON_CONFIG_PATH="None" num_of_src=8 debug=false @@ -128,7 +128,7 @@ function main() { determine_screen_size pipeline="$gst_top_command gst-launch-1.0 \ - hailoroundrobin funnel-mode=true name=fun ! video/x-raw, width=640, height=640 ! \ + hailoroundrobin mode=0 name=fun ! video/x-raw, width=640, height=640 ! \ queue name=hailo_pre_split leaky=downstream max-size-buffers=5 max-size-bytes=0 max-size-time=0 ! tee name=splitter \ hailomuxer name=hailomuxer ! queue name=hailo_draw0 leaky=no max-size-buffers=30 max-size-bytes=0 max-size-time=0 ! \ hailooverlay qos=false ! videoscale n-threads=8 ! video/x-raw,width=300,height=300 ! $streamrouter_disp_element \ diff --git a/apps/h8/gstreamer/general/multistream_multidevice/video_detection_and_pose_estimation.sh b/apps/h8/gstreamer/general/multistream_multidevice/video_detection_and_pose_estimation.sh index d233657..f50391e 100755 --- a/apps/h8/gstreamer/general/multistream_multidevice/video_detection_and_pose_estimation.sh +++ b/apps/h8/gstreamer/general/multistream_multidevice/video_detection_and_pose_estimation.sh @@ -12,11 +12,10 @@ function init_variables() { readonly POSE_ESTIMATION_HEF_PATH="$RESOURCES_DIR/centerpose_regnetx_1.6gf_fpn.hef" readonly POSTPROCESS_DIR="$TAPPAS_WORKSPACE/apps/h8/gstreamer/libs/post_processes/" - readonly DETECTION_POSTPROCESS_SO="$POSTPROCESS_DIR/libyolo_post.so" + readonly DETECTION_POSTPROCESS_SO="$POSTPROCESS_DIR/libyolo_hailortpp_post.so" readonly POSE_ESTIMATION_POSTPROCESS_SO="$POSTPROCESS_DIR/libcenterpose_post.so" readonly POSE_ESTIMATION_POSTPROCESS_FUNCTION_NAME="centerpose" readonly DETECTION_POSTPROCESS_FUNCTION_NAME="yolov5_no_persons" - readonly DEFAULT_JSON_CONFIG_PATH="$RESOURCES_DIR/configs/yolov5.json" num_of_src=8 debug=false @@ -25,7 +24,6 @@ function init_variables() { decode_element="decodebin" compositor_locations="sink_0::xpos=0 sink_0::ypos=0 sink_1::xpos=640 sink_1::ypos=0 sink_2::xpos=1280 sink_2::ypos=0 sink_3::xpos=1920 sink_3::ypos=0 sink_4::xpos=0 sink_4::ypos=640 sink_5::xpos=640 sink_5::ypos=640 sink_6::xpos=1280 sink_6::ypos=640 sink_7::xpos=1920 sink_7::ypos=640" print_gst_launch_only=false - json_config_path=$DEFAULT_JSON_CONFIG_PATH video_sink_element=$([ "$XV_SUPPORTED" = "true" ] && echo "xvimagesink" || echo "ximagesink") } @@ -101,13 +99,13 @@ function main() { create_sources pipeline="$gst_top_command gst-launch-1.0 \ - hailoroundrobin name=fun ! queue name=hailo_pre_split leaky=no max-size-buffers=30 max-size-bytes=0 max-size-time=0 ! tee name=splitter \ + hailoroundrobin mode=1 name=fun ! queue name=hailo_pre_split leaky=no max-size-buffers=30 max-size-bytes=0 max-size-time=0 ! tee name=splitter \ hailomuxer name=hailomuxer ! queue name=hailo_draw0 leaky=no max-size-buffers=30 max-size-bytes=0 max-size-time=0 ! \ hailooverlay qos=false ! $streamrouter_disp_element \ splitter. ! queue name=hailo_pre_infer_q_1 leaky=no max-size-buffers=30 max-size-bytes=0 max-size-time=0 ! \ hailonet hef-path=$DETECTION_HEF_PATH scheduling-algorithm=0 is-active=true ! \ queue name=hailo_postprocess0 leaky=no max-size-buffers=30 max-size-bytes=0 max-size-time=0 ! \ - hailofilter so-path=$DETECTION_POSTPROCESS_SO config-path=$json_config_path function-name=$DETECTION_POSTPROCESS_FUNCTION_NAME qos=false ! hailomuxer. \ + hailofilter so-path=$DETECTION_POSTPROCESS_SO function-name=$DETECTION_POSTPROCESS_FUNCTION_NAME qos=false ! hailomuxer. \ splitter. ! queue name=hailo_pre_infer_q_0 leaky=no max-size-buffers=30 max-size-bytes=0 max-size-time=0 ! \ hailonet hef-path=$POSE_ESTIMATION_HEF_PATH scheduling-algorithm=0 is-active=true ! \ queue name=hailo_postprocess1 leaky=no max-size-buffers=30 max-size-bytes=0 max-size-time=0 ! \ diff --git a/apps/h8/gstreamer/general/pose_estimation/pose_estimation.sh b/apps/h8/gstreamer/general/pose_estimation/pose_estimation.sh index dc431ef..012f7a4 100755 --- a/apps/h8/gstreamer/general/pose_estimation/pose_estimation.sh +++ b/apps/h8/gstreamer/general/pose_estimation/pose_estimation.sh @@ -36,7 +36,7 @@ function print_usage() { echo " --network NETWORK Set network to use. choose from [centerpose, centerpose_416], default is centerpose" echo " --show-fps Printing fps" echo " --print-gst-launch Print the ready gst-launch command without running it" - echo " --tcp-address If specified, set the sink to a TCP client (expected format is 'host:port')" + echo " --tcp-address Used for TAPPAS GUI, switchs the sink to TCP client" exit 0 } @@ -74,8 +74,8 @@ function parse_args() { tcp_host=$(echo $2 | awk -F':' '{print $1}') tcp_port=$(echo $2 | awk -F':' '{print $2}') video_sink="queue name=queue_before_sink leaky=no max-size-buffers=30 max-size-bytes=0 max-size-time=0 ! \ - x264enc tune=zerolatency ! \ - queue max-size-bytes=0 max-size-time=0 ! matroskamux ! tcpclientsink host=$tcp_host port=$tcp_port" + videoscale ! video/x-raw,width=836,height=546,format=RGB ! \ + queue max-size-bytes=0 max-size-time=0 ! tcpclientsink host=$tcp_host port=$tcp_port" shift else diff --git a/apps/h8/gstreamer/imx6/detection/README.rst b/apps/h8/gstreamer/imx6/detection/README.rst index aa352ff..967d47c 100644 --- a/apps/h8/gstreamer/imx6/detection/README.rst +++ b/apps/h8/gstreamer/imx6/detection/README.rst @@ -8,6 +8,9 @@ Detection The purpose of ``detection.sh`` is to demonstrate object detection using a camera/file as input with i.MX6 hardware accelerators. This is done by running a ``single-stream object detection pipeline`` on top of GStreamer i.MX6 elements using the Hailo-8 device. +.. warning:: + This application is currently not supported on i.MX6 + .. raw:: html
diff --git a/apps/h8/gstreamer/imx8/README.rst b/apps/h8/gstreamer/imx8/README.rst index 59f4003..ac47cd3 100644 --- a/apps/h8/gstreamer/imx8/README.rst +++ b/apps/h8/gstreamer/imx8/README.rst @@ -2,6 +2,7 @@ iMX8 GStreamer based applications ================================= +.. warning:: The Kirkstone applications portfolio is reduced on i.MX8-based devices, since the Kirkstone branch does not support OpenGL. #. `Detection `_ - single-stream object detection pipeline on top of GStreamer using the Hailo-8 device. #. `Depth Estimation `_ - single-stream depth estimation pipeline on top of GStreamer using the Hailo-8 device. diff --git a/apps/h8/gstreamer/imx8/cascading_networks/object_detection_and_pose_estimation.sh b/apps/h8/gstreamer/imx8/cascading_networks/object_detection_and_pose_estimation.sh index 5b4987a..73c236d 100755 --- a/apps/h8/gstreamer/imx8/cascading_networks/object_detection_and_pose_estimation.sh +++ b/apps/h8/gstreamer/imx8/cascading_networks/object_detection_and_pose_estimation.sh @@ -14,7 +14,7 @@ function init_variables() { readonly DEFAULT_DETECTION_HEF_PATH="$RESOURCES_DIR/yolov5m_wo_spp_60p.hef" readonly DEFAULT_POST_ESTIMATION_HEF_PATH="$RESOURCES_DIR/mspn_regnetx_800mf.hef" - readonly DEFAULT_DETECTION_POSTPROCESS_SO="$POSTPROCESS_DIR/libyolo_post.so" + readonly DEFAULT_DETECTION_POSTPROCESS_SO="$POSTPROCESS_DIR/libyolo_hailortpp_post.so" readonly DEFAULT_LANDMARKS_POSTPROCESS_SO="$POSTPROCESS_DIR/libmspn_post.so" readonly DEFAULT_CROP_SO="$CROPING_ALGORITHMS_DIR/libmspn.so" @@ -32,7 +32,6 @@ function init_variables() { crop_so=$DEFAULT_CROP_SO json_config_path=$DEFAULT_JSON_CONFIG_PATH - internal_offset=false print_gst_launch_only=false additonal_parameters="" diff --git a/apps/h8/gstreamer/imx8/cascading_networks/readme_resources/object_detection_and_pose_estimation.gif b/apps/h8/gstreamer/imx8/cascading_networks/readme_resources/object_detection_and_pose_estimation.gif index 98298a7..3f6437f 100644 Binary files a/apps/h8/gstreamer/imx8/cascading_networks/readme_resources/object_detection_and_pose_estimation.gif and b/apps/h8/gstreamer/imx8/cascading_networks/readme_resources/object_detection_and_pose_estimation.gif differ diff --git a/apps/h8/gstreamer/imx8/license_plate_recognition/license_plate_recognition.sh b/apps/h8/gstreamer/imx8/license_plate_recognition/license_plate_recognition.sh index be75d19..11aa8af 100755 --- a/apps/h8/gstreamer/imx8/license_plate_recognition/license_plate_recognition.sh +++ b/apps/h8/gstreamer/imx8/license_plate_recognition/license_plate_recognition.sh @@ -6,6 +6,9 @@ CURRENT_DIR="$(dirname "$(realpath "${BASH_SOURCE[0]}")")" function init_variables() { print_help_if_needed $@ + # Temporary file + readonly FILE_LOADED_TO_CACHE_PATH=/tmp/hailo_lpr + # Basic Directories readonly POSTPROCESS_DIR="/usr/lib/hailo-post-processes" readonly CROPPING_ALGORITHMS_DIR="$POSTPROCESS_DIR/cropping_algorithms" @@ -98,10 +101,24 @@ function parse_args() { done } +function load_file_to_cache() { + # Loading the file to the cache is required after every reboot when using iMX8 based machines + # This file is an indication that we already loaded the file to the cache + if [ ! -f "$FILE_LOADED_TO_CACHE_PATH" ]; then + load_file_to_cache_pipeline="$source_element ! fakesink sync=false" + eval "gst-launch-1.0 $load_file_to_cache_pipeline" + + # Indicate that the file is already loaded + touch "$FILE_LOADED_TO_CACHE_PATH" + fi +} + init_variables $@ parse_args $@ source_element="filesrc location=$input_source name=src_0 ! rawvideoparse format=yuy2 width=1920 height=1080" internal_offset=true +load_file_to_cache + function create_lp_detection_pipeline() { pipeline_1="queue leaky=no max-size-buffers=30 max-size-bytes=0 max-size-time=0 ! \ @@ -163,4 +180,4 @@ if [ "$print_gst_launch_only" = true ]; then exit 0 fi -eval ${PIPELINE} \ No newline at end of file +eval ${PIPELINE} diff --git a/apps/h8/gstreamer/imx8/segmentation/semantic_segmentation.sh b/apps/h8/gstreamer/imx8/segmentation/semantic_segmentation.sh index 9733bf7..65de339 100755 --- a/apps/h8/gstreamer/imx8/segmentation/semantic_segmentation.sh +++ b/apps/h8/gstreamer/imx8/segmentation/semantic_segmentation.sh @@ -10,7 +10,7 @@ function init_variables() { readonly POSTPROCESS_DIR="/usr/lib/hailo-post-processes" readonly DEFAULT_POSTPROCESS_SO="$POSTPROCESS_DIR/libsemantic_segmentation.so" readonly DEFAULT_VIDEO_SOURCE="$RESOURCES_DIR/full_mov_slow.mp4" - readonly DEFAULT_HEF_PATH="$RESOURCES_DIR/fcn8_resnet_v1_18.hef" + readonly DEFAULT_HEF_PATH="$RESOURCES_DIR/resnet18_fcn8_fhd.hef" input_source=$DEFAULT_VIDEO_SOURCE hef_path=$DEFAULT_HEF_PATH diff --git a/apps/h8/gstreamer/raspberrypi/classification/classification.sh b/apps/h8/gstreamer/raspberrypi/classification/classification.sh index b123bb4..7043317 100755 --- a/apps/h8/gstreamer/raspberrypi/classification/classification.sh +++ b/apps/h8/gstreamer/raspberrypi/classification/classification.sh @@ -9,7 +9,6 @@ function init_variables() { readonly RESOURCES_DIR="$TAPPAS_WORKSPACE/apps/h8/gstreamer/raspberrypi/classification/resources" readonly POSTPROCESS_DIR="$TAPPAS_WORKSPACE/apps/h8/gstreamer/libs/post_processes/" readonly DEFAULT_POSTPROCESS_SO="$POSTPROCESS_DIR/libclassification.so" - readonly DEFAULT_DRAW_SO="$POSTPROCESS_DIR/libdetection_draw.so" readonly DEFAULT_VIDEO_SOURCE="$RESOURCES_DIR/classification_movie.mp4" readonly DEFAULT_HEF_PATH="$RESOURCES_DIR/resnet_v1_50.hef" readonly DEFAULT_VDEVICE_KEY="1" @@ -17,7 +16,6 @@ function init_variables() { hef_path=$DEFAULT_HEF_PATH input_source=$DEFAULT_VIDEO_SOURCE postprocess_so=$DEFAULT_POSTPROCESS_SO - draw_so=$DEFAULT_DRAW_SO print_gst_launch_only=false additional_parameters="" diff --git a/apps/h8/gstreamer/raspberrypi/detection/detection.sh b/apps/h8/gstreamer/raspberrypi/detection/detection.sh index cbf1c39..8a2a55f 100755 --- a/apps/h8/gstreamer/raspberrypi/detection/detection.sh +++ b/apps/h8/gstreamer/raspberrypi/detection/detection.sh @@ -9,12 +9,11 @@ function init_variables() { readonly POSTPROCESS_DIR="$TAPPAS_WORKSPACE/apps/h8/gstreamer/libs/post_processes" readonly RESOURCES_DIR="$TAPPAS_WORKSPACE/apps/h8/gstreamer/raspberrypi/detection/resources" - readonly DEFAULT_POSTPROCESS_SO="$POSTPROCESS_DIR/libyolo_post.so" + readonly DEFAULT_POSTPROCESS_SO="$POSTPROCESS_DIR/libyolo_hailortpp_post.so" readonly DEFAULT_NETWORK_NAME="yolov5" readonly DEFAULT_BATCH_SIZE="1" readonly DEFAULT_VIDEO_SOURCE="$RESOURCES_DIR/detection.mp4" readonly DEFAULT_HEF_PATH="$RESOURCES_DIR/yolov5m_wo_spp_60p.hef" - readonly DEFAULT_JSON_CONFIG_PATH="$RESOURCES_DIR/configs/yolov5.json" postprocess_so=$DEFAULT_POSTPROCESS_SO network_name=$DEFAULT_NETWORK_NAME @@ -33,7 +32,6 @@ function init_variables() { camera_input_width=640 camera_input_height=640 camera_input_format="RGB" - json_config_path=$DEFAULT_JSON_CONFIG_PATH } function print_help_if_needed() { @@ -142,7 +140,7 @@ PIPELINE="${debug_stats_export} gst-launch-1.0 ${stats_element} \ queue max-size-buffers=5 max-size-bytes=0 max-size-time=0 ! \ hailonet hef-path=$hef_path $device_id_prop batch-size=$batch_size ! \ queue max-size-buffers=5 max-size-bytes=0 max-size-time=0 ! \ - hailofilter function-name=$network_name config-path=$json_config_path so-path=$postprocess_so qos=false ! \ + hailofilter function-name=$network_name so-path=$postprocess_so qos=false ! \ queue max-size-buffers=5 max-size-bytes=0 max-size-time=0 ! \ hailooverlay ! \ queue max-size-buffers=5 max-size-bytes=0 max-size-time=0 ! \ diff --git a/apps/h8/gstreamer/raspberrypi/face_detection/face_detection.sh b/apps/h8/gstreamer/raspberrypi/face_detection/face_detection.sh index 642abf6..25ac078 100755 --- a/apps/h8/gstreamer/raspberrypi/face_detection/face_detection.sh +++ b/apps/h8/gstreamer/raspberrypi/face_detection/face_detection.sh @@ -9,7 +9,6 @@ function init_variables() { readonly POSTPROCESS_DIR="$TAPPAS_WORKSPACE/apps/h8/gstreamer/libs/post_processes" readonly RESOURCES_DIR="$TAPPAS_WORKSPACE/apps/h8/gstreamer/raspberrypi/face_detection/resources" readonly DEFAULT_POSTPROCESS_SO="$POSTPROCESS_DIR/libface_detection_post.so" - readonly DEFAULT_DRAW_SO="$POSTPROCESS_DIR/libdetection_draw.so" readonly DEFAULT_VIDEO_SOURCE="$RESOURCES_DIR/face_detection.mp4" readonly DEFAULT_HEF_PATH="$RESOURCES_DIR/lightface_slim.hef" readonly DEFAULT_NETWORK_NAME="lightface" @@ -19,7 +18,6 @@ function init_variables() { input_source=$DEFAULT_VIDEO_SOURCE hef_path=$DEFAULT_HEF_PATH postprocess_so=$DEFAULT_POSTPROCESS_SO - draw_so=$DEFAULT_DRAW_SO sync_pipeline=false print_gst_launch_only=false diff --git a/apps/h8/gstreamer/rockchip/detection/detection.sh b/apps/h8/gstreamer/rockchip/detection/detection.sh index e3cd863..95e3ced 100755 --- a/apps/h8/gstreamer/rockchip/detection/detection.sh +++ b/apps/h8/gstreamer/rockchip/detection/detection.sh @@ -10,7 +10,7 @@ function init_variables() { readonly POSTPROCESS_DIR="$TAPPAS_WORKSPACE/apps/h8/gstreamer/libs/post_processes/" readonly RESOURCES_DIR="$TAPPAS_WORKSPACE/apps/h8/gstreamer/rockchip/detection/resources" - readonly DEFAULT_POSTPROCESS_SO="$POSTPROCESS_DIR/libyolo_post.so" + readonly DEFAULT_POSTPROCESS_SO="$POSTPROCESS_DIR/libyolo_hailortpp_post.so" readonly DEFAULT_NETWORK_NAME="yolov5" readonly DEFAULT_BATCH_SIZE="1" readonly DEFAULT_VIDEO_SOURCE="$RESOURCES_DIR/detection.mp4" @@ -63,6 +63,7 @@ function parse_args() { if [ $1 == "--network" ]; then if [ $2 == "yolov4" ]; then network_name="yolov4" + postprocess_so="$POSTPROCESS_DIR/libyolo_post.so" hef_path="$RESOURCES_DIR/yolov4_leaky.hef" batch_size="4" json_config_path="$RESOURCES_DIR/configs/yolov4.json" @@ -70,6 +71,7 @@ function parse_args() { network_name="yolov3" hef_path="$RESOURCES_DIR/yolov3.hef" batch_size="4" + postprocess_so="$POSTPROCESS_DIR/libyolo_post.so" json_config_path="$RESOURCES_DIR/configs/yolov3.json" elif [ $2 == "nanodet" ]; then network_name="nanodet_repvgg" diff --git a/apps/h8/gstreamer/rockchip/license_plate_recognition/license_plate_recognition.sh b/apps/h8/gstreamer/rockchip/license_plate_recognition/license_plate_recognition.sh index abe79ad..af99cf7 100755 --- a/apps/h8/gstreamer/rockchip/license_plate_recognition/license_plate_recognition.sh +++ b/apps/h8/gstreamer/rockchip/license_plate_recognition/license_plate_recognition.sh @@ -20,8 +20,8 @@ function init_variables() { # Vehicle Detection Macros readonly VEHICLE_DETECTION_HEF="$RESOURCES_DIR/yolov5m_vehicles.hef" - readonly VEHICLE_DETECTION_POST_SO="$POSTPROCESS_DIR/libyolo_post.so" - readonly VEHICLE_DETECTION_POST_FUNC="yolov5_vehicles_only" + readonly VEHICLE_DETECTION_POST_SO="$POSTPROCESS_DIR/libyolo_hailortpp_post.so" + readonly VEHICLE_DETECTION_POST_FUNC="yolov5m_vehicles" # License Plate Detection Macros readonly LICENSE_PLATE_DETECTION_HEF="$RESOURCES_DIR/tiny_yolov4_license_plates.hef" diff --git a/apps/h8/gstreamer/rockchip/multistream_detection/multi_stream_detection.sh b/apps/h8/gstreamer/rockchip/multistream_detection/multi_stream_detection.sh index 682ced6..fb1d9dc 100755 --- a/apps/h8/gstreamer/rockchip/multistream_detection/multi_stream_detection.sh +++ b/apps/h8/gstreamer/rockchip/multistream_detection/multi_stream_detection.sh @@ -9,9 +9,8 @@ function init_variables() { readonly RESOURCES_DIR="$TAPPAS_WORKSPACE/apps/h8/gstreamer/rockchip/multistream_detection/resources" readonly CONFIGS_DIR="$TAPPAS_WORKSPACE/apps/h8/gstreamer/rockchip/multistream_detection/resources/configs" readonly POSTPROCESS_DIR="$TAPPAS_WORKSPACE/apps/h8/gstreamer/libs/post_processes/" - readonly POSTPROCESS_SO="$POSTPROCESS_DIR/libyolo_post.so" + readonly POSTPROCESS_SO="$POSTPROCESS_DIR/libyolo_hailortpp_post.so" readonly HEF_PATH="$RESOURCES_DIR/yolov5s_nv12.hef" - readonly DEFAULT_JSON_CONFIG_PATH="$CONFIGS_DIR/yolov5.json" readonly OVERLAY_PIPELINE="queue name=hailo_stream leaky=no max-size-buffers=3 max-size-bytes=0 max-size-time=0 ! hailooverlay !" readonly DEFAULT_IP_ADDRESS="127.0.0.1" readonly BATCH_SIZE=8 @@ -22,7 +21,6 @@ function init_variables() { streamrouter_input_streams="" print_gst_launch_only=false overlay_element="" - json_config_path=$DEFAULT_JSON_CONFIG_PATH ip_address=$DEFAULT_IP_ADDRESS } @@ -98,7 +96,7 @@ function main() { parse_args $@ create_sources pipeline="gst-launch-1.0 \ - hailoroundrobin name=fun ! \ + hailoroundrobin mode=1 name=fun ! \ queue name=hailo_pre_tee_q_0 leaky=no max-size-buffers=5 max-size-bytes=0 max-size-time=0 ! \ tee name=t hailomuxer name=hmux \ t. ! queue name=bypass leaky=no max-size-buffers=20 max-size-bytes=0 max-size-time=0 ! hmux. \ @@ -108,7 +106,7 @@ function main() { queue name=hailo_pre_infer_q_0 leaky=no max-size-buffers=3 max-size-bytes=0 max-size-time=0 ! \ hailonet hef-path=$HEF_PATH is-active=true batch-size=$BATCH_SIZE ! \ queue name=hailo_postprocess0 leaky=no max-size-buffers=3 max-size-bytes=0 max-size-time=0 ! \ - hailofilter so-path=$POSTPROCESS_SO config-path=$DEFAULT_JSON_CONFIG_PATH qos=false ! \ + hailofilter so-path=$POSTPROCESS_SO qos=false ! \ hmux. hmux. ! \ $overlay_element \ queue name=hailo_over leaky=no max-size-buffers=5 max-size-bytes=0 max-size-time=0 ! \ diff --git a/apps/h8/gstreamer/rockchip/multistream_detection/multi_stream_detection_rtsp.sh b/apps/h8/gstreamer/rockchip/multistream_detection/multi_stream_detection_rtsp.sh index 791a4bb..51f03e4 100755 --- a/apps/h8/gstreamer/rockchip/multistream_detection/multi_stream_detection_rtsp.sh +++ b/apps/h8/gstreamer/rockchip/multistream_detection/multi_stream_detection_rtsp.sh @@ -18,9 +18,8 @@ function init_variables() { readonly RESOURCES_DIR="$TAPPAS_WORKSPACE/apps/h8/gstreamer/rockchip/multistream_detection/resources" readonly CONFIGS_DIR="$TAPPAS_WORKSPACE/apps/h8/gstreamer/rockchip/multistream_detection/resources/configs" readonly POSTPROCESS_DIR="$TAPPAS_WORKSPACE/apps/h8/gstreamer/libs/post_processes/" - readonly POSTPROCESS_SO="$POSTPROCESS_DIR/libyolo_post.so" + readonly POSTPROCESS_SO="$POSTPROCESS_DIR/libyolo_hailortpp_post.so" readonly HEF_PATH="$RESOURCES_DIR/yolov5s_nv12.hef" - readonly DEFAULT_JSON_CONFIG_PATH="$CONFIGS_DIR/yolov5.json" readonly OVERLAY_PIPELINE="queue name=hailo_stream leaky=no max-size-buffers=3 max-size-bytes=0 max-size-time=0 ! hailooverlay !" readonly DEFAULT_IP_ADDRESS="127.0.0.1" @@ -30,7 +29,6 @@ function init_variables() { streamrouter_input_streams="" print_gst_launch_only=false overlay_element="" - json_config_path=$DEFAULT_JSON_CONFIG_PATH ip_address=$DEFAULT_IP_ADDRESS } @@ -110,7 +108,7 @@ function main() { parse_args $@ create_sources pipeline="gst-launch-1.0 \ - hailoroundrobin name=fun ! \ + hailoroundrobin mode=1 name=fun ! \ queue name=hailo_pre_tee_q_0 leaky=no max-size-buffers=5 max-size-bytes=0 max-size-time=0 ! \ tee name=t hailomuxer name=hmux \ t. ! queue name=bypass leaky=no max-size-buffers=20 max-size-bytes=0 max-size-time=0 ! hmux. \ @@ -120,7 +118,7 @@ function main() { queue name=hailo_pre_infer_q_0 leaky=no max-size-buffers=3 max-size-bytes=0 max-size-time=0 ! \ hailonet hef-path=$HEF_PATH is-active=true batch-size=8 ! \ queue name=hailo_postprocess0 leaky=no max-size-buffers=3 max-size-bytes=0 max-size-time=0 ! \ - hailofilter so-path=$POSTPROCESS_SO config-path=$DEFAULT_JSON_CONFIG_PATH qos=false ! \ + hailofilter so-path=$POSTPROCESS_SO qos=false ! \ hmux. hmux. ! \ $overlay_element \ queue name=hailo_over leaky=no max-size-buffers=5 max-size-bytes=0 max-size-time=0 ! \ diff --git a/apps/h8/gstreamer/x86_hw_accelerated/century/century.sh b/apps/h8/gstreamer/x86_hw_accelerated/century/century.sh index d8466f9..b438110 100755 --- a/apps/h8/gstreamer/x86_hw_accelerated/century/century.sh +++ b/apps/h8/gstreamer/x86_hw_accelerated/century/century.sh @@ -21,12 +21,11 @@ function init_variables() { readonly POSTPROCESS_DIR="$TAPPAS_WORKSPACE/apps/h8/gstreamer/libs/post_processes/" readonly RESOURCES_DIR="$TAPPAS_WORKSPACE/apps/h8/gstreamer/x86_hw_accelerated/century/resources" - readonly DEFAULT_POSTPROCESS_SO="$POSTPROCESS_DIR/libyolo_post.so" + readonly DEFAULT_POSTPROCESS_SO="$POSTPROCESS_DIR/libyolo_hailortpp_post.so" readonly DEFAULT_VIDEO_SOURCE="$RESOURCES_DIR/detection_5m.mp4" readonly DEFAULT_HEF_PATH="$RESOURCES_DIR/yolov5m_wo_spp_60p.hef" readonly DEVICE_COUNT=4 readonly DEVICE_PREFIX="[-]" - readonly DEFAULT_JSON_CONFIG_PATH="$RESOURCES_DIR/configs/yolov5.json" video_sink_element=$([ "$XV_SUPPORTED" = "true" ] && echo "xvimagesink" || echo "ximagesink") postprocess_so=$DEFAULT_POSTPROCESS_SO @@ -34,7 +33,6 @@ function init_variables() { device_count=$DEVICE_COUNT input_source=$DEFAULT_VIDEO_SOURCE hef_path=$DEFAULT_HEF_PATH - json_config_path=$DEFAULT_JSON_CONFIG_PATH print_gst_launch_only=false additional_parameters="" @@ -121,11 +119,11 @@ function main() { queue leaky=no max-size-buffers=30 max-size-bytes=0 max-size-time=0 ! \ hailonet hef-path=$hef_path device-count=$device_count scheduling-algorithm=0 is-active=true ! \ queue leaky=no max-size-buffers=30 max-size-bytes=0 max-size-time=0 ! \ - hailofilter function-name=$network_name so-path=$postprocess_so config-path=$json_config_path qos=false ! \ + hailofilter function-name=$network_name so-path=$postprocess_so qos=false ! \ queue leaky=no max-size-buffers=30 max-size-bytes=0 max-size-time=0 ! \ hailooverlay qos=false ! \ queue leaky=no max-size-buffers=30 max-size-bytes=0 max-size-time=0 ! \ - videoconvert ! \ + videoconvert qos=false n-threads=4 ! \ fpsdisplaysink video-sink=$video_sink_element name=hailo_display sync=$sync_pipeline text-overlay=false ${additional_parameters}" echo "Running $network_name" diff --git a/apps/h8/gstreamer/x86_hw_accelerated/multistream_detection/multistream_detection.sh b/apps/h8/gstreamer/x86_hw_accelerated/multistream_detection/multistream_detection.sh index 270b1e1..0b75e2a 100755 --- a/apps/h8/gstreamer/x86_hw_accelerated/multistream_detection/multistream_detection.sh +++ b/apps/h8/gstreamer/x86_hw_accelerated/multistream_detection/multistream_detection.sh @@ -112,7 +112,7 @@ function main() { create_sources pipeline="gst-launch-1.0 \ - hailoroundrobin name=fun ! \ + hailoroundrobin mode=1 name=fun ! \ queue name=hailo_pre_infer_q_0 leaky=no max-size-buffers=30 max-size-bytes=0 max-size-time=0 ! \ hailonet hef-path=$HEF_PATH ! \ queue name=hailo_postprocess0 leaky=no max-size-buffers=30 max-size-bytes=0 max-size-time=0 ! \ diff --git a/apps/h8/gstreamer/x86_hw_accelerated/video_management_system/video_management_system_base.sh b/apps/h8/gstreamer/x86_hw_accelerated/video_management_system/video_management_system_base.sh index 7e0f793..fa5eb78 100755 --- a/apps/h8/gstreamer/x86_hw_accelerated/video_management_system/video_management_system_base.sh +++ b/apps/h8/gstreamer/x86_hw_accelerated/video_management_system/video_management_system_base.sh @@ -412,7 +412,7 @@ function create_pipeline_structure() { pipeline="gst-launch-1.0 \ $sources \ - hailoroundrobin name=roundrobin funnel-mode=false ! \ + hailoroundrobin name=roundrobin mode=1 ! \ queue leaky=no name=pre_detector_pipe_q max-size-buffers=30 max-size-bytes=0 max-size-time=0 ! \ $DETECTOR_PIPELINE ! \ queue leaky=no max-size-buffers=30 max-size-bytes=0 max-size-time=0 ! \ diff --git a/apps/h8/native/detection/detection_app.cpp b/apps/h8/native/detection/detection_app.cpp index 6d0370d..f4eae88 100644 --- a/apps/h8/native/detection/detection_app.cpp +++ b/apps/h8/native/detection/detection_app.cpp @@ -4,7 +4,7 @@ **/ /** * @file detection_app_c_api.cpp - * @brief This example demonstrates running inference with virtual streams using the Hailort's C API on yolov5m + * @brief This example demonstrates running inference with virtual streams using the Hailort's C API on yolov5 **/ #include "detection_app.hpp" @@ -60,8 +60,6 @@ hailo_status post_processing_all(std::vector> &feat { auto status = HAILO_SUCCESS; - YoloParams *init_params = init(CONFIG_FILE, "yolov5"); - std::sort(features.begin(), features.end(), &FeatureData::sort_tensors_by_size); for (size_t i = 0; i < frames_count; i++) { @@ -72,7 +70,7 @@ hailo_status post_processing_all(std::vector> &feat roi->add_tensor(std::make_shared(reinterpret_cast(features[j]->m_buffers.get_read_buffer().data()), features[j]->m_vstream_info)); // Perform the actual postprocess - yolov5(roi, init_params); + yolov5(roi); for (auto &feature : features) { diff --git a/apps/h8/native/detection/detection_app.hpp b/apps/h8/native/detection/detection_app.hpp index be2a359..6d63031 100644 --- a/apps/h8/native/detection/detection_app.hpp +++ b/apps/h8/native/detection/detection_app.hpp @@ -4,12 +4,12 @@ **/ /** * @file detection_app_c_api.cpp - * @brief This example demonstrates running inference with virtual streams using the Hailort's C API on yolov5m + * @brief This example demonstrates running inference with virtual streams using the Hailort's C API on yolov5 **/ #include "hailo/hailort.h" #include "double_buffer.hpp" -#include "yolo_postprocess.hpp" +#include "yolo_hailortpp.hpp" #include "hailo_objects.hpp" #include "hailo_tensors.hpp" #include "hailomat.hpp" @@ -29,7 +29,6 @@ #define OUTPUT_COUNT (3) #define INPUT_FILES_COUNT (10) #define HEF_FILE ("yolov5m_wo_spp_60p.hef") -#define CONFIG_FILE ("yolov5.json") #define YOLOV5M_IMAGE_WIDTH 640 #define YOLOV5M_IMAGE_HEIGHT 640 #define MAX_BOXES 50 diff --git a/apps/h8/native/detection/meson.build b/apps/h8/native/detection/meson.build index eaac3ab..f49a0de 100644 --- a/apps/h8/native/detection/meson.build +++ b/apps/h8/native/detection/meson.build @@ -22,8 +22,7 @@ relative_tappas_workspace = '../../../..' overlay_file = relative_tappas_workspace+'/core/hailo/plugins/overlay/overlay.cpp' # yolo postprocess paths -yolo_post_file = relative_tappas_workspace+'/core/hailo/libs/postprocesses/detection/yolo_postprocess.cpp' -yolo_out_file = relative_tappas_workspace+'/core/hailo/libs/postprocesses/detection/yolo_output.cpp' +yolo_post_file = relative_tappas_workspace+'/core/hailo/libs/postprocesses/detection/yolo_hailortpp.cpp' # include directories rapidjson_inc = [include_directories(relative_tappas_workspace + '/core/open_source/rapidjson', is_system: true)] @@ -40,7 +39,7 @@ hailort_lib = compiler.find_library('hailort') executable('detection_app', - [overlay_file, 'detection_app.cpp', yolo_post_file, yolo_out_file], + [overlay_file, 'detection_app.cpp', yolo_post_file], gnu_symbol_visibility : 'default', dependencies : [opencv_dep, hailort_lib], include_directories: [rapidjson_inc, hailo_general_inc, postproccess_inc], diff --git a/check_system_requirements.sh b/check_system_requirements.sh index 4db1e75..e593b82 100755 --- a/check_system_requirements.sh +++ b/check_system_requirements.sh @@ -27,6 +27,15 @@ apt_reqs[13]='cmake' apt_reqs[14]='libgstreamer-plugins-base1.0-dev' apt_reqs[15]='libzmq3-dev' apt_reqs[16]='rsync' +apt_reqs[17]='git' +apt_reqs[18]='libgstreamer-plugins-bad1.0-dev' +apt_reqs[19]='gstreamer1.0-plugins-base' +apt_reqs[20]='gstreamer1.0-plugins-good' +apt_reqs[21]='gstreamer1.0-plugins-bad' +apt_reqs[22]='gstreamer1.0-libav' +apt_reqs[23]='gstreamer1.0-tools' +apt_reqs[24]='gstreamer1.0-x' + declare -a pkg_config_reqs @@ -80,7 +89,7 @@ function check_system_requirements__pkg_config(){ found="X" else found="V" - echo "$log_found Package $requrement found." >> $log_file + echo "$log_found Package $requirement found." >> $log_file fi echo -e "Pkg_config-Package\t$requirement\t$found\tRequired" >> $table_file done diff --git a/core/hailo/apps/hailo15/encoder_applications/apps_common.cpp b/core/hailo/apps/hailo15/encoder_applications/apps_common.cpp new file mode 100644 index 0000000..206d39c --- /dev/null +++ b/core/hailo/apps/hailo15/encoder_applications/apps_common.cpp @@ -0,0 +1,51 @@ + +#include +#include "apps_common.hpp" + +GstFlowReturn wait_for_end_of_pipeline(GstElement *pipeline) +{ + GstBus *bus; + GstMessage *msg; + GstFlowReturn ret = GST_FLOW_ERROR; + bus = gst_element_get_bus(pipeline); + // This function blocks until an error or EOS message is received. + msg = gst_bus_timed_pop_filtered(bus, GST_CLOCK_TIME_NONE, (GstMessageType)(GST_MESSAGE_ERROR | GST_MESSAGE_EOS)); + + if (msg != NULL) + { + GError *err; + gchar *debug_info; + switch (GST_MESSAGE_TYPE(msg)) + { + case GST_MESSAGE_ERROR: + { + gst_message_parse_error(msg, &err, &debug_info); + GST_ERROR("Error received from element %s: %s", GST_OBJECT_NAME(msg->src), err->message); + + std::string dinfo = debug_info ? std::string(debug_info) : "none"; + GST_ERROR("Debugging information : %s", dinfo.c_str()); + + g_clear_error(&err); + g_free(debug_info); + ret = GST_FLOW_ERROR; + break; + } + case GST_MESSAGE_EOS: + { + GST_INFO("End-Of-Stream reached"); + ret = GST_FLOW_OK; + break; + } + default: + { + // We should not reach here because we only asked for ERRORs and EOS + GST_WARNING("Unexpected message received %d", GST_MESSAGE_TYPE(msg)); + ret = GST_FLOW_ERROR; + break; + } + } + gst_message_unref(msg); + } + gst_object_unref(bus); + return ret; +} \ No newline at end of file diff --git a/core/hailo/apps/hailo15/encoder_applications/apps_common.hpp b/core/hailo/apps/hailo15/encoder_applications/apps_common.hpp new file mode 100644 index 0000000..d72a5b7 --- /dev/null +++ b/core/hailo/apps/hailo15/encoder_applications/apps_common.hpp @@ -0,0 +1,4 @@ +#pragma once +#include + +GstFlowReturn wait_for_end_of_pipeline(GstElement *pipeline); \ No newline at end of file diff --git a/core/hailo/apps/hailo15/encoder_applications/encoder_appsink_tee.cpp b/core/hailo/apps/hailo15/encoder_applications/encoder_appsink_tee.cpp new file mode 100644 index 0000000..6bb01b4 --- /dev/null +++ b/core/hailo/apps/hailo15/encoder_applications/encoder_appsink_tee.cpp @@ -0,0 +1,115 @@ +#include +#include +#include +#include +#include "apps_common.hpp" + +/** + * Appsink's propose_allocation callback - Adding an GST_VIDEO_META_API_TYPE allocation meta + * + * @param[in] appsink The appsink object. + * @param[in] appsink The allocation query. + * @param[in] callback_data user data. + * @return TRUE + * @note The adding of allocation meta is required to work with v4l2src without it copying each buffer. + */ +static gboolean appsink_propose_allocation(GstAppSink * appsink, GstQuery * query, gpointer callback_data) +{ + gst_query_add_allocation_meta(query, GST_VIDEO_META_API_TYPE, NULL); + return TRUE; +} + +/** + * Appsink's new_sample callback + * + * @param[in] appsink The appsink object. + * @param[in] callback_data user data. + * @return GST_FLOW_OK + * @note Example only - only mapping the buffer to a GstVideoFrame, than unmapping. + */ +static GstFlowReturn appsink_new_sample(GstAppSink * appsink, gpointer callback_data) +{ + GstSample *sample; + GstBuffer *buffer; + GstCaps * caps; + GstVideoInfo *info = gst_video_info_new(); + GstVideoFrame frame; + + sample = gst_app_sink_pull_sample(appsink); + buffer = gst_sample_get_buffer(sample); + caps = gst_sample_get_caps(sample); + gst_video_info_from_caps(info, caps); + gst_video_frame_map(&frame, info, buffer, GstMapFlags(GST_MAP_READ)); + + GST_INFO("Got buffer video map"); + // Do Logic + + gst_video_frame_unmap(&frame); + gst_video_info_free(info); + gst_sample_unref(sample); + + return GST_FLOW_OK; +} + +/** + * Create the gstreamer pipeline as string + * + * @return A string containing the gstreamer pipeline. + * @note prints the return value to the stdout. + */ +std::string create_pipeline_string() +{ + std::string pipeline = ""; + + pipeline = "v4l2src name=src_element num-buffers=500 device=/dev/video0 io-mode=mmap ! " + "video/x-raw,format=NV12,width=3840,height=2160, framerate=30/1 ! " + "tee name=t t. ! " + "queue leaky=no max-size-buffers=5 max-size-bytes=0 max-size-time=0 ! " + "hailoh265enc ! h265parse config-interval=-1 ! " + "video/x-h265,framerate=30/1 ! filesink location=test.hevc " + "t. ! queue leaky=no max-size-buffers=5 max-size-bytes=0 max-size-time=0 ! appsink wait-on-eos=false name=hailo_sink"; + + std::cout << "Pipeline:" << std::endl; + std::cout << "gst-launch-1.0 " << pipeline << std::endl; + + return pipeline; +} + +/** + * Set the Appsink callbacks + * + * @param[in] pipeline The pipeline as a GstElement. + * @note Sets the new_sample and propose_allocation callbacks, without callback user data (NULL). + */ +void set_callbacks(GstElement *pipeline) +{ + GstAppSinkCallbacks callbacks={NULL}; + + GstElement *appsink = gst_bin_get_by_name(GST_BIN(pipeline), "hailo_sink"); + callbacks.new_sample = appsink_new_sample; + callbacks.propose_allocation = appsink_propose_allocation; + + gst_app_sink_set_callbacks(GST_APP_SINK(appsink), &callbacks, NULL, NULL); +} + +int main(int argc, char *argv[]) +{ + std::string src_pipeline_string; + GstFlowReturn ret; + + gst_init(&argc, &argv); + + std::string pipeline_string = create_pipeline_string(); + GstElement *pipeline = gst_parse_launch(pipeline_string.c_str(), NULL); + set_callbacks(pipeline); + gst_element_set_state(pipeline, GST_STATE_PLAYING); + + ret = wait_for_end_of_pipeline(pipeline); + + // Free resources + gst_element_set_state(pipeline, GST_STATE_NULL); + gst_deinit(); + gst_object_unref(pipeline); + + return ret; +} diff --git a/core/hailo/apps/hailo15/encoder_applications/encoder_change_bitrate.cpp b/core/hailo/apps/hailo15/encoder_applications/encoder_change_bitrate.cpp new file mode 100644 index 0000000..633cc29 --- /dev/null +++ b/core/hailo/apps/hailo15/encoder_applications/encoder_change_bitrate.cpp @@ -0,0 +1,163 @@ +#include +#include +#include + +#include "apps_common.hpp" + +#define BITRATE_FOR_VBR (1000000) +#define TOL_MOVING_BITRATE_FOR_VBR (2000) +#define PICTURE_RC_OFF (0) +#define PICTURE_RC_ON (1) +#define BITRATE_FOR_CBR (40000000) +#define TOL_MOVING_BITRATE_FOR_CBR (0) + +static int counter=0; + +/** + * Encoder's probe callback + * + * @param[in] pad The sinkpad of the encoder. + * @param[in] info Info about the probe + * @param[in] user_data user specified data for the probe + * @return GST_PAD_PROBE_OK + * @note Example only - Switches between "CBR" and "VBR" every 200 frames, user_data is the pipeline. + */ +static GstPadProbeReturn encoder_probe_callback(GstPad *pad, GstPadProbeInfo *info, gpointer user_data) +{ + GstElement *pipeline = GST_ELEMENT(user_data); + GstElement *encoder_element = gst_bin_get_by_name(GST_BIN(pipeline), "enco"); + + counter++; + + if (counter % 200 == 0) { + if (counter % 400 != 0) { + // Changing to VBR + GST_INFO("Changing encoder to VBR"); + g_object_set(encoder_element, "tol-moving-bitrate", TOL_MOVING_BITRATE_FOR_VBR, NULL); + g_object_set(encoder_element, "bitrate", BITRATE_FOR_VBR, NULL); + g_object_set(encoder_element, "picture-rc", PICTURE_RC_OFF, NULL); + } + else + { + // Changing to CBR + GST_INFO("Changing encoder to CBR"); + g_object_set(encoder_element, "tol-moving-bitrate", TOL_MOVING_BITRATE_FOR_CBR, NULL); + g_object_set(encoder_element, "bitrate", BITRATE_FOR_CBR, NULL); + g_object_set(encoder_element, "picture-rc", PICTURE_RC_ON, NULL); + } + } + + return GST_PAD_PROBE_OK; +} + +/** + * Appsink's new_sample callback + * + * @param[in] appsink The appsink object. + * @param[in] callback_data user data. + * @return GST_FLOW_OK + * @note Example only - only mapping the buffer to a GstMapInfo, than unmapping. + */ +static GstFlowReturn appsink_new_sample(GstAppSink * appsink, gpointer callback_data) +{ + GstSample *sample; + GstBuffer *buffer; + GstMapInfo mapinfo; + + sample = gst_app_sink_pull_sample(appsink); + buffer = gst_sample_get_buffer(sample); + gst_buffer_map(buffer, &mapinfo, GST_MAP_READ); + + GST_INFO_OBJECT(appsink, "Got Buffer from appsink: %p", mapinfo.data); + // Do Logic + + gst_buffer_unmap(buffer,&mapinfo); + gst_sample_unref(sample); + + return GST_FLOW_OK; +} + +/** + * Create the gstreamer pipeline as string + * + * @return A string containing the gstreamer pipeline. + * @note prints the return value to the stdout. + */ +std::string create_pipeline_string() +{ + std::string pipeline = ""; + std::string encoder_arguments; + + encoder_arguments = "tol-moving-bitrate=" + std::to_string(TOL_MOVING_BITRATE_FOR_CBR) + " " + "picture-rc=" + std::to_string(PICTURE_RC_ON) + " " + "bitrate=" + std::to_string(BITRATE_FOR_CBR) + " " + "ctb-rc=1"; + + + pipeline = "v4l2src name=src_element num-buffers=900 device=/dev/video0 io-mode=mmap ! " + "video/x-raw,format=NV12,width=1920,height=1080, framerate=30/1 ! " + "queue leaky=no max-size-buffers=5 max-size-bytes=0 max-size-time=0 ! " + "hailoh265enc name=enco " + encoder_arguments + " ! h265parse config-interval=-1 ! " + "queue leaky=no max-size-buffers=5 max-size-bytes=0 max-size-time=0 ! " + "video/x-h265,framerate=30/1 ! appsink wait-on-eos=false name=hailo_sink"; + + std::cout << "Pipeline:" << std::endl; + std::cout << "gst-launch-1.0 " << pipeline << std::endl; + + return pipeline; +} + +/** + * Set the Appsink callbacks + * + * @param[in] pipeline The pipeline as a GstElement. + * @note Sets the new_sample and propose_allocation callbacks, without callback user data (NULL). + */ +void set_callbacks(GstElement *pipeline) +{ + GstAppSinkCallbacks callbacks={NULL}; + + GstElement *appsink = gst_bin_get_by_name(GST_BIN(pipeline), "hailo_sink"); + callbacks.new_sample = appsink_new_sample; + + gst_app_sink_set_callbacks(GST_APP_SINK(appsink), &callbacks, NULL, NULL); +} + +/** + * Set the pipeline's probes + * + * @param[in] pipeline The pipeline as a GstElement. + * @note Sets a probe to the encoder sinkpad. + */ +void set_probes(GstElement *pipeline) +{ + // extract elements from pipeline + GstElement *encoder = gst_bin_get_by_name(GST_BIN(pipeline), "enco"); + // extract pads from elements + GstPad *pad_encoder = gst_element_get_static_pad(encoder, "sink"); + // set probes + gst_pad_add_probe(pad_encoder, GST_PAD_PROBE_TYPE_BUFFER, (GstPadProbeCallback)encoder_probe_callback, pipeline, NULL); +} + +int main(int argc, char *argv[]) +{ + std::string src_pipeline_string; + GstFlowReturn ret; + + gst_init(&argc, &argv); + + std::string pipeline_string = create_pipeline_string(); + GstElement *pipeline = gst_parse_launch(pipeline_string.c_str(), NULL); + set_callbacks(pipeline); + set_probes(pipeline); + gst_element_set_state(pipeline, GST_STATE_PLAYING); + + ret = wait_for_end_of_pipeline(pipeline); + + // Free resources + gst_element_set_state(pipeline, GST_STATE_NULL); + gst_deinit(); + gst_object_unref(pipeline); + + return ret; +} diff --git a/core/hailo/apps/hailo15/encoder_applications/encoder_change_framerate.cpp b/core/hailo/apps/hailo15/encoder_applications/encoder_change_framerate.cpp new file mode 100644 index 0000000..761ce97 --- /dev/null +++ b/core/hailo/apps/hailo15/encoder_applications/encoder_change_framerate.cpp @@ -0,0 +1,170 @@ +#include +#include +#include + +#include "apps_common.hpp" + +#define BIG_FRAMERATE (30) +#define SMALL_FRAMERATE (10) + +static int counter=0; + +/** + * Update the framerate of the pipeline + * + * @param[in] pipeline The pipeline as a GstElement. + * @param[in] framerate The new framerate. + * @note updates the framerate of the pipeline. + */ +static void update_framerate(GstElement *pipeline, int framerate) +{ + GstElement *videofilter = gst_bin_get_by_name(GST_BIN(pipeline), "videofilter"); + GstCaps *filtercaps; + + g_object_get(G_OBJECT(videofilter), "caps", &filtercaps, NULL); + filtercaps = gst_caps_make_writable(filtercaps); + gst_caps_set_simple(filtercaps, "framerate", GST_TYPE_FRACTION, framerate, 1, NULL); + g_object_set(G_OBJECT(videofilter), "caps", filtercaps, NULL); + gst_caps_unref(filtercaps); +} + +/** + * Update the framerate of the pipeline every 200 frames + * + * @param[in] pad The sinkpad of the encoder. + * @param[in] info Info about the probe + * @param[in] user_data user specified data for the probe + * @return GST_PAD_PROBE_OK + * @note This function is called every frame, and updates the framerate every 200 frames. + */ +static GstPadProbeReturn update_framerate_probe_callback(GstPad *pad, GstPadProbeInfo *info, gpointer user_data) +{ + GstElement *pipeline = GST_ELEMENT(user_data); + counter++; + + if (counter % 200 == 0) + { + if (counter % 400 == 0) + { + GST_INFO("Changing pipeline to %d fps", BIG_FRAMERATE); + update_framerate(pipeline, BIG_FRAMERATE); + } + else + { + GST_INFO("Changing pipeline to %d fps", SMALL_FRAMERATE); + update_framerate(pipeline, SMALL_FRAMERATE); + } + } + + return GST_PAD_PROBE_OK; +} + +/** + * Appsink's new_sample callback + * + * @param[in] appsink The appsink object. + * @param[in] callback_data user data. + * @return GST_FLOW_OK + * @note Example only - only mapping the buffer to a GstMapInfo, than unmapping. + */ +static GstFlowReturn appsink_new_sample(GstAppSink * appsink, gpointer callback_data) +{ + GstSample *sample; + GstBuffer *buffer; + GstMapInfo mapinfo; + + sample = gst_app_sink_pull_sample(appsink); + buffer = gst_sample_get_buffer(sample); + gst_buffer_map(buffer, &mapinfo, GST_MAP_READ); + + GST_INFO_OBJECT(appsink, "Got Buffer from appsink: %p", mapinfo.data); + // Do Logic + + gst_buffer_unmap(buffer,&mapinfo); + gst_sample_unref(sample); + + return GST_FLOW_OK; +} + +/** + * Create the gstreamer pipeline as string + * + * @return A string containing the gstreamer pipeline. + * @note prints the return value to the stdout. + */ +std::string create_pipeline_string() +{ + std::string pipeline = ""; + + pipeline = "v4l2src name=src_element num-buffers=2000 device=/dev/video0 io-mode=mmap ! " + "video/x-raw,format=NV12,width=1920,height=1080,framerate=30/1 ! " + "videorate name=videorate ! capsfilter name=videofilter caps=video/x-raw,framerate=30/1 ! " + "queue leaky=no max-size-buffers=5 max-size-bytes=0 max-size-time=0 ! " + "hailoh265enc name=enco ! h265parse config-interval=-1 ! tee name=t t. ! " + "queue leaky=no max-size-buffers=5 max-size-bytes=0 max-size-time=0 ! " + "fpsdisplaysink name=display_sink text-overlay=false video-sink=\"appsink name=hailo_sink\" sync=true t. ! " + "queue leaky=no max-size-buffers=5 max-size-bytes=0 max-size-time=0 ! " + "rtph265pay ! application/x-rtp, media=(string)video, encoding-name=(string)H265 ! " + "udpsink host=10.0.0.2 port=5000 sync=true name=udp_sink"; + + std::cout << "Pipeline:" << std::endl; + std::cout << "gst-launch-1.0 " << pipeline << std::endl; + + return pipeline; +} + +/** + * Set the Appsink callbacks + * + * @param[in] pipeline The pipeline as a GstElement. + * @note Sets the new_sample and propose_allocation callbacks, without callback user data (NULL). + */ +void set_callbacks(GstElement *pipeline) +{ + GstAppSinkCallbacks callbacks={NULL}; + + GstElement *display_sink = gst_bin_get_by_name(GST_BIN(pipeline), "display_sink"); + GstElement *appsink = gst_bin_get_by_name(GST_BIN(display_sink), "hailo_sink"); + callbacks.new_sample = appsink_new_sample; + + gst_app_sink_set_callbacks(GST_APP_SINK(appsink), &callbacks, NULL, NULL); +} + +/** + * Set the pipeline's probes + * + * @param[in] pipeline The pipeline as a GstElement. + * @note Sets a probe to the encoder sinkpad. + */ +void set_probes(GstElement *pipeline) +{ + // extract elements from pipeline + GstElement *encoder = gst_bin_get_by_name(GST_BIN(pipeline), "videorate"); + // extract pads from elements + GstPad *pad_encoder = gst_element_get_static_pad(encoder, "sink"); + // set probes + gst_pad_add_probe(pad_encoder, GST_PAD_PROBE_TYPE_BUFFER, (GstPadProbeCallback)update_framerate_probe_callback, pipeline, NULL); +} + +int main(int argc, char *argv[]) +{ + std::string src_pipeline_string; + GstFlowReturn ret; + + gst_init(&argc, &argv); + + std::string pipeline_string = create_pipeline_string(); + GstElement *pipeline = gst_parse_launch(pipeline_string.c_str(), NULL); + set_callbacks(pipeline); + set_probes(pipeline); + gst_element_set_state(pipeline, GST_STATE_PLAYING); + + ret = wait_for_end_of_pipeline(pipeline); + + // Free resources + gst_element_set_state(pipeline, GST_STATE_NULL); + gst_deinit(); + gst_object_unref(pipeline); + + return ret; +} diff --git a/core/hailo/apps/hailo15/encoder_applications/encoder_change_gop.cpp b/core/hailo/apps/hailo15/encoder_applications/encoder_change_gop.cpp new file mode 100644 index 0000000..621b9ca --- /dev/null +++ b/core/hailo/apps/hailo15/encoder_applications/encoder_change_gop.cpp @@ -0,0 +1,162 @@ +#include +#include +#include + +#include "apps_common.hpp" + +#define BIG_GOP (150) +#define MEDIUM_GOP (30) +#define SMALL_GOP (5) + +static int counter=0; + +/** + * Encoder's probe callback + * + * @param[in] pad The sinkpad of the encoder. + * @param[in] info Info about the probe + * @param[in] user_data user specified data for the probe + * @return GST_PAD_PROBE_OK + * @note Example only - Switches between GOP length's in 600 frame cycle: + * 300 frames in gop 150 + * 200 frames in gop 30 + * 100 frames in gop 5 + */ +static GstPadProbeReturn encoder_probe_callback(GstPad *pad, GstPadProbeInfo *info, gpointer user_data) +{ + GstElement *pipeline = GST_ELEMENT(user_data); + GstElement *encoder_element = gst_bin_get_by_name(GST_BIN(pipeline), "enco"); + + counter++; + + if (counter % 600 == 0) { + // Changing to big GOP + GST_INFO("Changing encoder to GOP %d", BIG_GOP); + g_object_set(encoder_element, "intra-pic-rate", BIG_GOP, NULL); + g_object_set(encoder_element, "gop-length", BIG_GOP, NULL); + } + else if (counter % 600 == 300) { + // Changing to MEDIUM_GOP + GST_INFO("Changing encoder to GOP %d", MEDIUM_GOP); + g_object_set(encoder_element, "intra-pic-rate", MEDIUM_GOP, NULL); + g_object_set(encoder_element, "gop-length", MEDIUM_GOP, NULL); + } + else if (counter % 600 == 500) { + // Changing to SMALL_GOP + GST_INFO("Changing encoder to GOP %d", SMALL_GOP); + g_object_set(encoder_element, "intra-pic-rate", SMALL_GOP, NULL); + g_object_set(encoder_element, "gop-length", SMALL_GOP, NULL); + } + + return GST_PAD_PROBE_OK; +} + +/** + * Appsink's new_sample callback + * + * @param[in] appsink The appsink object. + * @param[in] callback_data user data. + * @return GST_FLOW_OK + * @note Example only - only mapping the buffer to a GstMapInfo, than unmapping. + */ +static GstFlowReturn appsink_new_sample(GstAppSink * appsink, gpointer callback_data) +{ + GstSample *sample; + GstBuffer *buffer; + GstMapInfo mapinfo; + + sample = gst_app_sink_pull_sample(appsink); + buffer = gst_sample_get_buffer(sample); + gst_buffer_map(buffer, &mapinfo, GST_MAP_READ); + + GST_INFO_OBJECT(appsink, "Got Buffer from appsink: %p", mapinfo.data); + // Do Logic + + gst_buffer_unmap(buffer,&mapinfo); + gst_sample_unref(sample); + + return GST_FLOW_OK; +} + +/** + * Create the gstreamer pipeline as string + * + * @return A string containing the gstreamer pipeline. + * @note prints the return value to the stdout. + */ +std::string create_pipeline_string() +{ + std::string pipeline = ""; + std::string encoder_arguments; + + encoder_arguments = "intra-pic-rate=" + std::to_string(BIG_GOP) + " " + "gop-length=" + std::to_string(BIG_GOP); + + + pipeline = "v4l2src name=src_element num-buffers=2000 device=/dev/video0 io-mode=mmap ! " + "video/x-raw,format=NV12,width=1920,height=1080, framerate=30/1 ! " + "queue leaky=no max-size-buffers=5 max-size-bytes=0 max-size-time=0 ! " + "hailoh265enc name=enco " + encoder_arguments + " ! h265parse config-interval=-1 ! " + "queue leaky=no max-size-buffers=5 max-size-bytes=0 max-size-time=0 ! " + "video/x-h265,framerate=30/1 ! appsink wait-on-eos=false name=hailo_sink"; + + std::cout << "Pipeline:" << std::endl; + std::cout << "gst-launch-1.0 " << pipeline << std::endl; + + return pipeline; +} + +/** + * Set the Appsink callbacks + * + * @param[in] pipeline The pipeline as a GstElement. + * @note Sets the new_sample and propose_allocation callbacks, without callback user data (NULL). + */ +void set_callbacks(GstElement *pipeline) +{ + GstAppSinkCallbacks callbacks={NULL}; + + GstElement *appsink = gst_bin_get_by_name(GST_BIN(pipeline), "hailo_sink"); + callbacks.new_sample = appsink_new_sample; + + gst_app_sink_set_callbacks(GST_APP_SINK(appsink), &callbacks, NULL, NULL); +} + +/** + * Set the pipeline's probes + * + * @param[in] pipeline The pipeline as a GstElement. + * @note Sets a probe to the encoder sinkpad. + */ +void set_probes(GstElement *pipeline) +{ + // extract elements from pipeline + GstElement *encoder = gst_bin_get_by_name(GST_BIN(pipeline), "enco"); + // extract pads from elements + GstPad *pad_encoder = gst_element_get_static_pad(encoder, "sink"); + // set probes + gst_pad_add_probe(pad_encoder, GST_PAD_PROBE_TYPE_BUFFER, (GstPadProbeCallback)encoder_probe_callback, pipeline, NULL); +} + +int main(int argc, char *argv[]) +{ + std::string src_pipeline_string; + GstFlowReturn ret; + + gst_init(&argc, &argv); + + std::string pipeline_string = create_pipeline_string(); + GstElement *pipeline = gst_parse_launch(pipeline_string.c_str(), NULL); + set_callbacks(pipeline); + set_probes(pipeline); + gst_element_set_state(pipeline, GST_STATE_PLAYING); + + ret = wait_for_end_of_pipeline(pipeline); + + // Free resources + gst_element_set_state(pipeline, GST_STATE_NULL); + gst_deinit(); + gst_object_unref(pipeline); + + return ret; +} diff --git a/core/hailo/apps/hailo15/encoder_applications/encoder_change_gop_size.cpp b/core/hailo/apps/hailo15/encoder_applications/encoder_change_gop_size.cpp new file mode 100644 index 0000000..29593ef --- /dev/null +++ b/core/hailo/apps/hailo15/encoder_applications/encoder_change_gop_size.cpp @@ -0,0 +1,158 @@ +#include +#include +#include + +#include "apps_common.hpp" + +#define BIG_GOP_SIZE (7) +#define MEDIUM_GOP_SIZE (4) +#define SMALL_GOP_SIZE (1) + +static int counter=0; + +/** + * Encoder's probe callback + * + * @param[in] pad The sinkpad of the encoder. + * @param[in] info Info about the probe + * @param[in] user_data user specified data for the probe + * @return GST_PAD_PROBE_OK + * @note Example only - Switches between GOP size's (B-Frames) in 600 frame cycle: + * 300 frames in gop 7 + * 200 frames in gop 4 + * 100 frames in gop 1 + */ +static GstPadProbeReturn encoder_probe_callback(GstPad *pad, GstPadProbeInfo *info, gpointer user_data) +{ + GstElement *pipeline = GST_ELEMENT(user_data); + GstElement *encoder_element = gst_bin_get_by_name(GST_BIN(pipeline), "enco"); + + counter++; + + if (counter % 600 == 0) { + // Changing to BIG_GOP_SIZE + GST_INFO("Changing encoder to GOP %d", BIG_GOP_SIZE); + g_object_set(encoder_element, "gop-size", BIG_GOP_SIZE, NULL); + } + else if (counter % 600 == 300) { + // Changing to MEDIUM_GOP_SIZE + GST_INFO("Changing encoder to GOP %d", MEDIUM_GOP_SIZE); + g_object_set(encoder_element, "gop-size", MEDIUM_GOP_SIZE, NULL); + } + else if (counter % 600 == 500) { + // Changing to SMALL_GOP_SIZE + GST_INFO("Changing encoder to GOP %d", SMALL_GOP_SIZE); + g_object_set(encoder_element, "gop-size", SMALL_GOP_SIZE, NULL); + } + + return GST_PAD_PROBE_OK; +} + +/** + * Appsink's new_sample callback + * + * @param[in] appsink The appsink object. + * @param[in] callback_data user data. + * @return GST_FLOW_OK + * @note Example only - only mapping the buffer to a GstMapInfo, than unmapping. + */ +static GstFlowReturn appsink_new_sample(GstAppSink * appsink, gpointer callback_data) +{ + GstSample *sample; + GstBuffer *buffer; + GstMapInfo mapinfo; + + sample = gst_app_sink_pull_sample(appsink); + buffer = gst_sample_get_buffer(sample); + gst_buffer_map(buffer, &mapinfo, GST_MAP_READ); + + GST_INFO_OBJECT(appsink, "Got Buffer from appsink: %p", mapinfo.data); + // Do Logic + + gst_buffer_unmap(buffer,&mapinfo); + gst_sample_unref(sample); + + return GST_FLOW_OK; +} + +/** + * Create the gstreamer pipeline as string + * + * @return A string containing the gstreamer pipeline. + * @note prints the return value to the stdout. + */ +std::string create_pipeline_string() +{ + std::string pipeline = ""; + std::string encoder_arguments; + + encoder_arguments = "gop-size=" + std::to_string(BIG_GOP_SIZE); + + + pipeline = "v4l2src name=src_element num-buffers=2000 device=/dev/video0 io-mode=mmap ! " + "video/x-raw,format=NV12,width=1920,height=1080, framerate=30/1 ! " + "queue leaky=no max-size-buffers=5 max-size-bytes=0 max-size-time=0 ! " + "hailoh265enc name=enco " + encoder_arguments + " ! h265parse config-interval=-1 ! " + "queue leaky=no max-size-buffers=5 max-size-bytes=0 max-size-time=0 ! " + "video/x-h265,framerate=30/1 ! appsink wait-on-eos=false name=hailo_sink"; + + std::cout << "Pipeline:" << std::endl; + std::cout << "gst-launch-1.0 " << pipeline << std::endl; + + return pipeline; +} + +/** + * Set the Appsink callbacks + * + * @param[in] pipeline The pipeline as a GstElement. + * @note Sets the new_sample and propose_allocation callbacks, without callback user data (NULL). + */ +void set_callbacks(GstElement *pipeline) +{ + GstAppSinkCallbacks callbacks={NULL}; + + GstElement *appsink = gst_bin_get_by_name(GST_BIN(pipeline), "hailo_sink"); + callbacks.new_sample = appsink_new_sample; + + gst_app_sink_set_callbacks(GST_APP_SINK(appsink), &callbacks, NULL, NULL); +} + +/** + * Set the pipeline's probes + * + * @param[in] pipeline The pipeline as a GstElement. + * @note Sets a probe to the encoder sinkpad. + */ +void set_probes(GstElement *pipeline) +{ + // extract elements from pipeline + GstElement *encoder = gst_bin_get_by_name(GST_BIN(pipeline), "enco"); + // extract pads from elements + GstPad *pad_encoder = gst_element_get_static_pad(encoder, "sink"); + // set probes + gst_pad_add_probe(pad_encoder, GST_PAD_PROBE_TYPE_BUFFER, (GstPadProbeCallback)encoder_probe_callback, pipeline, NULL); +} + +int main(int argc, char *argv[]) +{ + std::string src_pipeline_string; + GstFlowReturn ret; + + gst_init(&argc, &argv); + + std::string pipeline_string = create_pipeline_string(); + GstElement *pipeline = gst_parse_launch(pipeline_string.c_str(), NULL); + set_callbacks(pipeline); + set_probes(pipeline); + gst_element_set_state(pipeline, GST_STATE_PLAYING); + + ret = wait_for_end_of_pipeline(pipeline); + + // Free resources + gst_element_set_state(pipeline, GST_STATE_NULL); + gst_deinit(); + gst_object_unref(pipeline); + + return ret; +} diff --git a/core/hailo/apps/hailo15/encoder_applications/encoder_change_qp.cpp b/core/hailo/apps/hailo15/encoder_applications/encoder_change_qp.cpp new file mode 100644 index 0000000..88f6846 --- /dev/null +++ b/core/hailo/apps/hailo15/encoder_applications/encoder_change_qp.cpp @@ -0,0 +1,158 @@ +#include +#include +#include + +#include "apps_common.hpp" + +static int counter=0; + +/** + * Encoder's probe callback + * + * @param[in] pad The sinkpad of the encoder. + * @param[in] info Info about the probe + * @param[in] user_data user specified data for the probe + * @return GST_PAD_PROBE_OK + * @note Example only - Switches qp params every 200 frames, user_data is the pipeline. + */ +static GstPadProbeReturn encoder_probe_callback(GstPad *pad, GstPadProbeInfo *info, gpointer user_data) +{ + GstElement *pipeline = GST_ELEMENT(user_data); + GstElement *encoder_element = gst_bin_get_by_name(GST_BIN(pipeline), "enco"); + + counter++; + + if (counter % 600 == 200) { + // Changing to low qp + GST_INFO("Changing to low qp"); + // Changing qp-hdr first to avoid qp-hdr < qp-min. + g_object_set(encoder_element, "qp-hdr", 5, NULL); + g_object_set(encoder_element, "qp-min", 3, NULL); + g_object_set(encoder_element, "qp-max", 10, NULL); + } + else if (counter % 600 == 400) { + // Changing to high qp + GST_INFO("Changing to high qp"); + // Changing qp-max first to avoid qp-max < qp-hdr and qp-max < qp-min. + g_object_set(encoder_element, "qp-max", 49, NULL); + g_object_set(encoder_element, "qp-hdr", 45, NULL); + g_object_set(encoder_element, "qp-min", 43, NULL); + } + else if (counter % 600 == 0) { + // Changing to variant qp (default) + GST_INFO("Changing to variant qp"); + g_object_set(encoder_element, "qp-min", 0, NULL); + g_object_set(encoder_element, "qp-max", 51, NULL); + g_object_set(encoder_element, "qp-hdr", 26, NULL); + } + + return GST_PAD_PROBE_OK; +} + +/** + * Appsink's new_sample callback + * + * @param[in] appsink The appsink object. + * @param[in] callback_data user data. + * @return GST_FLOW_OK + * @note Example only - only mapping the buffer to a GstMapInfo, than unmapping. + */ +static GstFlowReturn appsink_new_sample(GstAppSink * appsink, gpointer callback_data) +{ + GstSample *sample; + GstBuffer *buffer; + GstMapInfo mapinfo; + + sample = gst_app_sink_pull_sample(appsink); + buffer = gst_sample_get_buffer(sample); + gst_buffer_map(buffer, &mapinfo, GST_MAP_READ); + + GST_INFO_OBJECT(appsink, "Got Buffer from appsink: %p", mapinfo.data); + // Do Logic + + gst_buffer_unmap(buffer,&mapinfo); + gst_sample_unref(sample); + + return GST_FLOW_OK; +} + +/** + * Create the gstreamer pipeline as string + * + * @return A string containing the gstreamer pipeline. + * @note prints the return value to the stdout. + */ +std::string create_pipeline_string() +{ + std::string pipeline = ""; + std::string encoder_arguments; + + + + pipeline = "v4l2src name=src_element num-buffers=2000 device=/dev/video0 io-mode=mmap ! " + "video/x-raw,format=NV12,width=1920,height=1080, framerate=30/1 ! " + "queue leaky=no max-size-buffers=5 max-size-bytes=0 max-size-time=0 ! " + "hailoh265enc name=enco ! h265parse config-interval=-1 ! " + "queue leaky=no max-size-buffers=5 max-size-bytes=0 max-size-time=0 ! " + "video/x-h265,framerate=30/1 ! appsink wait-on-eos=false name=hailo_sink"; + + std::cout << "Pipeline:" << std::endl; + std::cout << "gst-launch-1.0 " << pipeline << std::endl; + + return pipeline; +} + +/** + * Set the Appsink callbacks + * + * @param[in] pipeline The pipeline as a GstElement. + * @note Sets the new_sample and propose_allocation callbacks, without callback user data (NULL). + */ +void set_callbacks(GstElement *pipeline) +{ + GstAppSinkCallbacks callbacks={NULL}; + + GstElement *appsink = gst_bin_get_by_name(GST_BIN(pipeline), "hailo_sink"); + callbacks.new_sample = appsink_new_sample; + + gst_app_sink_set_callbacks(GST_APP_SINK(appsink), &callbacks, NULL, NULL); +} + +/** + * Set the pipeline's probes + * + * @param[in] pipeline The pipeline as a GstElement. + * @note Sets a probe to the encoder sinkpad. + */ +void set_probes(GstElement *pipeline) +{ + // extract elements from pipeline + GstElement *encoder = gst_bin_get_by_name(GST_BIN(pipeline), "enco"); + // extract pads from elements + GstPad *pad_encoder = gst_element_get_static_pad(encoder, "sink"); + // set probes + gst_pad_add_probe(pad_encoder, GST_PAD_PROBE_TYPE_BUFFER, (GstPadProbeCallback)encoder_probe_callback, pipeline, NULL); +} + +int main(int argc, char *argv[]) +{ + std::string src_pipeline_string; + GstFlowReturn ret; + + gst_init(&argc, &argv); + + std::string pipeline_string = create_pipeline_string(); + GstElement *pipeline = gst_parse_launch(pipeline_string.c_str(), NULL); + set_callbacks(pipeline); + set_probes(pipeline); + gst_element_set_state(pipeline, GST_STATE_PLAYING); + + ret = wait_for_end_of_pipeline(pipeline); + + // Free resources + gst_element_set_state(pipeline, GST_STATE_NULL); + gst_deinit(); + gst_object_unref(pipeline); + + return ret; +} diff --git a/core/hailo/apps/hailo15/encoder_applications/encoder_change_roi.cpp b/core/hailo/apps/hailo15/encoder_applications/encoder_change_roi.cpp new file mode 100644 index 0000000..cb245c7 --- /dev/null +++ b/core/hailo/apps/hailo15/encoder_applications/encoder_change_roi.cpp @@ -0,0 +1,140 @@ +#include +#include +#include + +#include "apps_common.hpp" + +static int counter=0; + +/** + * Encoder's probe callback + * + * @param[in] pad The sinkpad of the encoder. + * @param[in] info Info about the probe + * @param[in] user_data user specified data for the probe + * @return GST_PAD_PROBE_OK + * @note Example only - Switches ROI area every 200 frames, user_data is the pipeline. + */ +static GstPadProbeReturn encoder_probe_callback(GstPad *pad, GstPadProbeInfo *info, gpointer user_data) +{ + GstElement *pipeline = GST_ELEMENT(user_data); + GstElement *encoder_element = gst_bin_get_by_name(GST_BIN(pipeline), "enco"); + + if (counter % 400 == 200) { + GST_INFO("Changing ROI to (200,200,500,500,20)"); + g_object_set(encoder_element, "roi-area1", "200:200:500:500:20", NULL); + } + else if (counter % 400 == 0) { + GST_INFO("Changing ROI to (700,700,900,900,5)"); + g_object_set(encoder_element, "roi-area1", "700:700:900:900:5", NULL); + } + counter++; + + return GST_PAD_PROBE_OK; +} + +/** + * Appsink's new_sample callback + * + * @param[in] appsink The appsink object. + * @param[in] callback_data user data. + * @return GST_FLOW_OK + * @note Example only - only mapping the buffer to a GstMapInfo, than unmapping. + */ +static GstFlowReturn appsink_new_sample(GstAppSink * appsink, gpointer callback_data) +{ + GstSample *sample; + GstBuffer *buffer; + GstMapInfo mapinfo; + + sample = gst_app_sink_pull_sample(appsink); + buffer = gst_sample_get_buffer(sample); + gst_buffer_map(buffer, &mapinfo, GST_MAP_READ); + + GST_INFO_OBJECT(appsink, "Got Buffer from appsink: %p", mapinfo.data); + // Do Logic + + gst_buffer_unmap(buffer,&mapinfo); + gst_sample_unref(sample); + + return GST_FLOW_OK; +} + +/** + * Create the gstreamer pipeline as string + * + * @return A string containing the gstreamer pipeline. + * @note prints the return value to the stdout. + */ +std::string create_pipeline_string() +{ + std::string pipeline = ""; + std::string encoder_arguments; + + pipeline = "v4l2src name=src_element num-buffers=2000 device=/dev/video0 io-mode=mmap ! " + "video/x-raw,format=NV12,width=1920,height=1080, framerate=30/1 ! " + "queue leaky=no max-size-buffers=5 max-size-bytes=0 max-size-time=0 ! " + "hailoh265enc name=enco ! h265parse config-interval=-1 ! " + "queue leaky=no max-size-buffers=5 max-size-bytes=0 max-size-time=0 ! " + "video/x-h265,framerate=30/1 ! appsink wait-on-eos=false name=hailo_sink"; + + std::cout << "Pipeline:" << std::endl; + std::cout << "gst-launch-1.0 " << pipeline << std::endl; + + return pipeline; +} + +/** + * Set the Appsink callbacks + * + * @param[in] pipeline The pipeline as a GstElement. + * @note Sets the new_sample and propose_allocation callbacks, without callback user data (NULL). + */ +void set_callbacks(GstElement *pipeline) +{ + GstAppSinkCallbacks callbacks={NULL}; + + GstElement *appsink = gst_bin_get_by_name(GST_BIN(pipeline), "hailo_sink"); + callbacks.new_sample = appsink_new_sample; + + gst_app_sink_set_callbacks(GST_APP_SINK(appsink), &callbacks, NULL, NULL); +} + +/** + * Set the pipeline's probes + * + * @param[in] pipeline The pipeline as a GstElement. + * @note Sets a probe to the encoder sinkpad. + */ +void set_probes(GstElement *pipeline) +{ + // extract elements from pipeline + GstElement *encoder = gst_bin_get_by_name(GST_BIN(pipeline), "enco"); + // extract pads from elements + GstPad *pad_encoder = gst_element_get_static_pad(encoder, "sink"); + // set probes + gst_pad_add_probe(pad_encoder, GST_PAD_PROBE_TYPE_BUFFER, (GstPadProbeCallback)encoder_probe_callback, pipeline, NULL); +} + +int main(int argc, char *argv[]) +{ + std::string src_pipeline_string; + GstFlowReturn ret; + + gst_init(&argc, &argv); + + std::string pipeline_string = create_pipeline_string(); + GstElement *pipeline = gst_parse_launch(pipeline_string.c_str(), NULL); + set_callbacks(pipeline); + set_probes(pipeline); + gst_element_set_state(pipeline, GST_STATE_PLAYING); + + ret = wait_for_end_of_pipeline(pipeline); + + // Free resources + gst_element_set_state(pipeline, GST_STATE_NULL); + gst_deinit(); + gst_object_unref(pipeline); + + return ret; +} diff --git a/core/hailo/apps/hailo15/encoder_applications/encoder_force_keyframe.cpp b/core/hailo/apps/hailo15/encoder_applications/encoder_force_keyframe.cpp new file mode 100644 index 0000000..b7d2603 --- /dev/null +++ b/core/hailo/apps/hailo15/encoder_applications/encoder_force_keyframe.cpp @@ -0,0 +1,96 @@ +#include +#include +#include + +#include "apps_common.hpp" + +static int counter=0; + +/** + * Encoder's probe callback + * + * @param[in] pad The sinkpad of the encoder. + * @param[in] info Info about the probe + * @param[in] user_data user specified data for the probe + * @return GST_PAD_PROBE_OK + * @note Example only - Forces Keyframe every 15 frames. + */ +static GstPadProbeReturn encoder_probe_callback(GstPad *pad, GstPadProbeInfo *info, gpointer user_data) +{ + GstElement *pipeline = GST_ELEMENT(user_data); + GstElement *encoder_element = gst_bin_get_by_name(GST_BIN(pipeline), "enco"); + GstEvent * event; + + if (counter % 10 == 0) { + GST_WARNING_OBJECT(encoder_element, "Force Keyframe from application"); + event = gst_video_event_new_downstream_force_key_unit(GST_CLOCK_TIME_NONE, GST_CLOCK_TIME_NONE, GST_CLOCK_TIME_NONE, TRUE, 1); + if (!gst_pad_send_event(pad, event)) + { + GST_ERROR_OBJECT(encoder_element, "Failed to send force key unit event to encoder"); + } + } + counter++; + + return GST_PAD_PROBE_OK; +} + +/** + * Create the gstreamer pipeline as string + * + * @return A string containing the gstreamer pipeline. + * @note prints the return value to the stdout. + */ +std::string create_pipeline_string() +{ + std::string pipeline = ""; + std::string encoder_arguments; + + pipeline = "v4l2src name=src_element num-buffers=300 device=/dev/video0 io-mode=mmap ! " + "video/x-raw,format=NV12,width=1920,height=1080, framerate=30/1 ! " + "queue name=q0 leaky=downstream max-size-buffers=5 max-size-bytes=0 max-size-time=0 ! " + "hailoh265enc name=enco ! " + "h265parse config-interval=-1 ! video/x-h265,framerate=30/1 ! filesink location=force_keyframe.hevc name=hailo_sink"; + + std::cout << "Pipeline:" << std::endl; + std::cout << "gst-launch-1.0 " << pipeline << std::endl; + + return pipeline; +} + +/** + * Set the pipeline's probes + * + * @param[in] pipeline The pipeline as a GstElement. + * @note Sets a probe to the encoder sinkpad. + */ +void set_probes(GstElement *pipeline) +{ + // extract elements from pipeline + GstElement *encoder = gst_bin_get_by_name(GST_BIN(pipeline), "enco"); + // extract pads from elements + GstPad *pad_encoder = gst_element_get_static_pad(encoder, "sink"); + // set probes + gst_pad_add_probe(pad_encoder, GST_PAD_PROBE_TYPE_BUFFER, (GstPadProbeCallback)encoder_probe_callback, pipeline, NULL); +} + +int main(int argc, char *argv[]) +{ + std::string src_pipeline_string; + GstFlowReturn ret; + + gst_init(&argc, &argv); + + std::string pipeline_string = create_pipeline_string(); + GstElement *pipeline = gst_parse_launch(pipeline_string.c_str(), NULL); + set_probes(pipeline); + gst_element_set_state(pipeline, GST_STATE_PLAYING); + + ret = wait_for_end_of_pipeline(pipeline); + + // Free resources + gst_element_set_state(pipeline, GST_STATE_NULL); + gst_deinit(); + gst_object_unref(pipeline); + + return ret; +} diff --git a/core/hailo/apps/hailo15/encoder_applications/meson.build b/core/hailo/apps/hailo15/encoder_applications/meson.build new file mode 100644 index 0000000..a5d12df --- /dev/null +++ b/core/hailo/apps/hailo15/encoder_applications/meson.build @@ -0,0 +1,99 @@ +################################################ +# HAILO 15 CPP APPLICATIONS +################################################ + +encoder_appsink_tee_src = ['encoder_appsink_tee.cpp', 'apps_common.cpp'] + +executable('encoder_appsink_tee', + encoder_appsink_tee_src, + cpp_args : hailo_lib_args, + include_directories: hailo_general_inc, + dependencies : plugin_deps, + gnu_symbol_visibility : 'default', + install: true, + install_dir: apps_install_dir + '/encoder_pipelines', +) + +encoder_change_bitrate_src = ['encoder_change_bitrate.cpp', 'apps_common.cpp'] + +executable('encoder_change_bitrate', + encoder_change_bitrate_src, + cpp_args : hailo_lib_args, + include_directories: hailo_general_inc, + dependencies : plugin_deps, + gnu_symbol_visibility : 'default', + install: true, + install_dir: apps_install_dir + '/encoder_pipelines', +) + +encoder_change_framerate_src = ['encoder_change_framerate.cpp', 'apps_common.cpp'] + +executable('encoder_change_framerate', + encoder_change_framerate_src, + cpp_args : hailo_lib_args, + include_directories: hailo_general_inc, + dependencies : plugin_deps, + gnu_symbol_visibility : 'default', + install: true, + install_dir: apps_install_dir + '/encoder_pipelines', +) + +encoder_change_gop_src = ['encoder_change_gop.cpp', 'apps_common.cpp'] + +executable('encoder_change_gop', + encoder_change_gop_src, + cpp_args : hailo_lib_args, + include_directories: hailo_general_inc, + dependencies : plugin_deps, + gnu_symbol_visibility : 'default', + install: true, + install_dir: apps_install_dir + '/encoder_pipelines', +) + +encoder_change_gop_size_src = ['encoder_change_gop_size.cpp', 'apps_common.cpp'] + +executable('encoder_change_gop_size', + encoder_change_gop_size_src, + cpp_args : hailo_lib_args, + include_directories: hailo_general_inc, + dependencies : plugin_deps, + gnu_symbol_visibility : 'default', + install: true, + install_dir: apps_install_dir + '/encoder_pipelines', +) + +encoder_change_qp_src = ['encoder_change_qp.cpp', 'apps_common.cpp'] + +executable('encoder_change_qp', + encoder_change_qp_src, + cpp_args : hailo_lib_args, + include_directories: hailo_general_inc, + dependencies : plugin_deps, + gnu_symbol_visibility : 'default', + install: true, + install_dir: apps_install_dir + '/encoder_pipelines', +) + +encoder_change_roi_src = ['encoder_change_roi.cpp', 'apps_common.cpp'] + +executable('encoder_change_roi', + encoder_change_roi_src, + cpp_args : hailo_lib_args, + include_directories: hailo_general_inc, + dependencies : plugin_deps, + gnu_symbol_visibility : 'default', + install: true, + install_dir: apps_install_dir + '/encoder_pipelines', +) + +encoder_force_keyframe_src = ['encoder_force_keyframe.cpp', 'apps_common.cpp'] + +executable('encoder_force_keyframe', + encoder_force_keyframe_src, + cpp_args : hailo_lib_args, + include_directories: hailo_general_inc, + dependencies : plugin_deps, + gnu_symbol_visibility : 'default', + install: true, + install_dir: apps_install_dir + '/encoder_pipelines', +) \ No newline at end of file diff --git a/core/hailo/apps/hailo15/meson.build b/core/hailo/apps/hailo15/meson.build index 5609208..29bb895 100644 --- a/core/hailo/apps/hailo15/meson.build +++ b/core/hailo/apps/hailo15/meson.build @@ -26,4 +26,6 @@ lpr_ocrsink_lib = shared_library('lpr_ocrsink', gnu_symbol_visibility : 'default', install: install_lpr, install_dir: apps_install_dir + '/license_plate_recognition/resources', -) \ No newline at end of file +) + +subdir('encoder_applications') \ No newline at end of file diff --git a/core/hailo/apps/meson.build b/core/hailo/apps/meson.build index f207da7..09d16e1 100644 --- a/core/hailo/apps/meson.build +++ b/core/hailo/apps/meson.build @@ -9,44 +9,44 @@ endif thread_deps = [dependency('threads')] ################################################ -# CV Singleton +# GST Image Handling ################################################ -cv_singleton_src = '../general/hailo_cv_singleton.cpp' +image_src = '../plugins/common/image.cpp' -cv_singleton_lib = shared_library('hailo_cv_singleton', - cv_singleton_src, +image_lib = shared_library('hailo_gst_image', + image_src, cpp_args : hailo_lib_args, - include_directories: [hailo_general_inc], - dependencies : [opencv_dep], + include_directories: [hailo_general_inc, include_directories('../plugins')], + dependencies : plugin_deps + [opencv_dep], gnu_symbol_visibility : 'default', version: meson.project_version(), install: true, install_dir: get_option('libdir'), ) -cv_singleton_dep = declare_dependency( - include_directories: [hailo_general_inc], - link_with : cv_singleton_lib) +image_dep = declare_dependency( + include_directories: [include_directories('../plugins/common')], + link_with : image_lib) ################################################ -# GST Image Handling +# CV Singleton ################################################ -image_src = '../plugins/common/image.cpp' +cv_singleton_src = '../general/hailo_cv_singleton.cpp' -image_lib = shared_library('hailo_gst_image', - image_src, +cv_singleton_lib = shared_library('hailo_cv_singleton', + cv_singleton_src, cpp_args : hailo_lib_args, - include_directories: [hailo_general_inc, include_directories('../plugins')], - dependencies : plugin_deps + [opencv_dep], + include_directories: [hailo_general_inc], + dependencies : [opencv_dep, image_dep, plugin_deps], gnu_symbol_visibility : 'default', version: meson.project_version(), install: true, install_dir: get_option('libdir'), ) -image_dep = declare_dependency( - include_directories: [include_directories('../plugins/common')], - link_with : image_lib) +cv_singleton_dep = declare_dependency( + include_directories: [hailo_general_inc], + link_with : cv_singleton_lib) # App Subdirectories if target_platform == 'x86' or target_platform == 'rpi' or target_platform == 'rockchip' diff --git a/core/hailo/apps/x86/lpr/lpr_ocrsink.cpp b/core/hailo/apps/x86/lpr/lpr_ocrsink.cpp index 98c82af..c628e5a 100644 --- a/core/hailo/apps/x86/lpr/lpr_ocrsink.cpp +++ b/core/hailo/apps/x86/lpr/lpr_ocrsink.cpp @@ -31,6 +31,46 @@ std::vector seen_ocr_track_ids; const gchar *OCR_LABEL_TYPE = "ocr"; std::string tracker_name = "hailo_tracker"; +void catalog_nv12_mat(std::string text, cv::Mat &mat) +{ + // Resize the mat to a presentable size, add padding + int target_h = 114; + int target_w = 300; + cv::Mat resized_nv12 = cv::Mat(target_h, target_w, CV_8UC1); + resize_nv12(mat, resized_nv12); + + // Split planes + int y_h = target_h * 2 / 3; + int y_w = target_w; + int uv_h = target_h / 3; + int uv_w = target_w / 2; + cv::Mat y_mat = cv::Mat(y_h, y_w, CV_8UC1, (char *)resized_nv12.data, resized_nv12.step); + cv::Mat uv_mat = cv::Mat(uv_h, uv_w, CV_8UC2, (char *)resized_nv12.data + (y_h * y_w), resized_nv12.step); + + // To make padding, prepare a padded mat and split channels from that mat + int padded_h = target_h + 45; + cv::Mat padded_nv12 = cv::Mat(padded_h, target_w, CV_8UC1, cv::Scalar(0)); + int padded_y_h = padded_h * 2 / 3; + int padded_y_w = target_w; + int padded_uv_h = padded_h / 3; + int padded_uv_w = target_w / 2; + cv::Mat padded_y_mat = cv::Mat(padded_y_h, padded_y_w, CV_8UC1, (char *)padded_nv12.data, padded_nv12.step); + cv::Mat padded_uv_mat = cv::Mat(padded_uv_h, padded_uv_w, CV_8UC2, (char *)padded_nv12.data + (padded_y_h * padded_y_w), padded_nv12.step); + + // Fill the padded image with white padding + cv::copyMakeBorder(y_mat, padded_y_mat, 30, 0, 0, 0, cv::BORDER_CONSTANT, cv::Scalar(235)); + cv::copyMakeBorder(uv_mat, padded_uv_mat, 15, 0, 0, 0, cv::BORDER_CONSTANT, cv::Scalar(128, 128)); + + // Draw text on the two channels + cv::Point y_position = cv::Point(4, 24); + cv::Point uv_position = cv::Point(2, 12); + cv::putText(padded_y_mat, text, y_position, cv::FONT_HERSHEY_SIMPLEX, 1, cv::Scalar(65), 2); + cv::putText(padded_uv_mat, text, uv_position, cv::FONT_HERSHEY_SIMPLEX, 0.5, cv::Scalar(110, 255), 1); + + CVMatSingleton::GetInstance().set_mat_at_key(singleton_map_key % MAP_LIMIT, padded_nv12); + CVMatSingleton::GetInstance().set_mat_type(HAILO_MAT_NV12); +} + void catalog_yuy2_mat(std::string text, cv::Mat &mat) { // Resize the mat to a presentable size, add padding @@ -48,6 +88,7 @@ void catalog_yuy2_mat(std::string text, cv::Mat &mat) // Set the new license plate in our CV Map singleton CVMatSingleton::GetInstance().set_mat_at_key(singleton_map_key % MAP_LIMIT, padded_yuy2); + CVMatSingleton::GetInstance().set_mat_type(HAILO_MAT_YUY2); } void catalog_rgb_mat(std::string text, cv::Mat &mat) @@ -66,9 +107,10 @@ void catalog_rgb_mat(std::string text, cv::Mat &mat) // Set the new license plate in our CV Map singleton CVMatSingleton::GetInstance().set_mat_at_key(singleton_map_key % MAP_LIMIT, padded_image); + CVMatSingleton::GetInstance().set_mat_type(HAILO_MAT_RGB); } -void catalog_license_plate(std::string label, float confidence, HailoBBox license_plate_box, std::shared_ptr hmat) +void catalog_license_plate(std::string label, float confidence, HailoBBox license_plate_box, std::shared_ptr hmat, HailoROIPtr crop_roi) { cv::Mat &mat = hmat->get_mat(); // Prepare the cropped license plate and text @@ -80,7 +122,7 @@ void catalog_license_plate(std::string label, float confidence, HailoBBox licens rect.height = CLAMP(license_plate_box.height() * mat.rows, 0, mat.rows - rect.y); if (rect.width == 0 || rect.height == 0) return; - cv::Mat cropped_image = mat(rect); + cv::Mat cropped_image = hmat->crop(crop_roi); switch (hmat->get_type()) { @@ -94,6 +136,11 @@ void catalog_license_plate(std::string label, float confidence, HailoBBox licens catalog_rgb_mat(text, cropped_image); break; } + case HAILO_MAT_NV12: + { + catalog_nv12_mat(text, cropped_image); + break; + } default: break; } @@ -150,7 +197,7 @@ void ocr_sink(HailoROIPtr roi, std::shared_ptr hmat) unique_ids[0]->get_id(), classification); - catalog_license_plate(license_plate_ocr_label, confidence, license_plate_box, hmat); + catalog_license_plate(license_plate_ocr_label, confidence, license_plate_box, hmat, lp_detection); } } } @@ -160,6 +207,6 @@ void ocr_sink(HailoROIPtr roi, std::shared_ptr hmat) void filter(HailoROIPtr roi, GstVideoFrame *frame) { - std::shared_ptr hmat = get_mat_by_format(*(&frame->buffer), &frame->info, frame->map, 1, 1); + std::shared_ptr hmat = get_mat_by_format(*(&frame->buffer), &frame->info, 1, 1); ocr_sink(roi, hmat); } \ No newline at end of file diff --git a/core/hailo/apps/x86/lpr/lpr_overlay.cpp b/core/hailo/apps/x86/lpr/lpr_overlay.cpp index 8d93e41..9b9e814 100644 --- a/core/hailo/apps/x86/lpr/lpr_overlay.cpp +++ b/core/hailo/apps/x86/lpr/lpr_overlay.cpp @@ -30,19 +30,67 @@ void draw_lpr(cv::Mat &mat) { cv::Mat singleton_image; cv::Mat destinationROI; + cv::Mat y_destinationROI; + cv::Mat uv_destinationROI; + hailo_mat_t mat_format; for (int i = 0; i < OCR_LIMIT; i++) { // Get the cv::Mat for this index, if any singleton_image = CVMatSingleton::GetInstance().get_mat_at_key(i); - if (!singleton_image.empty()) + mat_format = CVMatSingleton::GetInstance().get_mat_type(); + if (!singleton_image.empty() && mat_format != HAILO_MAT_NONE) { - // The CV Map singleton had an image for this key, so draw it - int xmin = mat.cols - singleton_image.cols; - int ymin = i * singleton_image.rows; + switch (mat_format) + { + case HAILO_MAT_YUY2: + case HAILO_MAT_RGB: + { + // The CV Map singleton had an image for this key, so draw it + int xmin = mat.cols - singleton_image.cols; + int ymin = i * singleton_image.rows; - cv::Rect roi(cv::Point(xmin, ymin), cv::Size(singleton_image.cols, singleton_image.rows)); - destinationROI = mat(roi); - singleton_image.copyTo(destinationROI); + cv::Rect roi(cv::Point(xmin, ymin), cv::Size(singleton_image.cols, singleton_image.rows)); + destinationROI = mat(roi); + singleton_image.copyTo(destinationROI); + break; + } + case HAILO_MAT_NV12: + { + // Split mat image planes + int y_h = mat.rows * 2 / 3; + int y_w = mat.cols; + int uv_h = mat.rows / 3; + int uv_w = mat.cols / 2; + cv::Mat y_mat = cv::Mat(y_h, y_w, CV_8UC1, (char *)mat.data, mat.step); + cv::Mat uv_mat = cv::Mat(uv_h, uv_w, CV_8UC2, (char *)mat.data + (y_h * y_w), mat.step); + + // Split singleton image planes + int s_y_h = singleton_image.rows * 2 / 3; + int s_y_w = singleton_image.cols; + int s_uv_h = singleton_image.rows / 3; + int s_uv_w = singleton_image.cols / 2; + cv::Mat y_singleton_mat = cv::Mat(s_y_h, s_y_w, CV_8UC1, (char *)singleton_image.data, singleton_image.step); + cv::Mat uv_singleton_mat = cv::Mat(s_uv_h, s_uv_w, CV_8UC2, (char *)singleton_image.data + (s_y_h * s_y_w), singleton_image.step); + + // Copy over the y channels + int y_xmin = y_mat.cols - y_singleton_mat.cols; + int y_ymin = i * y_singleton_mat.rows; + cv::Rect y_roi(cv::Point(y_xmin, y_ymin), cv::Size(y_singleton_mat.cols, y_singleton_mat.rows)); + y_destinationROI = y_mat(y_roi); + y_singleton_mat.copyTo(y_destinationROI); + + // Copy over the uv channels + int uv_xmin = uv_mat.cols - uv_singleton_mat.cols; + int uv_ymin = i * uv_singleton_mat.rows; + cv::Rect uv_roi(cv::Point(uv_xmin, uv_ymin), cv::Size(uv_singleton_mat.cols, uv_singleton_mat.rows)); + uv_destinationROI = uv_mat(uv_roi); + uv_singleton_mat.copyTo(uv_destinationROI); + + break; + } + default: + break; + } } singleton_image.release(); diff --git a/core/hailo/general/hailo_cv_singleton.cpp b/core/hailo/general/hailo_cv_singleton.cpp index 210ce2c..484303c 100644 --- a/core/hailo/general/hailo_cv_singleton.cpp +++ b/core/hailo/general/hailo_cv_singleton.cpp @@ -30,4 +30,14 @@ cv::Mat CVMatSingleton::get_mat_at_key(int key) { return cv::Mat{}; } return _mat_map[key].clone(); +} + +void CVMatSingleton::set_mat_type(hailo_mat_t new_type) +{ + _mat_type = new_type; +} + +hailo_mat_t CVMatSingleton::get_mat_type() +{ + return _mat_type; } \ No newline at end of file diff --git a/core/hailo/general/hailo_cv_singleton.hpp b/core/hailo/general/hailo_cv_singleton.hpp index c7b47b6..4c108fe 100644 --- a/core/hailo/general/hailo_cv_singleton.hpp +++ b/core/hailo/general/hailo_cv_singleton.hpp @@ -5,23 +5,29 @@ #include #include +// Tappas includes +#include "image.hpp" + // Open source includes #include class CVMatSingleton { private: - CVMatSingleton(): _mat_map({}){} + CVMatSingleton(): _mat_map({}),_mat_type(HAILO_MAT_NONE){} ~CVMatSingleton() {} CVMatSingleton(CVMatSingleton const&); // Don't Implement. CVMatSingleton should not be cloneable. void operator=(CVMatSingleton const&); // Don't implement. CVMatSingleton should not be assignable. std::map _mat_map; static std::mutex mutex_; + hailo_mat_t _mat_type; public: static CVMatSingleton& GetInstance(); void set_mat_at_key(int key, cv::Mat mat); cv::Mat get_mat_at_key(int key); + void set_mat_type(hailo_mat_t new_type); + hailo_mat_t get_mat_type(); }; diff --git a/core/hailo/libs/postprocesses/common/json_config.hpp b/core/hailo/general/json_config.hpp similarity index 100% rename from core/hailo/libs/postprocesses/common/json_config.hpp rename to core/hailo/general/json_config.hpp diff --git a/core/hailo/libs/croppers/lpr/lpr_croppers.cpp b/core/hailo/libs/croppers/lpr/lpr_croppers.cpp index d003094..36283de 100644 --- a/core/hailo/libs/croppers/lpr/lpr_croppers.cpp +++ b/core/hailo/libs/croppers/lpr/lpr_croppers.cpp @@ -121,7 +121,7 @@ std::vector license_plate_quality_estimation(std::shared_ptrget_label()) - continue; + continue; // For each detection, check the inner detections std::vector license_plate_ptrs = hailo_common::get_hailo_detections(vehicle); for (HailoDetectionPtr &license_plate : license_plate_ptrs) diff --git a/core/hailo/libs/postprocesses/classification/classification.cpp b/core/hailo/libs/postprocesses/classification/classification.cpp index 49d8444..e889335 100644 --- a/core/hailo/libs/postprocesses/classification/classification.cpp +++ b/core/hailo/libs/postprocesses/classification/classification.cpp @@ -1,7 +1,7 @@ /** -* Copyright (c) 2021-2022 Hailo Technologies Ltd. All rights reserved. -* Distributed under the LGPL license (https://www.gnu.org/licenses/old-licenses/lgpl-2.1.txt) -**/ + * Copyright (c) 2021-2022 Hailo Technologies Ltd. All rights reserved. + * Distributed under the LGPL license (https://www.gnu.org/licenses/old-licenses/lgpl-2.1.txt) + **/ #include #include #include "common/labels/imagenet.hpp" @@ -13,6 +13,7 @@ #define RESNET_50_LAYER_NAME "resnet_v1_50/softmax1" #define MOBILENET_V1_LAYER_NAME "mobilenet_v1/softmax1" +#define RESNET_V1_18_LAYER_NAME "resnet_v1_18/softmax1" #define COMMA "," void top1(HailoROIPtr roi, std::string layer_name, int label_offset) @@ -67,3 +68,8 @@ void mobilenet_v1(HailoROIPtr roi) { top1(roi, MOBILENET_V1_LAYER_NAME, 1); } + +void resnet_v1_18(HailoROIPtr roi) +{ + top1(roi, RESNET_V1_18_LAYER_NAME, 0); +} \ No newline at end of file diff --git a/core/hailo/libs/postprocesses/classification/classification.hpp b/core/hailo/libs/postprocesses/classification/classification.hpp index f3759f5..202805c 100644 --- a/core/hailo/libs/postprocesses/classification/classification.hpp +++ b/core/hailo/libs/postprocesses/classification/classification.hpp @@ -10,4 +10,5 @@ __BEGIN_DECLS void filter(HailoROIPtr roi); void resnet_v1_50(HailoROIPtr roi); void mobilenet_v1(HailoROIPtr roi); +void resnet_v1_18(HailoROIPtr roi); __END_DECLS \ No newline at end of file diff --git a/core/hailo/libs/postprocesses/detection/face_detection.cpp b/core/hailo/libs/postprocesses/detection/face_detection.cpp index a0b18e3..4c2db7c 100644 --- a/core/hailo/libs/postprocesses/detection/face_detection.cpp +++ b/core/hailo/libs/postprocesses/detection/face_detection.cpp @@ -12,7 +12,7 @@ #include "common/math.hpp" #include "common/tensors.hpp" #include "common/nms.hpp" -#include "common/json_config.hpp" +#include "json_config.hpp" #include "face_detection.hpp" #include "xtensor/xadapt.hpp" #include "xtensor/xarray.hpp" diff --git a/core/hailo/libs/postprocesses/detection/hailo_nms_decode.hpp b/core/hailo/libs/postprocesses/detection/hailo_nms_decode.hpp new file mode 100644 index 0000000..5c17767 --- /dev/null +++ b/core/hailo/libs/postprocesses/detection/hailo_nms_decode.hpp @@ -0,0 +1,153 @@ +/** + * Copyright (c) 2021-2022 Hailo Technologies Ltd. All rights reserved. + * Distributed under the LGPL license (https://www.gnu.org/licenses/old-licenses/lgpl-2.1.txt) + **/ +#include +#include +#include + +#include "hailo_objects.hpp" +#include "common/structures.hpp" +#include "common/nms.hpp" +#include "common/labels/coco_ninety.hpp" +#include "common/labels/coco_visdrone.hpp" + +static const int DEFAULT_MAX_BOXES = 100; +static const float DEFAULT_THRESHOLD = 0.4; + +class HailoNMSDecode +{ +private: + HailoTensorPtr _nms_output_tensor; + std::map labels_dict; + float _detection_thr; + uint _max_boxes; + bool _filter_by_score; + const hailo_vstream_info_t _vstream_info; + + common::hailo_bbox_float32_t dequantize_hailo_bbox(const auto *bbox_struct) + { + // Dequantization of common::hailo_bbox_t (uint16_t) to common::hailo_bbox_float32_t (float32_t) + common::hailo_bbox_float32_t dequant_bbox = { + .y_min = _nms_output_tensor->fix_scale(bbox_struct->y_min), + .x_min = _nms_output_tensor->fix_scale(bbox_struct->x_min), + .y_max = _nms_output_tensor->fix_scale(bbox_struct->y_max), + .x_max = _nms_output_tensor->fix_scale(bbox_struct->x_max), + .score = _nms_output_tensor->fix_scale(bbox_struct->score)}; + + return dequant_bbox; + } + + void parse_bbox_to_detection_object(auto dequant_bbox, uint32_t class_index, std::vector &_objects) + { + float confidence = CLAMP(dequant_bbox.score, 0.0f, 1.0f); + // filter score by detection threshold if needed. + if (!_filter_by_score || dequant_bbox.score > _detection_thr) + { + float32_t w, h = 0.0f; + // parse width and height of the box + std::tie(w, h) = get_shape(&dequant_bbox); + // create new detection object and add it to the vector of detections + _objects.push_back(HailoDetection(HailoBBox(dequant_bbox.x_min, dequant_bbox.y_min, w, h), class_index, labels_dict[class_index], confidence)); + } + } + + std::pair get_shape(auto *bbox_struct) + { + float32_t w = bbox_struct->x_max - bbox_struct->x_min; + float32_t h = bbox_struct->y_max - bbox_struct->y_min; + return std::pair(w, h); + } + +public: + HailoNMSDecode(HailoTensorPtr tensor, std::map &labels_dict, float detection_thr = DEFAULT_THRESHOLD, uint max_boxes = DEFAULT_MAX_BOXES, bool filter_by_score = false) + : _nms_output_tensor(tensor), labels_dict(labels_dict), _detection_thr(detection_thr), _max_boxes(max_boxes), _filter_by_score(filter_by_score), _vstream_info(tensor->vstream_info()) + { + // making sure that the network's output is indeed an NMS type, by checking the order type value included in the metadata + if (HAILO_FORMAT_ORDER_HAILO_NMS != _vstream_info.format.order) + throw std::invalid_argument("Output tensor " + _nms_output_tensor->name() + " is not an NMS type"); + }; + + template + std::vector decode() + { + /* + NMS output decode method + ------------------------ + + decodes the nms buffer received from the output tensor of the network. + returns a vector of DetectonObject filtered by the detection threshold. + + The data is sorted by the number of the classes. + for each class - first comes the number of boxes in the class, then the boxes one after the other, + each box contains x_min, y_min, x_max, y_max and score (uint16_t\float32 each) and can be casted to common::hailo_bbox_t struct (5*uint16_t). + means that a frame size of one class is sizeof(bbox_count) + bbox_count * sizeof(common::hailo_bbox_t). + and the actual size of the data is (frame size of one class)*number of classes. + + If the data comes after quantization - so dequantization to float32 is needed. + + As an example - quantized data buffer of a frame that contains a person and two dogs: + (person class id = 1, dog class id = 18) + + 1 107 96 143 119 172 0 0 0 0 0 0 0 0 0 0 0 0 0 + 0 0 2 123 124 140 150 92 112 125 138 147 91 0 0 + 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 + 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 + 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 + + taking the dogs as example - 2 123 124 140 150 92 112 125 138 147 91 + can be splitted to two different boxes + common::hailo_bbox_t st_1 = 123 124 140 150 92 + common::hailo_bbox_t st_2 = 112 125 138 147 91 + now after dequntization of st_1 - we get common::hailo_bbox_float32_t: + ymin = 0.551805 xmin = 0.389635 ymax = 0.741805 xmax = 0.561974 score = 0.95 + */ + + //_nms_output_tensor - pointer to the output tensor's buffer of the network. + std::vector _objects; + if (!_nms_output_tensor) + return _objects; + + _objects.reserve(_max_boxes); + uint8_t *src_ptr = _nms_output_tensor->data(); + uint32_t actual_frame_size = 0; + + uint32_t num_of_classes = _vstream_info.nms_shape.number_of_classes; + uint32_t max_bboxes_per_class = _vstream_info.nms_shape.max_bboxes_per_class; + + for (uint32_t class_index = 1; class_index <= num_of_classes; class_index++) + { + T bbox_count = *reinterpret_cast(src_ptr + actual_frame_size); + + if (bbox_count > max_bboxes_per_class) + throw std::runtime_error(("Runtime error - Got more than the maximum bboxes per class in the nms buffer")); + + if (bbox_count > 0) + { + uint8_t *class_ptr = src_ptr + actual_frame_size + sizeof(bbox_count); + // iterate over the boxes and parse each box to common::hailo_bbox_t + for (uint8_t box_index = 0; box_index < bbox_count; box_index++) + { + BBoxType *bbox_struct = (BBoxType *)(class_ptr + (box_index * sizeof(BBoxType))); + + if (std::is_same::value) + { + // output type (T) is uint16, so we need to do dequantization before parsing + common::hailo_bbox_float32_t dequant_bbox = dequantize_hailo_bbox(bbox_struct); + parse_bbox_to_detection_object(dequant_bbox, class_index, _objects); + } + else + { + parse_bbox_to_detection_object(*bbox_struct, class_index, _objects); + } + } + } + + // calculate the frame size of the class - sums up the size of the output during iteration + T class_frame_size = static_cast(sizeof(bbox_count) + bbox_count * sizeof(BBoxType)); + actual_frame_size += class_frame_size; + } + + return _objects; + } +}; diff --git a/core/hailo/libs/postprocesses/detection/mobilenet_ssd.cpp b/core/hailo/libs/postprocesses/detection/mobilenet_ssd.cpp index 4b0e5d7..1b72226 100644 --- a/core/hailo/libs/postprocesses/detection/mobilenet_ssd.cpp +++ b/core/hailo/libs/postprocesses/detection/mobilenet_ssd.cpp @@ -1,161 +1,31 @@ /** -* Copyright (c) 2021-2022 Hailo Technologies Ltd. All rights reserved. -* Distributed under the LGPL license (https://www.gnu.org/licenses/old-licenses/lgpl-2.1.txt) -**/ -#include -#include -#include "mobilenet_ssd.hpp" -#include "hailo_objects.hpp" -#include "common/structures.hpp" -#include "common/nms.hpp" -#include "common/labels/coco_ninety.hpp" -#include "common/labels/coco_visdrone.hpp" - -static const int DEFAULT_MAX_BOXES = 100; -static const float DEFAULT_MOBILENET_THRESHOLD = 0.4; -static const std::string DEFAULT_NMS_OUTPUT_LAYER = "ssd_mobilenet_v1/nms1"; - -class MobilenetSSDPost -{ -private: - HailoTensorPtr _nms_output_tensor; - std::map labels_dict; - float _detection_thr; - uint _max_boxes; - const hailo_vstream_info_t _vstream_info; - - common::hailo_bbox_float32_t dequantize_hailo_bbox(const common::hailo_bbox_t *bbox_struct) - { - // Dequantization of common::hailo_bbox_t (uint16_t) to common::hailo_bbox_float32_t (float32_t) - common::hailo_bbox_float32_t dequant_bbox = { - .y_min = _nms_output_tensor->fix_scale(bbox_struct->y_min), - .x_min = _nms_output_tensor->fix_scale(bbox_struct->x_min), - .y_max = _nms_output_tensor->fix_scale(bbox_struct->y_max), - .x_max = _nms_output_tensor->fix_scale(bbox_struct->x_max), - .score = _nms_output_tensor->fix_scale(bbox_struct->score)}; - - return dequant_bbox; - } - - void parse_bbox_to_detection_object(common::hailo_bbox_t *bbox_struct, uint32_t class_index, std::vector& _objects) - { - // dequantize common::hailo_bbox_t - common::hailo_bbox_float32_t dequant_bbox = dequantize_hailo_bbox(bbox_struct); - float confidence = CLAMP(dequant_bbox.score, 0.0f, 1.0f); - // filter score by detection threshold. - if (dequant_bbox.score > _detection_thr) - { - float32_t w, h = 0.0f; - // parse width and height of the box - std::tie(w, h) = get_shape(&dequant_bbox); - // create new detection object and add it to the vector of detections - _objects.push_back(HailoDetection(HailoBBox(dequant_bbox.x_min, dequant_bbox.y_min, w, h), class_index, labels_dict[class_index], confidence)); - } - } - - std::pair get_shape(common::hailo_bbox_float32_t *bbox_struct) - { - float32_t w = bbox_struct->x_max - bbox_struct->x_min; - float32_t h = bbox_struct->y_max - bbox_struct->y_min; - return std::pair(w, h); - } - -public: - MobilenetSSDPost(HailoTensorPtr tensor, std::map &labels_dict, float detection_thr = DEFAULT_MOBILENET_THRESHOLD, uint max_boxes = DEFAULT_MAX_BOXES) - : _nms_output_tensor(tensor), labels_dict(labels_dict), _detection_thr(detection_thr), _max_boxes(max_boxes), _vstream_info(tensor->vstream_info()) - { - // making sure that the network's output is indeed an NMS type, by checking the order type value included in the metadata - if (HAILO_FORMAT_ORDER_HAILO_NMS != _vstream_info.format.order) - throw std::invalid_argument("Output tensor " + _nms_output_tensor->name() + " is not an NMS type"); - }; - - std::vector decode() - { - /* - NMS output decode method - decodes the nms buffer received from the output tensor of the network. - returns a vector of DetectonObject filtered by the detection threshold. + * Copyright (c) 2021-2022 Hailo Technologies Ltd. All rights reserved. + * Distributed under the LGPL license (https://www.gnu.org/licenses/old-licenses/lgpl-2.1.txt) + **/ - The data is sorted by the number of the classes. - for each class - first comes the number of boxes in the class, then the boxes one after the other, - each box contains x_min, y_min, x_max, y_max and score (uint16_t each) and can be casted to common::hailo_bbox_t struct (5*uint16_t). - means that a frame size of one class is sizeof(bbox_count) + bbox_count * sizeof(common::hailo_bbox_t). - and the actual size of the data is (frame size of one class)*number of classes. - - The data comes after quantization - so dequantization to float32 is needed. - - As an example - data buffer of a frame that contains a person and two dogs: - (person class id = 1, dog class id = 18) - - 1 107 96 143 119 172 0 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 2 123 124 140 150 92 112 125 138 147 91 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 - 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 - - taking the dogs as example - 2 123 124 140 150 92 112 125 138 147 91 - can be splitted to two different boxes - common::hailo_bbox_t st_1 = 123 124 140 150 92 - common::hailo_bbox_t st_2 = 112 125 138 147 91 - now after dequntization of st_1 - we get common::hailo_bbox_float32_t: - ymin = 0.551805 xmin = 0.389635 ymax = 0.741805 xmax = 0.561974 score = 0.95 - */ - - //_nms_output_tensor - pointer to the output tensor's buffer of the network. - - std::vector _objects; - if (!_nms_output_tensor) - return _objects; - - _objects.reserve(_max_boxes); - uint8_t *src_ptr = _nms_output_tensor->data(); - uint32_t actual_frame_size = 0; - - uint32_t num_of_classes = _vstream_info.nms_shape.number_of_classes; - uint32_t max_bboxes_per_class = _vstream_info.nms_shape.max_bboxes_per_class; - for (uint32_t class_index = 1; class_index <= num_of_classes; class_index++) - { - uint16_t bbox_count = *reinterpret_cast(src_ptr + actual_frame_size); - if (bbox_count > max_bboxes_per_class) - throw std::runtime_error(("Runtime error - Got more than the maximun bboxes per class in the nms buffer")); - - if (bbox_count > 0) - { - uint8_t *class_ptr = src_ptr + actual_frame_size + sizeof(bbox_count); - // iterate over the boxes and parse each box to common::hailo_bbox_t - for (uint8_t box_index = 0; box_index < bbox_count; box_index++) - { - common::hailo_bbox_t *bbox_struct = (common::hailo_bbox_t *)(class_ptr + (box_index * sizeof(common::hailo_bbox_t))); - parse_bbox_to_detection_object(bbox_struct, class_index, _objects); - } - } - // calculate the frame size of the class - sums up the size of the output during iteration - uint16_t class_frame_size = static_cast(sizeof(bbox_count) + bbox_count * sizeof(common::hailo_bbox_t)); - actual_frame_size += class_frame_size; - } +#include "mobilenet_ssd.hpp" +#include "hailo_nms_decode.hpp" - return _objects; - } -}; +static const std::string DEFAULT_SSD_OUTPUT_LAYER = "ssd_mobilenet_v1/nms1"; void mobilenet_ssd(HailoROIPtr roi) { - auto post = MobilenetSSDPost(roi->get_tensor(DEFAULT_NMS_OUTPUT_LAYER), common::coco_ninety_classes); - auto detections = post.decode(); + auto post = HailoNMSDecode(roi->get_tensor(DEFAULT_SSD_OUTPUT_LAYER), common::coco_ninety_classes); + auto detections = post.decode(); hailo_common::add_detections(roi, detections); } void mobilenet_ssd_merged(HailoROIPtr roi) { - auto post = MobilenetSSDPost(roi->get_tensor("ssd_mobilenet_v1_no_alls/nms1"), common::coco_ninety_classes); - auto detections = post.decode(); + auto post = HailoNMSDecode(roi->get_tensor("ssd_mobilenet_v1_no_alls/nms1"), common::coco_ninety_classes); + auto detections = post.decode(); hailo_common::add_detections(roi, detections); } void mobilenet_ssd_visdrone(HailoROIPtr roi) { - auto post = MobilenetSSDPost(roi->get_tensor("ssd_mobilenet_v1_visdrone/nms1"), common::coco_visdrone_classes); - auto detections = post.decode(); + auto post = HailoNMSDecode(roi->get_tensor("ssd_mobilenet_v1_visdrone/nms1"), common::coco_visdrone_classes); + auto detections = post.decode(); hailo_common::add_detections(roi, detections); } diff --git a/core/hailo/libs/postprocesses/detection/scrfd.cpp b/core/hailo/libs/postprocesses/detection/scrfd.cpp index 7a9bc80..411e6f5 100644 --- a/core/hailo/libs/postprocesses/detection/scrfd.cpp +++ b/core/hailo/libs/postprocesses/detection/scrfd.cpp @@ -12,7 +12,7 @@ #include "common/math.hpp" #include "common/tensors.hpp" #include "common/nms.hpp" -#include "common/json_config.hpp" +#include "json_config.hpp" #include "scrfd.hpp" #include "xtensor/xarray.hpp" #include "xtensor/xio.hpp" diff --git a/core/hailo/libs/postprocesses/detection/yolo_hailortpp.cpp b/core/hailo/libs/postprocesses/detection/yolo_hailortpp.cpp new file mode 100644 index 0000000..d8d502b --- /dev/null +++ b/core/hailo/libs/postprocesses/detection/yolo_hailortpp.cpp @@ -0,0 +1,54 @@ +#include "hailo_nms_decode.hpp" +#include "yolo_hailortpp.hpp" +#include "common/labels/coco_eighty.hpp" + +static const std::string DEFAULT_YOLOV5M_OUTPUT_LAYER = "yolov5_nms_postprocess"; + +static std::map yolo_vehicles_labels = { + {0, "unlabeled"}, + {1, "car"}}; + +void yolov5(HailoROIPtr roi) +{ + auto post = HailoNMSDecode(roi->get_tensor(DEFAULT_YOLOV5M_OUTPUT_LAYER), common::coco_eighty); + auto detections = post.decode(); + hailo_common::add_detections(roi, detections); +} + +void yolox(HailoROIPtr roi) +{ + auto post = HailoNMSDecode(roi->get_tensor("yolox_nms_postprocess"), common::coco_eighty); + auto detections = post.decode(); + hailo_common::add_detections(roi, detections); +} + +void yolov5m_vehicles(HailoROIPtr roi) +{ + // auto post = HailoNMSDecode(roi->get_tensor(DEFAULT_YOLOV5M_OUTPUT_LAYER), yolo_vehicles_labels, 0.4, DEFAULT_MAX_BOXES, true); + auto post = HailoNMSDecode(roi->get_tensor(DEFAULT_YOLOV5M_OUTPUT_LAYER), yolo_vehicles_labels); + auto detections = post.decode(); + hailo_common::add_detections(roi, detections); +} + +void yolov5_no_persons(HailoROIPtr roi) +{ + auto post = HailoNMSDecode(roi->get_tensor(DEFAULT_YOLOV5M_OUTPUT_LAYER), common::coco_eighty); + auto detections = post.decode(); + for (auto it = detections.begin(); it != detections.end();) + { + if (it->get_label() == "person") + { + it = detections.erase(it); + } + else + { + ++it; + } + } + hailo_common::add_detections(roi, detections); +} + +void filter(HailoROIPtr roi) +{ + yolov5(roi); +} diff --git a/core/hailo/libs/postprocesses/detection/yolo_hailortpp.hpp b/core/hailo/libs/postprocesses/detection/yolo_hailortpp.hpp new file mode 100644 index 0000000..f759cc3 --- /dev/null +++ b/core/hailo/libs/postprocesses/detection/yolo_hailortpp.hpp @@ -0,0 +1,16 @@ +/** +* Copyright (c) 2021-2022 Hailo Technologies Ltd. All rights reserved. +* Distributed under the LGPL license (https://www.gnu.org/licenses/old-licenses/lgpl-2.1.txt) +**/ +#pragma once +#include "hailo_objects.hpp" +#include "hailo_common.hpp" + + +__BEGIN_DECLS +void filter(HailoROIPtr roi); +void yolov5(HailoROIPtr roi); +void yolox(HailoROIPtr roi); +void yolov5_no_persons(HailoROIPtr roi); +void yolov5m_vehicles(HailoROIPtr roi); +__END_DECLS \ No newline at end of file diff --git a/core/hailo/libs/postprocesses/detection/yolo_postprocess.cpp b/core/hailo/libs/postprocesses/detection/yolo_postprocess.cpp index 036e30b..a365b00 100644 --- a/core/hailo/libs/postprocesses/detection/yolo_postprocess.cpp +++ b/core/hailo/libs/postprocesses/detection/yolo_postprocess.cpp @@ -10,7 +10,7 @@ #include "yolo_postprocess.hpp" #include "common/nms.hpp" -#include "common/json_config.hpp" +#include "json_config.hpp" #include "rapidjson/document.h" #include "rapidjson/stringbuffer.h" diff --git a/core/hailo/libs/postprocesses/instance_segmentation/mask_decoding.hpp b/core/hailo/libs/postprocesses/instance_segmentation/mask_decoding.hpp index a5716ea..e5d56e7 100644 --- a/core/hailo/libs/postprocesses/instance_segmentation/mask_decoding.hpp +++ b/core/hailo/libs/postprocesses/instance_segmentation/mask_decoding.hpp @@ -12,6 +12,12 @@ auto xtensor_sigmoid(auto &tensor) return 1 / (1 + xt::exp(-tensor)); } +/* + * @brief sigmoid on a single float + * + * */ +inline float sigmoid(float x) { return 1.0f / (1.0f + std::exp(-1.0 * x)); } + /** * @brief Compute tensor dot product along specified axes for arrays. In this case along axis 2 of the first matrix and axis 0 of the second. diff --git a/core/hailo/libs/postprocesses/instance_segmentation/yolov5seg.cpp b/core/hailo/libs/postprocesses/instance_segmentation/yolov5seg.cpp index e593a44..4c196a2 100644 --- a/core/hailo/libs/postprocesses/instance_segmentation/yolov5seg.cpp +++ b/core/hailo/libs/postprocesses/instance_segmentation/yolov5seg.cpp @@ -6,14 +6,44 @@ #include "common/nms.hpp" #include "common/labels/coco_eighty.hpp" #include "mask_decoding.hpp" + +#include "json_config.hpp" +#include "rapidjson/document.h" +#include "rapidjson/stringbuffer.h" +#include "rapidjson/error/en.h" +#include "rapidjson/filereadstream.h" +#include "rapidjson/schema.h" + #include #include #include +#if __GNUC__ > 8 +#include +namespace fs = std::filesystem; +#else +#include +namespace fs = std::experimental::filesystem; +#endif // the net returns 32 values representing the mask coefficients, and 4 values representing the box coordinates #define MASK_CO 32 #define BOX_CO 4 +/** + * @brief Compute sigmoid's inverse + */ +inline float inverse_sigmoid(float y) { return std::log(y/(1-y));} + +/** + * @brief perform quantization + */ +inline uint16_t quant(float num, float qp_zp, float qp_scale) { return uint16_t((num / qp_scale) + qp_zp); } + +/** + * @brief perform dequantization + */ +inline float dequant(uint16_t num, float qp_zp, float qp_scale) { return (float(num) - qp_zp) * qp_scale;} + /* * @brief Creates the grid and the anchor grid that will be used for each decoding * @@ -33,12 +63,14 @@ std::tuple, xt::xarray> make_grid(xt::xarray &an // making grid auto stack = xt::stack(xt::xtuple(xv, yv), 2); stack.reshape({1, ny, nx, 2}); - auto grid = xt::broadcast(stack, {num_anchors, ny, nx, 2}) - 0.5; + xt::xarray grid = xt::broadcast(stack, {num_anchors, ny, nx, 2}) - 0.5; + xt::xarray transposed_grid = xt::transpose(grid, {1, 2, 0, 3}); // num_anchors, h, w, features // making anchor grid anchors *= stride; anchors.reshape({num_anchors, 1, 1, 2}); - auto anchor_grid = xt::broadcast(anchors, {num_anchors, ny, nx, 2}); - return std::tuple, xt::xarray>(std::move(grid), std::move(anchor_grid)); + xt::xarray anchor_grid = xt::broadcast(anchors, {num_anchors, ny, nx, 2}); + xt::xarray transposed_anchor_grid = xt::transpose(anchor_grid, {1, 2, 0, 3}); // num_anchors, h, w, features + return std::tuple, xt::xarray>(std::move(transposed_grid), std::move(transposed_anchor_grid)); } /* @@ -48,26 +80,33 @@ std::tuple, xt::xarray> make_grid(xt::xarray &an * @param all_is_object an xview with the confidence that this detection is an object, for each detection * @param score_threshold float */ -std::vector filter_above_threshold(auto &all_scores, auto &all_is_object, const float score_threshold) +auto filter_above_threshold(auto &all_scores, auto &is_object_threshold, const float score_threshold, const uint16_t threshold_quantized, const float qp_zp, const float qp_scale) { - std::vector indices; + std::vector indices; + std::vector scores; + std::vector classes; int this_index; - float conf; - for (uint i = 0; i < all_is_object.size(); i++) + float conf_deq, is_object_deq; + uint16_t is_object; + for (uint i = 0; i < is_object_threshold.size(); i++) { // first check if the object parameter is bigger than threshold - if (all_is_object(i, 0) > score_threshold) + is_object = is_object_threshold(i, 0); + if (is_object > threshold_quantized) { this_index = xt::argmax(xt::row(all_scores, i))(0) + 1; - conf = all_scores(i, this_index - 1); - // now check if confidence of the class with highest score * object is bigger than threshold - if (conf > score_threshold) + // dequantize and decode + conf_deq = sigmoid(dequant(all_scores(i, this_index - 1), qp_zp, qp_scale)); + is_object_deq = sigmoid(dequant(is_object, qp_zp, qp_scale)); + if (conf_deq*is_object_deq > score_threshold) { indices.emplace_back(i); - } + scores.emplace_back(conf_deq * is_object_deq); + classes.emplace_back(this_index); } } - return indices; + } + return std::tuple, std::vector, std::vector>(std::move(indices), std::move(scores), std::move(classes)); } /* @@ -80,25 +119,25 @@ std::vector filter_above_threshold(auto &all_scores, auto &all_is_object, c * @param masks an xview with 32 coefficients representing a mask per detection * @param objects a vecor of HailoDetections, to which the detections will be added * */ -void create_hailo_detections(const uint size, auto &boxes, auto &is_object, auto &scores, auto &masks, std::vector &objects) +std::vector create_hailo_detections(auto &scores_vec, auto &classes_vec, auto &xy, auto wh, auto &masks, const int input_width, const int input_height) { int class_index; float confidence, w, h, x, y = 0.0; - for (uint index = 0; index < size; index++) + std::vector objects; + for (uint i = 0; i < scores_vec.size(); i++) { // Get the box parameters for this box - x = (boxes(index, 0)); - y = (boxes(index, 1)); - w = (boxes(index, 2)); - h = (boxes(index, 3)); + x = (xy(i, 0)) / input_width; + y = (xy(i, 1)) / input_height; + w = (wh(i, 0)) / input_width; + h = (wh(i, 1)) / input_height; // x and y represented center of box, so they need to be changed to left bottom corner HailoBBox bbox(x - w / 2, y - h / 2, w, h); - // calculate label and confidence - class_index = xt::argmax(xt::row(scores, index))(0) + 1; + class_index = classes_vec[i]; std::string label = common::coco_eighty[class_index]; - confidence = scores(index, class_index - 1) * is_object(index, 0); // Decrement class_index since scores excludes class 0 (background) + confidence = scores_vec[i]; // create mask - xt::xarray mask_coefficients = xt::squeeze(xt::view(masks, xt::keep(index), xt::all())); + xt::xarray mask_coefficients = xt::squeeze(xt::view(masks, xt::keep(i), xt::all())); HailoDetection detected_instance(bbox, class_index, label, confidence); std::vector data(mask_coefficients.shape(0)); memcpy(data.data(), mask_coefficients.data(), sizeof(float) * mask_coefficients.shape(0)); @@ -106,42 +145,54 @@ void create_hailo_detections(const uint size, auto &boxes, auto &is_object, auto detected_instance.add_object((std::make_shared(data, mask_coefficients.shape(0), 1))); objects.push_back(detected_instance); } - return; + return objects; } /* * @brief Does the decoding and the filtering for the output, and adds the results to the HailoDetections vector * * */ -std::vector yolov5_decoding(xt::xarray &output, const int stride, xt::xarray &anchors, xt::xarray &grid, xt::xarray &anchor_grid, const int num_anchors, const float score_threshold) +std::vector yolov5_decoding(xt::xarray &output, const int stride, xt::xarray &anchors, xt::xarray &grid, xt::xarray &anchor_grid, const int num_anchors, const float score_threshold, float qp_zp, float qp_scale, const int input_width, const int input_height) { int h = output.shape()[0]; int w = output.shape()[1]; int num_classes = (output.shape()[2] / 3) - BOX_CO - 1 - MASK_CO; - auto reshaped_output = xt::reshape_view(output, {h, w, num_anchors, BOX_CO + 1 + num_classes + MASK_CO}); - auto new_output = xt::transpose(reshaped_output, {2, 0, 1, 3}); - auto xy = xt::view(new_output, xt::all(), xt::all(), xt::all(), xt::range(_, 2)); - auto wh = xt::view(new_output, xt::all(), xt::all(), xt::all(), xt::range(2, 4)); - auto conf = xt::view(new_output, xt::all(), xt::all(), xt::all(), xt::range(4, 4 + num_classes + 1)); - auto mask = xt::view(new_output, xt::all(), xt::all(), xt::all(), xt::range(4 + num_classes + 1, _)); - // decoding - auto new_xy = (xtensor_sigmoid(xy) * 2 + grid) * stride; - auto new_wh = xt::square(xtensor_sigmoid(wh) * 2) * anchor_grid; - auto new_conf = xtensor_sigmoid(conf); - auto out = xt::concatenate(xt::xtuple(new_xy, new_wh, new_conf, mask), 3); - // organize as number of detections x 117 (coordinates, is_object, classes confidence, mask) - auto all_decoded = xt::reshape_view(out, {num_anchors * h * w, BOX_CO + 1 + num_classes + MASK_CO}); - // filter out detections with confidence above threshold + + // prepare data for filter function + auto all_decoded = xt::reshape_view(output, {num_anchors * h * w, BOX_CO + 1 + num_classes + MASK_CO}); // {number of detections, 117} auto all_is_object = xt::view(all_decoded, xt::all(), xt::range(4, 5)); - auto all_scores = xt::view(all_decoded, xt::all(), xt::range(5, 85)); - std::vector indices = filter_above_threshold(all_scores, all_is_object, score_threshold); - auto boxes = xt::view(all_decoded, xt::keep(indices), xt::range(_, 4)) / 640; - auto is_object = xt::view(all_is_object, xt::keep(indices), xt::all()); - auto scores = xt::view(all_scores, xt::keep(indices), xt::all()); - auto masks = xt::view(all_decoded, xt::keep(indices), xt::range(85, _)); + auto all_scores = xt::view(all_decoded, xt::all(), xt::range(5, num_classes + 5)); + // quantize the score threshold + "undecode" it (do inverse of sigmoid), to avoid doing dequantization and decoding on all class scores + uint16_t threshold_quantized = quant(inverse_sigmoid(score_threshold), qp_zp, qp_scale); + auto filtered = filter_above_threshold(all_scores, all_is_object, score_threshold, threshold_quantized, qp_zp, qp_scale); + std::vector indices = std::get<0>(filtered); + std::vector scores_vec = std::get<1>(filtered); + std::vector classes_vec = std::get<2>(filtered); + + // filter xy and grid + auto xy = xt::view(all_decoded, xt::all(), xt::range(_, 2)); + auto reshaped_grid = xt::reshape_view(grid, xy.shape()); + auto filtered_xy = xt::view(xy, xt::keep(indices), xt::all()); + auto filtered_grid = xt::view(reshaped_grid, xt::keep(indices), xt::all()); + // dequantize and decode xy + xt::xarray deq_xy = (filtered_xy - qp_zp) * qp_scale; + deq_xy = (xtensor_sigmoid(deq_xy) * 2 + filtered_grid) * stride; + + // filter wh and anchor grid + auto wh = xt::view(all_decoded, xt::all(), xt::range(2, 4)); + auto reshaped_anchor_grid = xt::reshape_view(anchor_grid, wh.shape()); + auto filtered_wh = xt::view(wh, xt::keep(indices), xt::all()); + auto filtered_anchor_grid = xt::view(reshaped_anchor_grid, xt::keep(indices), xt::all()); + // dequantize and decode wh + xt::xarray deq_wh = (filtered_wh - qp_zp) * qp_scale; + deq_wh = xt::square(xtensor_sigmoid(deq_wh) * 2) * filtered_anchor_grid; + + // filter and dequantize masks + auto filtered_masks = xt::view(all_decoded, xt::keep(indices), xt::range(num_classes + 5, _)); + xt::xarray masks = (filtered_masks - qp_zp) * qp_scale; + // create HailoDetections for the NMS and the mask decoding - std::vector objects; - create_hailo_detections(indices.size(), boxes, is_object, scores, masks, objects); + std::vector objects = create_hailo_detections(scores_vec, classes_vec, deq_xy, deq_wh, masks, input_width, input_height); return objects; } @@ -149,24 +200,26 @@ std::vector yolov5_decoding(xt::xarray &output, const int * @brief Does dequantize and decoding for each output seperately * * */ -std::vector post_per_branch(std::string branch_name, const int index, std::map tensors, std::vector> anchor_list, std::vector stride_list, const float iou_threshold, const float score_threshold, std::vector> grids, std::vector> anchor_grids, const int num_anchors) +std::vector post_per_branch(std::string branch_name, const int index, std::map tensors, std::vector> anchor_list, std::vector stride_list, const float iou_threshold, const float score_threshold, std::vector> grids, std::vector> anchor_grids, const int num_anchors, const int input_width, const int input_height) { - auto output = common::dequantize(common::get_xtensor_uint16(tensors[branch_name]), tensors[branch_name]->vstream_info().quant_info.qp_scale, tensors[branch_name]->vstream_info().quant_info.qp_zp); - return yolov5_decoding(output, stride_list[index], anchor_list[index], grids[index], anchor_grids[index], num_anchors, score_threshold); + auto output = common::get_xtensor_uint16(tensors[branch_name]); + float qp_zp = tensors[branch_name]->vstream_info().quant_info.qp_zp; + float qp_scale = tensors[branch_name]->vstream_info().quant_info.qp_scale; + return yolov5_decoding(output, stride_list[index], anchor_list[index], grids[index], anchor_grids[index], num_anchors, score_threshold, qp_zp, qp_scale, input_width, input_height); } /* * @brief Does dequantize and decoding for each output, and then calls nms and decode masks * * */ -std::vector yolov5seg_post(auto &tensors, auto &anchor_list, auto &stride_list, const float iou_threshold, const float score_threshold, auto &grids, auto &anchor_grids, const int num_anchors) +std::vector yolov5seg_post(auto &tensors, auto &anchor_list, auto &stride_list, const float iou_threshold, const float score_threshold, auto &grids, auto &anchor_grids, const int num_anchors, const int input_width, const int input_height, auto &outputs_name) { - auto proto_tensor = common::dequantize(common::get_xtensor(tensors["yolov5n_seg/conv63"]), tensors["yolov5n_seg/conv63"]->vstream_info().quant_info.qp_scale, tensors["yolov5n_seg/conv63"]->vstream_info().quant_info.qp_zp); + auto proto_tensor = common::dequantize(common::get_xtensor(tensors[outputs_name[0]]), tensors[outputs_name[0]]->vstream_info().quant_info.qp_scale, tensors[outputs_name[0]]->vstream_info().quant_info.qp_zp); // run the postprocess for each branch seperately - std::future> t2 = std::async(post_per_branch, "yolov5n_seg/conv48", 2, tensors, anchor_list, stride_list, iou_threshold, score_threshold, grids, anchor_grids, num_anchors); - std::future> t1 = std::async(post_per_branch, "yolov5n_seg/conv55", 1, tensors, anchor_list, stride_list, iou_threshold, score_threshold, grids, anchor_grids, num_anchors); - std::future> t0 = std::async(post_per_branch, "yolov5n_seg/conv61", 0, tensors, anchor_list, stride_list, iou_threshold, score_threshold, grids, anchor_grids, num_anchors); + std::future> t2 = std::async(post_per_branch, outputs_name[1], 2, tensors, anchor_list, stride_list, iou_threshold, score_threshold, grids, anchor_grids, num_anchors, input_width, input_height); + std::future> t1 = std::async(post_per_branch, outputs_name[2], 1, tensors, anchor_list, stride_list, iou_threshold, score_threshold, grids, anchor_grids, num_anchors, input_width, input_height); + std::future> t0 = std::async(post_per_branch, outputs_name[3], 0, tensors, anchor_list, stride_list, iou_threshold, score_threshold, grids, anchor_grids, num_anchors, input_width, input_height); std::vector d2 = t2.get(); std::vector d1 = t1.get(); std::vector d0 = t0.get(); @@ -186,6 +239,135 @@ std::vector yolov5seg_post(auto &tensors, auto &anchor_list, aut Yolov5segParams *init(const std::string config_path, const std::string function_name) { Yolov5segParams *params = new Yolov5segParams(); + if (!fs::exists(config_path)) + { + std::cerr << "Config file doesn't exist, using default parameters" << std::endl; + } + else { + char config_buffer[4096]; + const char *json_schema = R""""({ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Generated schema for Root", + "type": "object", + "properties": { + "iou_threshold": { + "type": "number" + }, + "score_threshold": { + "type": "number" + }, + "outputs_size": { + "type": "array", + "items": { + "type": "number" + } + }, + "outputs_name": { + "type": "array", + "items": { + "type": "string" + } + }, + "anchors": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "number" + } + } + }, + "input_shape": { + "type": "array", + "items": { + "type": "number" + } + }, + "strides": { + "type": "array", + "items": { + "type": "number" + } + } + }, + "required": [ + "iou_threshold", + "score_threshold", + "outputs_size", + "anchors", + "input_shape", + "strides" + ] + })""""; + std::FILE *fp = fopen(config_path.c_str(), "r"); + if (fp == nullptr) + { + throw std::runtime_error("JSON config file is not valid"); + } + rapidjson::FileReadStream stream(fp, config_buffer, sizeof(config_buffer)); + bool valid = common::validate_json_with_schema(stream, json_schema); + if (valid) + { + rapidjson::Document doc_config_json; + doc_config_json.ParseStream(stream); + + params->iou_threshold = doc_config_json["iou_threshold"].GetFloat(); + params->score_threshold = doc_config_json["score_threshold"].GetFloat(); + + // parse anchors + auto config_anchors = doc_config_json["anchors"].GetArray(); + std::vector> anchors_vec; + for (uint j = 0; j < config_anchors.Size(); j++) + { + uint size = config_anchors[j].GetArray().Size(); + std::vector anchor; + for (uint k = 0; k < size; k++) + { + anchor.push_back(config_anchors[j].GetArray()[k].GetFloat()); + } + auto anchors_tensor = xt::adapt(anchor); + anchors_vec.push_back(anchors_tensor); + } + params->anchors = anchors_vec; + + // parse outputs_size + auto config_outputs_size = doc_config_json["outputs_size"].GetArray(); + std::vector outputs_size_vec; + for (uint j = 0; j < config_outputs_size.Size(); j++) + { + outputs_size_vec.push_back(config_outputs_size[j].GetInt()); + } + params->outputs_size = outputs_size_vec; + + // parse outputs_name + auto config_outputs_name = doc_config_json["outputs_name"].GetArray(); + std::vector outputs_name_vec; + for (uint j = 0; j < config_outputs_name.Size(); j++) + { + outputs_name_vec.push_back(config_outputs_name[j].GetString()); + } + params->outputs_name = outputs_name_vec; + + // parse input_shape + auto config_input_shape = doc_config_json["input_shape"].GetArray(); + std::vector input_shape_vec; + for (uint j = 0; j < config_input_shape.Size(); j++) + { + input_shape_vec.push_back(config_input_shape[j].GetInt()); + } + params->input_shape = input_shape_vec; + + // parse strides + auto config_strides = doc_config_json["strides"].GetArray(); + std::vector strides_vec; + for (uint j = 0; j < config_strides.Size(); j++) + { + strides_vec.push_back(config_strides[j].GetInt()); + } + params->strides = strides_vec; + + fclose(fp); + } } std::vector outputs_size = params->outputs_size; std::vector> anchors = params->anchors; std::vector strides = params->strides; @@ -224,7 +406,7 @@ void yolov5seg(HailoROIPtr roi, void *params_void_ptr) { Yolov5segParams *params = reinterpret_cast(params_void_ptr); std::map tensors = roi->get_tensors_by_name(); - std::vector detections = yolov5seg_post(tensors, params->anchors, params->strides, params->iou_threshold, params->score_threshold, params->grids, params->anchor_grids, params->num_anchors); + std::vector detections = yolov5seg_post(tensors, params->anchors, params->strides, params->iou_threshold, params->score_threshold, params->grids, params->anchor_grids, params->num_anchors, params->input_shape[0], params->input_shape[1], params->outputs_name); hailo_common::add_detections(roi, detections); } diff --git a/core/hailo/libs/postprocesses/instance_segmentation/yolov5seg.hpp b/core/hailo/libs/postprocesses/instance_segmentation/yolov5seg.hpp index 8adffe9..d12c8ff 100644 --- a/core/hailo/libs/postprocesses/instance_segmentation/yolov5seg.hpp +++ b/core/hailo/libs/postprocesses/instance_segmentation/yolov5seg.hpp @@ -15,6 +15,7 @@ class Yolov5segParams float score_threshold; int num_anchors; std::vector outputs_size; + std::vector outputs_name; std::vector> anchors; std::vector input_shape; std::vector strides; @@ -25,6 +26,7 @@ class Yolov5segParams iou_threshold = 0.6; score_threshold = 0.25; outputs_size = {20, 40, 80}; + outputs_name = {"yolov5n_seg/conv63", "yolov5n_seg/conv48", "yolov5n_seg/conv55", "yolov5n_seg/conv61"}; anchors = {{116, 90, 156, 198, 373, 326}, {30, 61, 62, 45, 59, 119}, {10, 13, 16, 30, 33, 23} }; diff --git a/core/hailo/libs/postprocesses/meson.build b/core/hailo/libs/postprocesses/meson.build index 9affcbe..d2accd9 100644 --- a/core/hailo/libs/postprocesses/meson.build +++ b/core/hailo/libs/postprocesses/meson.build @@ -17,6 +17,23 @@ shared_library('mobilenet_ssd_post', install_dir: post_proc_install_dir, ) +################################################ +# DETECTION SOURCES - YOLOV5M +################################################ +yolo_hailortpp_sources = [ + 'detection/yolo_hailortpp.cpp' +] + +shared_library('yolo_hailortpp_post', + yolo_hailortpp_sources, + cpp_args : hailo_lib_args, + include_directories: [hailo_general_inc, include_directories('./')], + dependencies : post_deps, + gnu_symbol_visibility : 'default', + install: true, + install_dir: post_proc_install_dir, +) + ################################################ # FACE DETECTION SOURCES ################################################ diff --git a/core/hailo/libs/postprocesses/ocr/ocr_postprocess.cpp b/core/hailo/libs/postprocesses/ocr/ocr_postprocess.cpp index 6ec7821..417e042 100644 --- a/core/hailo/libs/postprocesses/ocr/ocr_postprocess.cpp +++ b/core/hailo/libs/postprocesses/ocr/ocr_postprocess.cpp @@ -1,7 +1,7 @@ /** -* Copyright (c) 2021-2022 Hailo Technologies Ltd. All rights reserved. -* Distributed under the LGPL license (https://www.gnu.org/licenses/old-licenses/lgpl-2.1.txt) -**/ + * Copyright (c) 2021-2022 Hailo Technologies Ltd. All rights reserved. + * Distributed under the LGPL license (https://www.gnu.org/licenses/old-licenses/lgpl-2.1.txt) + **/ #include #include "xtensor/xarray.hpp" @@ -17,14 +17,14 @@ const char *OUTPUT_LAYER_NAME = "lprnet/conv31"; /** * @brief recognize the characters that are in the license plate - * + * * @param roi holds the network output data */ void OCR_postprocess(HailoROIPtr roi) { HailoTensorPtr net_output = roi->get_tensor(OUTPUT_LAYER_NAME); if (nullptr == net_output) - return; + return; xt::xarray output_dequantize; output_dequantize = common::get_xtensor_float(net_output); @@ -71,9 +71,10 @@ void OCR_postprocess(HailoROIPtr roi) } float conf_mean = std::accumulate(no_repeat_label_conf.begin(), no_repeat_label_conf.end(), 0.0) / no_repeat_label_conf.size(); - if (conf_mean >= MIN_SCORE_THRESHOLD && no_repeat_label.str().size() > MIN_CHARS) + { hailo_common::add_classification(roi, std::string("ocr"), no_repeat_label.str(), conf_mean); + } } void filter(HailoROIPtr roi) diff --git a/core/hailo/libs/postprocesses/pose_estimation/mspn.cpp b/core/hailo/libs/postprocesses/pose_estimation/mspn.cpp index c224942..106366b 100644 --- a/core/hailo/libs/postprocesses/pose_estimation/mspn.cpp +++ b/core/hailo/libs/postprocesses/pose_estimation/mspn.cpp @@ -9,7 +9,7 @@ #include "mspn.hpp" #include "common/tensors.hpp" -#include "common/json_config.hpp" +#include "json_config.hpp" #include "rapidjson/document.h" #include "rapidjson/stringbuffer.h" diff --git a/core/hailo/libs/postprocesses/semantic_segmentation/semantic_segmentation.cpp b/core/hailo/libs/postprocesses/semantic_segmentation/semantic_segmentation.cpp index aef456a..bc45a2a 100644 --- a/core/hailo/libs/postprocesses/semantic_segmentation/semantic_segmentation.cpp +++ b/core/hailo/libs/postprocesses/semantic_segmentation/semantic_segmentation.cpp @@ -5,7 +5,7 @@ #include "semantic_segmentation.hpp" #include "common/tensors.hpp" -const char *output_layer_name = "fcn8_resnet_v1_18/argmax1"; +const char *output_layer_name = "argmax1"; void semantic_segmentation(HailoROIPtr roi) { diff --git a/core/hailo/meson.build b/core/hailo/meson.build index aefc207..ba5d26a 100644 --- a/core/hailo/meson.build +++ b/core/hailo/meson.build @@ -1,7 +1,7 @@ # Project Declaration project('gst-hailo-tools', 'c', 'cpp', - version : '3.24.0', + version : '3.25.0', default_options : [ 'warning_level=1', 'buildtype=debugoptimized', 'c_std=c11', 'cpp_std=c++17'] diff --git a/core/hailo/plugins/common/hailomat.hpp b/core/hailo/plugins/common/hailomat.hpp index 44a29b5..2f783af 100644 --- a/core/hailo/plugins/common/hailomat.hpp +++ b/core/hailo/plugins/common/hailomat.hpp @@ -26,6 +26,7 @@ typedef enum { + HAILO_MAT_NONE = -1, HAILO_MAT_RGB, HAILO_MAT_RGBA, HAILO_MAT_YUY2, @@ -121,11 +122,18 @@ class HailoMat */ virtual cv::Mat crop(HailoROIPtr crop_roi) { - auto bbox = hailo_common::create_flattened_bbox(crop_roi->get_bbox(), crop_roi->get_scaling_bbox()); - cv::Rect rect = get_bounding_rect(bbox, m_width, m_height); + cv::Rect rect = get_crop_rect(crop_roi); cv::Mat cropped_cv_mat = get_mat()(rect); return cropped_cv_mat; } + + virtual cv::Rect get_crop_rect(HailoROIPtr crop_roi) + { + auto bbox = hailo_common::create_flattened_bbox(crop_roi->get_bbox(), crop_roi->get_scaling_bbox()); + cv::Rect rect = get_bounding_rect(bbox, m_width, m_height); + return rect; + } + /** * @brief Get the type of mat * @@ -439,9 +447,8 @@ class HailoNV12Mat : public HailoMat cv::blur(target_roi, target_roi, ksize); } - virtual cv::Mat crop(HailoROIPtr crop_roi) + virtual cv::Rect get_crop_rect(HailoROIPtr crop_roi) { - // Wrap the mat with Y and UV channel windows auto bbox = hailo_common::create_flattened_bbox(crop_roi->get_bbox(), crop_roi->get_scaling_bbox()); // The Y channel is packed before the U & V on it's own @@ -452,6 +459,14 @@ class HailoNV12Mat : public HailoMat y_rect.x = floor_to_even_number(y_rect.x); y_rect.y = floor_to_even_number(y_rect.y); + return y_rect; + } + + virtual cv::Mat crop(HailoROIPtr crop_roi) + { + // Wrap the mat with Y and UV channel windows + cv::Rect y_rect = get_crop_rect(crop_roi); + // The U and V channels are interlaced together after the Y channel, // so they need to be cropped separately cv::Rect uv_rect; diff --git a/core/hailo/plugins/common/image.cpp b/core/hailo/plugins/common/image.cpp index b907d1b..8d5d671 100644 --- a/core/hailo/plugins/common/image.cpp +++ b/core/hailo/plugins/common/image.cpp @@ -157,15 +157,25 @@ HailoBBox resize_letterbox_nv12(cv::Mat &cropped_image, cv::Mat &resized_image, return letterboxed_scale; } -std::shared_ptr get_mat_by_format(GstBuffer *buffer, GstVideoInfo *info, GstMapInfo *map, int line_thickness, int font_thickness) +std::shared_ptr get_mat_by_format(GstBuffer *buffer, GstVideoInfo *info, int line_thickness, int font_thickness) { std::shared_ptr hmat = nullptr; + GstVideoFrame frame; + bool success = gst_video_frame_map(&frame, info, buffer, GstMapFlags(GST_MAP_READ)); + if (!success) + { + gst_video_frame_unmap(&frame); + GST_CAT_ERROR(GST_CAT_DEFAULT, "Failed to map buffer to video frame, Buffer may be not writable"); + throw std::runtime_error("Failed to map buffer to video frame, Buffer may be not writable"); + } + + uint8_t *plane0_data = (uint8_t *)GST_VIDEO_FRAME_PLANE_DATA(&frame, 0); switch (GST_VIDEO_INFO_FORMAT(info)) { case GST_VIDEO_FORMAT_RGB: { - hmat = std::make_shared(map->data, + hmat = std::make_shared(plane0_data, GST_VIDEO_INFO_HEIGHT(info), GST_VIDEO_INFO_WIDTH(info), GST_VIDEO_INFO_PLANE_STRIDE(info, 0), @@ -175,7 +185,7 @@ std::shared_ptr get_mat_by_format(GstBuffer *buffer, GstVideoInfo *inf } case GST_VIDEO_FORMAT_RGBA: { - hmat = std::make_shared(map->data, + hmat = std::make_shared(plane0_data, GST_VIDEO_INFO_HEIGHT(info), GST_VIDEO_INFO_WIDTH(info), GST_VIDEO_INFO_PLANE_STRIDE(info, 0), @@ -185,7 +195,7 @@ std::shared_ptr get_mat_by_format(GstBuffer *buffer, GstVideoInfo *inf } case GST_VIDEO_FORMAT_YUY2: { - hmat = std::make_shared(map->data, + hmat = std::make_shared(plane0_data, GST_VIDEO_INFO_HEIGHT(info), GST_VIDEO_INFO_WIDTH(info), GST_VIDEO_INFO_PLANE_STRIDE(info, 0), @@ -195,40 +205,22 @@ std::shared_ptr get_mat_by_format(GstBuffer *buffer, GstVideoInfo *inf } case GST_VIDEO_FORMAT_NV12: { - - GstVideoFrame frame; - bool success = gst_video_frame_map(&frame, info, buffer, GstMapFlags(GST_MAP_READ)); - - if (success) - { - hmat = std::make_shared(map->data, - GST_VIDEO_INFO_HEIGHT(info), - GST_VIDEO_INFO_WIDTH(info), - GST_VIDEO_INFO_PLANE_STRIDE(info, 0), - GST_VIDEO_INFO_PLANE_STRIDE(info, 1), - line_thickness, - font_thickness, - GST_VIDEO_FRAME_PLANE_DATA(&frame, 0), - GST_VIDEO_FRAME_PLANE_DATA(&frame, 1)); - gst_video_frame_unmap(&frame); - } - else - { - hmat = std::make_shared(map->data, - GST_VIDEO_INFO_HEIGHT(info), - GST_VIDEO_INFO_WIDTH(info), - GST_VIDEO_INFO_PLANE_STRIDE(info, 0), - GST_VIDEO_INFO_PLANE_STRIDE(info, 1), - line_thickness, - font_thickness, - nullptr, - nullptr); - } + hmat = std::make_shared(plane0_data, + GST_VIDEO_INFO_HEIGHT(info), + GST_VIDEO_INFO_WIDTH(info), + GST_VIDEO_INFO_PLANE_STRIDE(info, 0), + GST_VIDEO_INFO_PLANE_STRIDE(info, 1), + line_thickness, + font_thickness, + plane0_data, + GST_VIDEO_FRAME_PLANE_DATA(&frame, 1)); break; } default: break; } + + gst_video_frame_unmap(&frame); return hmat; } \ No newline at end of file diff --git a/core/hailo/plugins/common/image.hpp b/core/hailo/plugins/common/image.hpp index 840a748..047268c 100644 --- a/core/hailo/plugins/common/image.hpp +++ b/core/hailo/plugins/common/image.hpp @@ -130,4 +130,4 @@ HailoBBox resize_letterbox_rgb(cv::Mat &cropped_image, cv::Mat &resized_image, c HailoBBox resize_letterbox_nv12(cv::Mat &cropped_image, cv::Mat &resized_image, cv::Scalar color, int interpolation = cv::INTER_LINEAR); __END_DECLS -std::shared_ptr get_mat_by_format(GstBuffer *buffer, GstVideoInfo *info, GstMapInfo *map, int line_thickness = 1, int font_thickness = 1); +std::shared_ptr get_mat_by_format(GstBuffer *buffer, GstVideoInfo *info, int line_thickness = 1, int font_thickness = 1); diff --git a/core/hailo/plugins/cropping/gsthailoaggregator.cpp b/core/hailo/plugins/cropping/gsthailoaggregator.cpp index 2c9dac5..184f692 100644 --- a/core/hailo/plugins/cropping/gsthailoaggregator.cpp +++ b/core/hailo/plugins/cropping/gsthailoaggregator.cpp @@ -7,7 +7,6 @@ * @title: hailoaggregator * * Takes packets from various input sinks into one output source. - * TODO: Add description * */ #include "cropping/gsthailoaggregator.hpp" @@ -62,6 +61,9 @@ static gboolean gst_hailoaggregator_sink_event(GstPad *pad, static GstFlowReturn gst_hailoaggregator_chain_main(GstPad *pad, GstObject *parent, GstBuffer *buf); static GstFlowReturn gst_hailoaggregator_chain_sub(GstPad *pad, GstObject *parent, GstBuffer *buf); +static gboolean gst_hailoaggregator_sink_query(GstPad *pad, + GstObject *parent, GstQuery *query); + static void gst_hailoaggregator_class_init(GstHailoAggregatorClass *klass) { @@ -97,6 +99,7 @@ gst_hailoaggregator_init(GstHailoAggregator *hailoaggregator) hailoaggregator->sinkpad_main = gst_pad_new_from_static_template(&sink_template, "sink_0"); gst_pad_set_event_function(hailoaggregator->sinkpad_main, GST_DEBUG_FUNCPTR(gst_hailoaggregator_sink_event)); gst_pad_set_chain_function(hailoaggregator->sinkpad_main, GST_DEBUG_FUNCPTR(gst_hailoaggregator_chain_main)); + gst_pad_set_query_function(hailoaggregator->sinkpad_main, GST_DEBUG_FUNCPTR(gst_hailoaggregator_sink_query)); GST_PAD_SET_PROXY_CAPS(hailoaggregator->sinkpad_main); gst_element_add_pad(GST_ELEMENT(hailoaggregator), hailoaggregator->sinkpad_main); @@ -173,6 +176,33 @@ forward_events(GstPad *pad, GstEvent **event, gpointer user_data) return TRUE; } +static gboolean gst_hailoaggregator_sink_query(GstPad *pad, + GstObject *parent, GstQuery *query) +{ + GstHailoAggregator *hailoaggregator = GST_HAILO_AGGREGATOR_CAST(parent); + gboolean ret = FALSE; + switch (GST_QUERY_TYPE(query)) + { + case GST_QUERY_ALLOCATION: + { + GST_DEBUG_OBJECT(hailoaggregator, "Received allocation query from sinkpad in hailoaggregator"); + ret = gst_pad_peer_query(hailoaggregator->srcpad, query); + if (!ret) + GST_DEBUG_OBJECT(hailoaggregator, "Failed to query peer for allocation"); + ret = true; + break; + } + default: + { + /* just call the default handler */ + ret = gst_pad_query_default(pad, parent, query); + break; + } + } + GST_DEBUG_OBJECT(hailoaggregator, "Received query from sinkpad in hailo aggregator %s", GST_QUERY_TYPE_NAME(query)); + return ret; +} + /** * Checks whether all sinkpads got eos. * diff --git a/core/hailo/plugins/cropping/gsthailobasecropper.cpp b/core/hailo/plugins/cropping/gsthailobasecropper.cpp index 8c76eff..12f7a81 100644 --- a/core/hailo/plugins/cropping/gsthailobasecropper.cpp +++ b/core/hailo/plugins/cropping/gsthailobasecropper.cpp @@ -18,6 +18,10 @@ #include "hailo_objects.hpp" #include "hailo_common.hpp" #include "gst_hailo_meta.hpp" +#ifdef HAILO15_TARGET + #include "dsp/gsthailodsp.h" + #include "dsp/gsthailodspbufferpoolutils.hpp" +#endif GST_DEBUG_CATEGORY_STATIC(gst_hailo_basecropper_debug); #define GST_CAT_DEFAULT gst_hailo_basecropper_debug @@ -29,6 +33,10 @@ enum PROP_DROP_UNCROPPED_BUFFERS, PROP_CROPPING_PERIOD, PROP_FILTER_STREAMS, + #ifdef HAILO15_TARGET + PROP_USE_DSP, + PROP_POOL_SIZE, + #endif }; static GstStaticPadTemplate sink_factory = GST_STATIC_PAD_TEMPLATE("sink", @@ -36,7 +44,14 @@ static GstStaticPadTemplate sink_factory = GST_STATIC_PAD_TEMPLATE("sink", GST_PAD_ALWAYS, gst_caps_from_string(HAILO_BASE_CROPPER_VIDEO_CAPS)); -static GstStaticPadTemplate src_factory = GST_STATIC_PAD_TEMPLATE("src", +// We define two source pad templates, one for the main stream and one for the cropped stream. +// Altough they are the same, we need to define them separately to support a proper caps negotiation in some platforms. +static GstStaticPadTemplate main_src_factory = GST_STATIC_PAD_TEMPLATE("src_0", + GST_PAD_SRC, + GST_PAD_ALWAYS, + gst_caps_from_string(HAILO_BASE_CROPPER_VIDEO_CAPS)); + +static GstStaticPadTemplate crop_src_factory = GST_STATIC_PAD_TEMPLATE("src_1", GST_PAD_SRC, GST_PAD_ALWAYS, gst_caps_from_string(HAILO_BASE_CROPPER_VIDEO_CAPS)); @@ -54,9 +69,23 @@ static gboolean gst_hailo_basecropper_sink_event(GstPad *pad, GstObject *parent, GstEvent *event); static gboolean gst_hailo_basecropper_sink_query(GstPad *pad, GstObject *parent, GstQuery *query); +static gboolean gst_hailo_basecropper_src_query(GstPad *pad, + GstObject *parent, GstQuery *query); static GstFlowReturn gst_hailo_basecropper_chain(GstPad *pad, GstObject *parent, GstBuffer *buf); +static void gst_hailo_basecropper_dispose(GObject *object); + +static gboolean gst_hailo_basecropper_decide_allocation(GstHailoBaseCropper *hailo_basecropper, GstQuery *query); + +static GstBuffer* gst_hailo_basecropper_allocate_new_buffer(GstHailoBaseCropper *hailo_basecropper, size_t buffer_size); + +#ifdef HAILO15_TARGET +static gboolean dsp_crop_and_resize(GstHailoBaseCropper *hailo_basecropper, cv::Rect crop_rect, std::shared_ptr resized_image, \ + GstBuffer *input_buffer, GstVideoInfo *input_video_info, GstBuffer *output_buffer, GstVideoInfo *output_video_info); +static gboolean gst_hailo_basecropper_propose_allocation(GstHailoBaseCropper *hailo_basecropper, GstPad *pad, GstQuery *query); +#endif + static void gst_hailo_basecropper_class_init(GstHailoBaseCropperClass *klass) { @@ -68,6 +97,7 @@ gst_hailo_basecropper_class_init(GstHailoBaseCropperClass *klass) gobject_class->set_property = gst_hailo_basecropper_set_property; gobject_class->get_property = gst_hailo_basecropper_get_property; + gobject_class->dispose = GST_DEBUG_FUNCPTR(gst_hailo_basecropper_dispose); g_object_class_install_property(gobject_class, PROP_USE_INTERNAL_OFFSET, g_param_spec_boolean("internal-offset", "Internal Offset", @@ -90,8 +120,22 @@ gst_hailo_basecropper_class_init(GstHailoBaseCropperClass *klass) (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS)), (GParamFlags)(G_PARAM_READWRITE | GST_PARAM_CONTROLLABLE | G_PARAM_STATIC_STRINGS))); + # ifdef HAILO15_TARGET + g_object_class_install_property(gobject_class, PROP_USE_DSP, + g_param_spec_boolean("use-dsp", "Use DSP", + "Whether to use DSP for cropping. Default true.", true, + (GParamFlags)(GST_PARAM_CONTROLLABLE | G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); + g_object_class_install_property(gobject_class, PROP_POOL_SIZE, + g_param_spec_uint("pool-size", "Pool Size", + "Size of the pool of buffers to use for cropping. Default 10", + 1, G_MAXINT, 10, + (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS | GST_PARAM_MUTABLE_READY))); + # endif + gst_element_class_add_pad_template(gstelement_class, - gst_static_pad_template_get(&src_factory)); + gst_static_pad_template_get(&main_src_factory)); + gst_element_class_add_pad_template(gstelement_class, + gst_static_pad_template_get(&crop_src_factory)); gst_element_class_add_pad_template(gstelement_class, gst_static_pad_template_get(&sink_factory)); @@ -109,25 +153,85 @@ gst_hailo_basecropper_init(GstHailoBaseCropper *hailo_basecropper) GST_PAD_SET_PROXY_CAPS(hailo_basecropper->sinkpad); gst_element_add_pad(GST_ELEMENT(hailo_basecropper), hailo_basecropper->sinkpad); - hailo_basecropper->srcpad_main = gst_pad_new_from_static_template(&src_factory, "src_0"); + hailo_basecropper->srcpad_main = gst_pad_new_from_static_template(&main_src_factory, "src_0"); GST_PAD_SET_PROXY_CAPS(hailo_basecropper->srcpad_main); gst_element_add_pad(GST_ELEMENT(hailo_basecropper), hailo_basecropper->srcpad_main); - hailo_basecropper->srcpad_crop = gst_pad_new_from_static_template(&src_factory, "src_1"); + hailo_basecropper->srcpad_crop = gst_pad_new_from_static_template(&crop_src_factory, "src_1"); GST_PAD_SET_PROXY_CAPS(hailo_basecropper->srcpad_crop); gst_element_add_pad(GST_ELEMENT(hailo_basecropper), hailo_basecropper->srcpad_crop); + gst_pad_set_query_function(hailo_basecropper->srcpad_crop, GST_DEBUG_FUNCPTR(gst_hailo_basecropper_src_query)); // Set default values. + #ifdef HAILO15_TARGET + hailo_basecropper->use_dsp = true; + hailo_basecropper->bufferpool_max_size = 10; + hailo_basecropper->bufferpool_min_size = 1; + #endif hailo_basecropper->use_internal_offset = false; hailo_basecropper->internal_offset = 0; hailo_basecropper->cropping_period = 1; hailo_basecropper->num_streams_to_filter = 0; hailo_basecropper->drop_uncropped_buffers = false; + hailo_basecropper->buffer_pool = NULL; hailo_basecropper->stream_ids_buff_offset.clear(); for (uint i = 0; i < GST_HAILO_CROPPER_MAX_FILTER_STREAMS; i++) hailo_basecropper->filter_streams[i] = ""; } +#ifdef HAILO15_TARGET +static gboolean +gst_hailo_basecropper_propose_allocation(GstHailoBaseCropper *hailo_basecropper, GstPad *pad, GstQuery *query) +{ + gboolean ret = gst_pad_peer_query(hailo_basecropper->srcpad_main, query); + if (!ret) + GST_DEBUG_OBJECT(hailo_basecropper, "Peer query allocation failed"); + gst_query_add_allocation_meta(query, GST_VIDEO_META_API_TYPE, NULL); + + return true; +} +#endif + +static void gst_hailo_basecropper_dispose(GObject *object) +{ + GstHailoBaseCropper *hailo_basecropper = GST_HAILO_BASE_CROPPER(object); + GST_INFO_OBJECT(hailo_basecropper, "Performing Dispose"); + + if (hailo_basecropper->buffer_pool) + { + GST_DEBUG_OBJECT(hailo_basecropper, "Unreffing buffer pool"); + gst_object_unref(hailo_basecropper->buffer_pool); + hailo_basecropper->buffer_pool = NULL; + } + + G_OBJECT_CLASS(gst_hailo_basecropper_parent_class)->dispose(object); +} + +static gboolean +gst_hailo_basecropper_decide_allocation(GstHailoBaseCropper *hailo_basecropper, GstQuery *query) +{ + gboolean ret = TRUE; + + #ifdef HAILO15_TARGET + if (!hailo_basecropper->use_dsp) + return ret; + + GST_DEBUG_OBJECT(hailo_basecropper, "Performing decide allocation"); + + GstElement *element = GST_ELEMENT_CAST(hailo_basecropper); + hailo_basecropper->buffer_pool = gst_create_hailo_dsp_bufferpool_from_allocation_query(element, query, hailo_basecropper->bufferpool_min_size, hailo_basecropper->bufferpool_max_size, 0); + if (hailo_basecropper->buffer_pool == NULL) + { + GST_ERROR_OBJECT(hailo_basecropper, "Decide Allocation - Failed to create buffer pool"); + return FALSE; + } + + GST_INFO_OBJECT(hailo_basecropper, "Decide allocation - hailo buffer pool created"); + + #endif + return ret; +} + static void set_filter_streams(GstHailoBaseCropper *hailo_basecropper, const GValue *value) { @@ -197,6 +301,14 @@ gst_hailo_basecropper_set_property(GObject *object, guint prop_id, case PROP_FILTER_STREAMS: set_filter_streams(hailo_basecropper, value); break; + #ifdef HAILO15_TARGET + case PROP_USE_DSP: + hailo_basecropper->use_dsp = g_value_get_boolean(value); + break; + case PROP_POOL_SIZE: + hailo_basecropper->bufferpool_max_size = g_value_get_uint(value); + break; + #endif default: G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec); break; @@ -223,6 +335,14 @@ gst_hailo_basecropper_get_property(GObject *object, guint prop_id, case PROP_FILTER_STREAMS: get_filter_streams(hailo_basecropper, value); break; + #ifdef HAILO15_TARGET + case PROP_USE_DSP: + g_value_set_boolean(value, hailo_basecropper->use_dsp); + break; + case PROP_POOL_SIZE: + g_value_set_uint(value, hailo_basecropper->bufferpool_max_size); + break; + #endif default: G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec); break; @@ -255,38 +375,83 @@ gst_crop_scale_setcaps(GstHailoBaseCropper *hailo_basecropper) // Set The caps, unref all objects and return. gboolean ret = gst_pad_set_caps(hailo_basecropper->srcpad_crop, outcaps); gst_query_unref(query); + return ret; } static gboolean -gst_hailo_basecropper_sink_query(GstPad *pad, +gst_hailo_handle_caps_query(GstPad *pad, GstQuery *query) +{ + GstCaps *caps_result, *allowed_caps, *qcaps; + /* we should report the supported caps here which are all */ + allowed_caps = gst_pad_get_pad_template_caps(pad); + gst_query_parse_caps(query, &qcaps); + + if (qcaps) + { + // caps query - intersect template caps (allowed caps) with incomming caps query + caps_result = gst_caps_intersect(allowed_caps, qcaps); + } + else + { + // no caps query - return template caps + caps_result = allowed_caps; + } + + // set the caps result + gst_query_set_caps_result(query, caps_result); + gst_caps_unref(caps_result); + + return TRUE; +} + +static gboolean +gst_hailo_basecropper_src_query(GstPad *pad, GstObject *parent, GstQuery *query) { gboolean ret; - GstCaps *caps_result, *allowed_caps, *qcaps; switch (GST_QUERY_TYPE(query)) { case GST_QUERY_CAPS: { - /* we should report the supported caps here which are all */ - allowed_caps = gst_pad_get_pad_template_caps(pad); - gst_query_parse_caps(query, &qcaps); + ret = gst_hailo_handle_caps_query(pad, query); + break; + } + default: + { + /* just call the default handler */ + ret = gst_pad_query_default(pad, parent, query); + break; + } + } + return ret; +} - if (qcaps) - { - // caps query - intersect template caps (allowed caps) with incomming caps query - caps_result = gst_caps_intersect(allowed_caps, qcaps); - gst_caps_unref(allowed_caps); - } - else - { - gst_caps_unref(allowed_caps); - caps_result = allowed_caps; - } +static gboolean +gst_hailo_basecropper_sink_query(GstPad *pad, + GstObject *parent, GstQuery *query) +{ + gboolean ret; + GstHailoBaseCropper *hailo_basecropper = GST_HAILO_BASE_CROPPER(parent); - gst_query_set_caps_result(query, caps_result); - ret = TRUE; + switch (GST_QUERY_TYPE(query)) + { + case GST_QUERY_CAPS: + { + ret = gst_hailo_handle_caps_query(pad, query); + break; + } + case GST_QUERY_ALLOCATION: + { + GST_DEBUG_OBJECT(hailo_basecropper, "Received allocation query from sinkpad in hailo_basecropper"); + #ifdef HAILO15_TARGET + ret = gst_hailo_basecropper_propose_allocation(hailo_basecropper, pad, query); + if (!ret) + GST_DEBUG_OBJECT(hailo_basecropper, "Failed to query peer srcpad_main"); + #else + ret = gst_pad_query_default(pad, parent, query); + #endif break; } case GST_QUERY_ACCEPT_CAPS: @@ -322,20 +487,40 @@ gst_hailo_basecropper_sink_event(GstPad *pad, GstObject *parent, case GST_EVENT_CAPS: { - GstCaps *caps; + GstCaps *caps, *crop_caps; gst_event_parse_caps(event, &caps); ret = gst_pad_set_caps(hailo_basecropper->srcpad_main, caps); if (!ret) { - GST_ELEMENT_ERROR(hailo_basecropper, STREAM, FAILED, - ("Unable to set main src caps"), (NULL)); + GST_ERROR_OBJECT(hailo_basecropper, "Unable to set caps on main srcpad"); return FALSE; } ret = gst_crop_scale_setcaps(hailo_basecropper); + if (!ret) + { + GST_ERROR_OBJECT(hailo_basecropper, "Unable to set caps on crop srcpad"); + return FALSE; + } + // Get caps from the crop pad + crop_caps = gst_pad_get_current_caps(hailo_basecropper->srcpad_crop); + + // Create new allocation query with the crop caps + GstQuery *crop_query = gst_query_new_allocation(crop_caps, FALSE); + + // Propose allocation to the crop srcpad + GST_DEBUG_OBJECT(hailo_basecropper, "Sending allocation query to the crop srcpad"); + if (!gst_pad_peer_query (hailo_basecropper->srcpad_crop, crop_query)) { + GST_DEBUG_OBJECT(hailo_basecropper, "Peer crop srcpad allocation query failed"); + } + + // Peer srcpad_crop has replied, perform decide allocation + gst_hailo_basecropper_decide_allocation(hailo_basecropper, crop_query); + + gst_query_unref(crop_query); break; } case GST_EVENT_STREAM_START: @@ -364,30 +549,185 @@ gst_hailo_basecropper_sink_event(GstPad *pad, GstObject *parent, return ret; } +#ifdef HAILO15_TARGET +dsp_interpolation_type_t get_dsp_interpolation_type_from_cv(GstHailoBaseCropper *hailo_basecropper, cv::InterpolationFlags interpolation) +{ + switch (interpolation) + { + case cv::INTER_NEAREST: + return INTERPOLATION_TYPE_NEAREST_NEIGHBOR; + case cv::INTER_LINEAR: + return INTERPOLATION_TYPE_BILINEAR; + case cv::INTER_AREA: + return INTERPOLATION_TYPE_AREA; + case cv::INTER_CUBIC: + return INTERPOLATION_TYPE_BICUBIC; + default: + GST_ERROR_OBJECT(hailo_basecropper, "Unsupported interpolation type %d", interpolation); + std::cerr << "Unsupported interpolation type " << interpolation << std::endl; + return INTERPOLATION_TYPE_BILINEAR; + } +} +#endif + +static GstBuffer* gst_hailo_basecropper_allocate_new_buffer(GstHailoBaseCropper *hailo_basecropper, size_t buffer_size) +{ + GstBuffer *output_buffer = NULL; + + #ifdef HAILO15_TARGET + if (hailo_basecropper->use_dsp) + { + if (!hailo_basecropper->buffer_pool) + { + GST_ERROR_OBJECT(hailo_basecropper, "DSP buffer allocation requested form pool - but buffer pool is not initialized"); + return NULL; + } + GstFlowReturn ret = GST_FLOW_OK; + ret = gst_buffer_pool_acquire_buffer(GST_BUFFER_POOL(hailo_basecropper->buffer_pool), &output_buffer, NULL); + if (ret != GST_FLOW_OK) + { + GST_ERROR_OBJECT(hailo_basecropper, "Failed to acquire buffer from pool"); + return NULL; + } + + } + else + { + output_buffer = gst_buffer_new_allocate(NULL, buffer_size, NULL); + } + #else + output_buffer = gst_buffer_new_allocate(NULL, buffer_size, NULL); + #endif + + return output_buffer; +} + +#ifdef HAILO15_TARGET +static gboolean dsp_crop_and_resize(GstHailoBaseCropper *hailo_basecropper, cv::Rect crop_rect, std::shared_ptr resized_image, \ + GstBuffer *input_buffer, GstVideoInfo *input_video_info, GstBuffer *output_buffer, GstVideoInfo *output_video_info) +{ + crop_resize_dims_t crop_resize_dims = { + .perform_crop = 1, + .crop_start_x = (size_t)crop_rect.x, + .crop_end_x = (size_t)crop_rect.x + crop_rect.width, + .crop_start_y = (size_t)crop_rect.y, + .crop_end_y = (size_t)crop_rect.y + crop_rect.height, + .destination_width = (size_t)resized_image->native_width(), + .destination_height = (size_t)resized_image->native_height(), + }; + + int input_width = GST_VIDEO_INFO_WIDTH(input_video_info); + int input_height = GST_VIDEO_INFO_HEIGHT(input_video_info); + + // Map input and output buffers to GstVideoFrame + GstVideoFrame input_video_frame; + if (!gst_video_frame_map(&input_video_frame, input_video_info, input_buffer, GST_MAP_READ)) + { + GST_ERROR_OBJECT(hailo_basecropper, "Cannot map input buffer to frame"); + throw std::runtime_error("Cannot map input buffer to frame"); + } + + if (!gst_buffer_is_writable(output_buffer)) + { + GST_ERROR_OBJECT(hailo_basecropper, "Output buffer is not writable"); + throw std::runtime_error("Output buffer is not writable"); + } + + GstVideoFrame output_video_frame; + if (!gst_video_frame_map(&output_video_frame, output_video_info, output_buffer, GST_MAP_READWRITE)) + { + GST_ERROR_OBJECT(hailo_basecropper, "Cannot map output buffer to frame"); + throw std::runtime_error("Cannot map output buffer to frame"); + } + GstVideoFormat format = GST_VIDEO_FRAME_FORMAT(&input_video_frame); + + // If the crop rect is the same as the input image (whole buffer), we request a resize only + if (crop_rect.x == 0 && crop_rect.y == 0 && crop_rect.width == input_width && crop_rect.height == input_height) + { + crop_resize_dims.perform_crop = 0; + GST_DEBUG_OBJECT(hailo_basecropper, "DSP Resize (Format: %d): Input Width: %d, Height: %d. \ + Resize target Width: %ld Height: %ld", format, input_width, input_height, + crop_resize_dims.destination_width, crop_resize_dims.destination_height); + } + else + { + GST_DEBUG_OBJECT(hailo_basecropper, "DSP Crop + Resize (Format: %d): Input Width: %d, Height: %d. \ + Target Crop shape X: %d Y: %d Width: %d Height: %d. and Resize target Width: %ld Height: %ld", + format, input_width, input_height, crop_rect.x, crop_rect.y, crop_rect.width, crop_rect.height, + crop_resize_dims.destination_width, crop_resize_dims.destination_height); + } + + // Create dsp image properties from both input and output video frame objects + dsp_image_properties_t input_image_properties = create_image_properties_from_video_frame(&input_video_frame); + dsp_image_properties_t output_image_properties = create_image_properties_from_video_frame(&output_video_frame); + + // Perform the crop and resize + dsp_status result = perform_dsp_crop_and_resize(&input_image_properties, &output_image_properties, crop_resize_dims, + get_dsp_interpolation_type_from_cv(hailo_basecropper, cv::InterpolationFlags::INTER_LINEAR)); + + // Free resources + free_image_property_planes(&input_image_properties); + free_image_property_planes(&output_image_properties); + gst_video_frame_unmap(&input_video_frame); + gst_video_frame_unmap(&output_video_frame); + + if (result != DSP_SUCCESS) + { + GST_ERROR_OBJECT(hailo_basecropper, "Failed to perform dsp resize. return status: %d", result); + return false; + } + + return true; +} +#endif + +static gboolean opencv_crop_and_resize(GstHailoBaseCropper *hailo_basecropper, std::shared_ptr resized_image, std::shared_ptr full_image, GstVideoInfo *full_image_info, HailoROIPtr crop_roi) +{ + GstHailoBaseCropperClass *hailo_basecropperclass = GST_HAILO_BASE_CROPPER_GET_CLASS(hailo_basecropper); + cv::Mat &resized_cv_mat = resized_image->get_mat(); + + GST_DEBUG_OBJECT(hailo_basecropper, "Opencv Crop + Resize: Input Width: %d, Height: %d. \ + Target Crop shape X: %f Y: %f Width: %f Height: %f. \ + Resize width %d height %d\n", full_image->width(), full_image->height(), + crop_roi->get_bbox().xmin(), crop_roi->get_bbox().ymin(), + crop_roi->get_bbox().width(), crop_roi->get_bbox().height(), resized_cv_mat.cols, resized_cv_mat.rows); + cv::Mat cropped_cv_mat = full_image->crop(crop_roi); + + GstVideoFormat image_format = GST_VIDEO_INFO_FORMAT(full_image_info); + hailo_basecropperclass->resize(hailo_basecropper, cropped_cv_mat, resized_cv_mat, crop_roi, image_format); + cropped_cv_mat.release(); + resized_cv_mat.release(); + return true; +} + /** * Create a new GstBuffer, crop and resize the frame to match the crop_roi, add the buffer to the metadata. * - * @param[in] hailo_basecropper cropping element. - * @param[in] buf Buffer to crop. - * @param[in] crop_roi Reference to a ROI Object to crop. + * @param[in] hailo_basecropper Cropping element. + * @param[in] input_buffer Buffer to crop & resize. + * @param[in] crop_roi Reference to a ROI Object to crop dimensions. * @return A new buffer, cropped and scaled for a second network. */ -static GstBuffer *handle_one_crop(GstHailoBaseCropper *hailo_basecropper, GstBuffer *buf, HailoROIPtr crop_roi) +static GstBuffer *handle_one_crop(GstHailoBaseCropper *hailo_basecropper, GstBuffer *input_buffer, HailoROIPtr crop_roi) { - GstHailoBaseCropperClass *hailo_basecropperclass = GST_HAILO_BASE_CROPPER_GET_CLASS(hailo_basecropper); - GstCaps *incaps, *outcaps; - GstBuffer *cropped_buf = NULL; + GstBuffer *output_buffer = NULL; incaps = gst_pad_get_current_caps(hailo_basecropper->sinkpad); + if (!incaps) + { + GST_ERROR_OBJECT(hailo_basecropper, "Failed to get input CAPS from sinkpad"); + return NULL; + } + outcaps = gst_pad_get_current_caps(hailo_basecropper->srcpad_crop); - // Allocate new GstBuffer - if (!incaps || !outcaps) + if (!outcaps) { + GST_ERROR_OBJECT(hailo_basecropper, "Failed to get output CAPS from srcpad (crop)"); gst_caps_unref(incaps); - gst_caps_unref(outcaps); - return cropped_buf; + return NULL; } + GstVideoInfo *full_image_info = gst_video_info_new(); gst_video_info_from_caps(full_image_info, incaps); @@ -397,47 +737,58 @@ static GstBuffer *handle_one_crop(GstHailoBaseCropper *hailo_basecropper, GstBuf HailoBBox roi_bbox = crop_roi->get_bbox(); bool crop_roi_is_whole_buffer = (roi_bbox.width() == 1.0f && roi_bbox.height() == 1.0f && roi_bbox.xmin() == 0.0f && roi_bbox.ymin() == 0.0f); bool input_res_equals_output_res = (full_image_info->width == resized_image_info->width && full_image_info->height == resized_image_info->height); + + // If the crop ROI is the whole buffer and the input and output resolutions are the same, we can just return a copy of the buffer if (crop_roi_is_whole_buffer && input_res_equals_output_res) { - GstBuffer *buffer_copy = gst_buffer_copy(buf); - gst_buffer_add_hailo_meta(buffer_copy, crop_roi); + GST_DEBUG_OBJECT(hailo_basecropper, "Crop ROI is the whole buffer and input and output resolutions are the same, returning a copy of the buffer"); + output_buffer = gst_buffer_ref(input_buffer); + gst_buffer_add_hailo_meta(output_buffer, crop_roi); gst_video_info_free(full_image_info); gst_video_info_free(resized_image_info); gst_caps_unref(incaps); gst_caps_unref(outcaps); - return buffer_copy; + return output_buffer; } - cropped_buf = gst_buffer_new_allocate(NULL, get_size(outcaps), NULL); + size_t buffer_size = get_size(outcaps); + GST_DEBUG_OBJECT(hailo_basecropper, "Allocating output buffer size: %d", (int)buffer_size); + // Allocate new GstBuffer + output_buffer = gst_hailo_basecropper_allocate_new_buffer(hailo_basecropper, buffer_size); + + // Get cv matrix of full image from buffer + std::shared_ptr full_image = get_mat_by_format(input_buffer, full_image_info); - // get cv matrix of full image from buffer - GstMapInfo full_image_map; - gst_buffer_map(buf, &full_image_map, GST_MAP_READ); - std::shared_ptr full_image = get_mat_by_format(buf, full_image_info, &full_image_map); - GstVideoFormat image_format = GST_VIDEO_INFO_FORMAT(full_image_info); - gst_video_info_free(full_image_info); + // Get cv matrix of cropped image from buffer + std::shared_ptr resized_image = get_mat_by_format(output_buffer, resized_image_info); - // get cv matrix of cropped image from buffer - GstMapInfo resized_image_map; - gst_buffer_map(cropped_buf, &resized_image_map, GST_MAP_READWRITE); - std::shared_ptr resized_image = get_mat_by_format(cropped_buf, resized_image_info, &resized_image_map); - gst_video_info_free(resized_image_info); + // Crop and resize the frame + #ifdef HAILO15_TARGET + if (hailo_basecropper->use_dsp) + { + cv::Rect crop_rect = full_image->get_crop_rect(crop_roi); + dsp_crop_and_resize(hailo_basecropper, crop_rect, resized_image, input_buffer, full_image_info, output_buffer, resized_image_info); + } + else + { + opencv_crop_and_resize(hailo_basecropper,resized_image, full_image, full_image_info, crop_roi); + } + #else + opencv_crop_and_resize(hailo_basecropper,resized_image, full_image, full_image_info, crop_roi); + #endif - // Crop and resize the the frame - cv::Mat cropped_cv_mat = full_image->crop(crop_roi); - cv::Mat &resized_cv_mat = resized_image->get_mat(); - hailo_basecropperclass->resize(hailo_basecropper, cropped_cv_mat, resized_cv_mat, crop_roi, image_format); + GST_DEBUG_OBJECT(hailo_basecropper, "Crop and resize done, freeing resources and returning buffer"); + + gst_video_info_free(full_image_info); + gst_video_info_free(resized_image_info); // Add the croopped ROI to the buffer - gst_buffer_add_hailo_meta(cropped_buf, crop_roi); + gst_buffer_add_hailo_meta(output_buffer, crop_roi); - cropped_cv_mat.release(); - resized_cv_mat.release(); gst_caps_unref(incaps); gst_caps_unref(outcaps); - gst_buffer_unmap(buf, &full_image_map); - gst_buffer_unmap(cropped_buf, &resized_image_map); - return cropped_buf; + + return output_buffer; } /** @@ -452,6 +803,11 @@ static gboolean handle_crops(GstHailoBaseCropper *hailo_basecropper, GstBuffer * { for (HailoROIPtr &crop_roi : crop_rois) { + if (!gst_pad_is_active(hailo_basecropper->srcpad_crop)) + { + GST_INFO_OBJECT(hailo_basecropper, "Crop src pad is not active, dropping buffer"); + return TRUE; + } GstBuffer *newbuf = handle_one_crop(hailo_basecropper, buf, crop_roi); if (!newbuf) { @@ -534,10 +890,14 @@ static GstFlowReturn gst_hailo_basecropper_chain(GstPad *pad, GstObject *parent, else { gst_pad_push(hailo_basecropper->srcpad_main, gst_buffer_ref(buf)); - handle_crops(hailo_basecropper, buf, crop_rois); + gboolean handle_crops_ret = handle_crops(hailo_basecropper, buf, crop_rois); gst_buffer_unref(buf); + if (!handle_crops_ret) + { + g_free(stream_id); + return GST_FLOW_ERROR; + } } - g_free(stream_id); return GST_FLOW_OK; } @@ -609,7 +969,6 @@ void resize_letterbox(cv::InterpolationFlags method, cv::Mat &cropped_image, cv::Mat &resized_image, HailoROIPtr roi, GstVideoFormat image_format) { - switch (image_format) { case GST_VIDEO_FORMAT_NV12: diff --git a/core/hailo/plugins/cropping/gsthailobasecropper.hpp b/core/hailo/plugins/cropping/gsthailobasecropper.hpp index c61c687..cdd392e 100644 --- a/core/hailo/plugins/cropping/gsthailobasecropper.hpp +++ b/core/hailo/plugins/cropping/gsthailobasecropper.hpp @@ -35,6 +35,12 @@ struct _GstHailoBaseCropper gboolean drop_uncropped_buffers; uint internal_offset; uint cropping_period; + #ifdef HAILO15_TARGET + bool use_dsp; + guint bufferpool_max_size; + guint bufferpool_min_size; + #endif + GstBufferPool *buffer_pool; uint num_streams_to_filter = 0; GstPad *sinkpad, *srcpad_crop, *srcpad_main; std::map stream_ids_buff_offset; diff --git a/core/hailo/plugins/cropping/gsthailocropper.cpp b/core/hailo/plugins/cropping/gsthailocropper.cpp index 36e59e4..47818b3 100644 --- a/core/hailo/plugins/cropping/gsthailocropper.cpp +++ b/core/hailo/plugins/cropping/gsthailocropper.cpp @@ -199,16 +199,13 @@ static std::vector gst_hailocropper_prepare_crops(GstHailoBaseCropp GstCaps *caps = gst_pad_get_current_caps(basecropper->sinkpad); GstVideoInfo *info = gst_video_info_new(); - GstMapInfo map; - gst_buffer_map(buf, &map, GST_MAP_READ); gst_video_info_from_caps(info, caps); - std::shared_ptr image = get_mat_by_format(buf, info, &map); + std::shared_ptr image = get_mat_by_format(buf, info); gst_video_info_free(info); std::vector crop_rois = hailocropper->handler(image, hailo_roi); gst_caps_unref(caps); - gst_buffer_unmap(buf, &map); return crop_rois; } diff --git a/core/hailo/plugins/dsp/gsthailodsp.c b/core/hailo/plugins/dsp/gsthailodsp.c new file mode 100644 index 0000000..d31b150 --- /dev/null +++ b/core/hailo/plugins/dsp/gsthailodsp.c @@ -0,0 +1,397 @@ +#include "gsthailodsp.h" +#include +#include +#include + +static dsp_device device = NULL; +static guint dsp_device_refcount = 0; + +static const char *interpolations_strings[INTERPOLATION_TYPE_COUNT] = { + [INTERPOLATION_TYPE_NEAREST_NEIGHBOR] = "nearest", + [INTERPOLATION_TYPE_BILINEAR] = "bilinear", + [INTERPOLATION_TYPE_AREA] = "area", + [INTERPOLATION_TYPE_BICUBIC] = "bicubic"}; + +/** + * Create a DSP device, and store it globally + * The function requests a device from the DSP library. + * @return dsp_status + */ +static dsp_status create_device() +{ + dsp_status create_device_status = DSP_UNINITIALIZED; + // If the device is not initialized, initialize it, else return SUCCESS + if (device == NULL) + { + GST_CAT_DEBUG(GST_CAT_DEFAULT, "Openning DSP device"); + create_device_status = dsp_create_device(&device); + if (create_device_status != DSP_SUCCESS) + { + GST_CAT_ERROR(GST_CAT_DEFAULT, "Open DSP device failed with status %d", create_device_status); + } + } + + return DSP_SUCCESS; +} + +/** + * Release the DSP device. + * The function releases the device using the DSP library. + * If there are other references to the device, just decrement the refcount and skip the release. + * @return dsp_status + */ +dsp_status release_device() +{ + if (device == NULL) + { + GST_CAT_WARNING(GST_CAT_DEFAULT, "Release device skipped: Dsp device is already NULL"); + return DSP_SUCCESS; + } + + dsp_device_refcount--; + if (dsp_device_refcount > 0) + { + GST_CAT_INFO(GST_CAT_DEFAULT, "Release dsp device skipped, refcount is %d", dsp_device_refcount); + } + else + { + GST_CAT_DEBUG(GST_CAT_DEFAULT, "Releasing dsp device, refcount is %d", dsp_device_refcount); + dsp_status status = dsp_release_device(device); + if (status != DSP_SUCCESS) + { + GST_CAT_ERROR(GST_CAT_DEFAULT, "Release device failed with status %d", status); + return status; + } + device = NULL; + GST_CAT_INFO(GST_CAT_DEFAULT, "Dsp device released successfully"); + } + + return DSP_SUCCESS; +} + +/** + * Acquire the DSP device. + * This function creates the DSP device using the DSP library once, and then increases the reference count. + * + * @return dsp_status + */ +dsp_status acquire_device() +{ + if (device == NULL) + { + dsp_status status = create_device(); + if (status != DSP_SUCCESS) + return status; + } + dsp_device_refcount++; + GST_CAT_DEBUG(GST_CAT_DEFAULT, "Acquired dsp device, refcount is %d", dsp_device_refcount); + return DSP_SUCCESS; +} + +/** + * Create a buffer on the DSP + * The function requests a buffer from the DSP library. + * The buffer can be used later for DSP operations. + * @param[in] size the size of the buffer to create + * @param[out] buffer a pointer to a buffer - DSP library will allocate the buffer + * @return dsp_status + */ +dsp_status create_hailo_dsp_buffer(size_t size, void **buffer) +{ + if (device != NULL) + { + GST_CAT_DEBUG(GST_CAT_DEFAULT, "Creating dsp buffer with size %ld", size); + dsp_status status = dsp_create_buffer(device, size, buffer); + if (status != DSP_SUCCESS) + { + GST_CAT_ERROR(GST_CAT_DEFAULT, "Create buffer failed with status %d", status); + return status; + } + } + else + { + GST_CAT_ERROR(GST_CAT_DEFAULT, "Create buffer failed: device is NULL"); + return DSP_UNINITIALIZED; + } + + return DSP_SUCCESS; +} + +/** + * Release a buffer allocated by the DSP + * @param[in] buffer the buffer to release + * @return dsp_status + */ +dsp_status release_hailo_dsp_buffer(void *buffer) +{ + if (device == NULL) + { + GST_CAT_ERROR(GST_CAT_DEFAULT, "DSP release buffer failed: device is NULL"); + return DSP_UNINITIALIZED; + } + + GST_CAT_DEBUG(GST_CAT_DEFAULT, "performing DSP buffer release"); + dsp_status status = dsp_release_buffer(device, buffer); + if (status != DSP_SUCCESS) + { + GST_CAT_ERROR(GST_CAT_DEFAULT, "DSP release buffer failed with status %d", status); + return status; + } + + GST_CAT_DEBUG(GST_CAT_DEFAULT, "DSP buffer released successfully"); + return DSP_SUCCESS; +} + +/** + * Perform DSP Resize + * The function calls the DSP library to perform resize on a given buffer. + * DSP will place the result in the output buffer. + * + * @param[in] input_image_properties input image properties + * @param[in] output_image_properties output image properties + * @param[in] dsp_interpolation_type interpolation type to use + * @return dsp_status + */ +dsp_status perform_dsp_resize(dsp_image_properties_t *input_image_properties, dsp_image_properties_t *output_image_properties, dsp_interpolation_type_t dsp_interpolation_type) +{ + if (device == NULL) + { + GST_CAT_ERROR(GST_CAT_DEFAULT, "Perform DSP crop and resize ERROR: Device is NULL"); + return DSP_UNINITIALIZED; + } + + dsp_resize_params_t resize_params = { + .src = input_image_properties, + .dst = output_image_properties, + .interpolation = dsp_interpolation_type, + }; + + GST_CAT_DEBUG(GST_CAT_DEFAULT, + "Perform DSP resize %s (no crop) to destination width: %ld, destination height: %ld", + interpolations_strings[dsp_interpolation_type], output_image_properties->width, output_image_properties->height); + dsp_status status = dsp_resize(device, &resize_params); + + if (status != DSP_SUCCESS) + { + GST_CAT_ERROR(GST_CAT_DEFAULT, "DSP Resize command failed with status %d", status); + return status; + } + + GST_CAT_DEBUG(GST_CAT_DEFAULT, "DSP Resize command completed successfully"); + return DSP_SUCCESS; +} + +/** + * Perform DSP crop and resize + * The function calls the DSP library to perform crop and resize on a given buffer. + * DSP will place the result in the output buffer. + * + * @param[in] input_image_properties input image properties + * @param[in] output_image_properties output image properties + * @param[in] args crop and resize arguments + * @param[in] dsp_interpolation_type interpolation type to use + * @return dsp_status + */ +dsp_status perform_dsp_crop_and_resize(dsp_image_properties_t *input_image_properties, dsp_image_properties_t *output_image_properties, + crop_resize_dims_t args, dsp_interpolation_type_t dsp_interpolation_type) +{ + if (device == NULL) + { + GST_CAT_ERROR(GST_CAT_DEFAULT, "Perform DSP crop and resize ERROR: Device is NULL"); + return DSP_UNINITIALIZED; + } + + dsp_resize_params_t resize_params = { + .src = input_image_properties, + .dst = output_image_properties, + .interpolation = dsp_interpolation_type, + }; + + dsp_status status; + if (args.perform_crop) + { + GST_CAT_DEBUG(GST_CAT_DEFAULT, + "Perform DSP crop & resize %s to destination width: %ld, destination height: %ld, crop: (%ld,%ld)-(%ld-%ld)", + interpolations_strings[dsp_interpolation_type], args.destination_width, args.destination_height, + args.crop_start_x, args.crop_start_y, args.crop_end_x, args.crop_end_y); + + dsp_crop_api_t crop_params = { + .start_x = args.crop_start_x, + .start_y = args.crop_start_y, + .end_x = args.crop_end_x, + .end_y = args.crop_end_y, + }; + status = dsp_crop_and_resize(device, &resize_params, &crop_params); + } + else + { + GST_CAT_DEBUG(GST_CAT_DEFAULT, + "Perform DSP resize %s (no crop) to destination width: %ld, destination height: %ld", + interpolations_strings[dsp_interpolation_type], args.destination_width, args.destination_height); + status = dsp_resize(device, &resize_params); + } + + if (status != DSP_SUCCESS) + { + GST_CAT_ERROR(GST_CAT_DEFAULT, "DSP Crop & resize command failed with status %d", status); + return status; + } + + GST_CAT_DEBUG(GST_CAT_DEFAULT, "DSP Crop & resize command completed successfully"); + return DSP_SUCCESS; +} + +/** + * Perform DSP blending using multiple overlays + * The function calls the DSP library to perform blending between one + * main buffer and multiple overlay buffers. + * DSP will blend the overlay buffers onto the image frame in place + * + * @param[in] image_frame pointer to input image to blend on + * @param[in] overlay pointer to input images to overlay with + * @param[in] overlays_count number of overlays to blend + * @return dsp_status + */ +dsp_status perform_dsp_multiblend(dsp_image_properties_t *image_frame, dsp_overlay_properties_t *overlay, size_t overlays_count) +{ + return dsp_blend(device, image_frame, overlay, overlays_count); +} + +/** + * Free DSP struct resources + * + * @param[in] overlay_properties pointer to the properties to free + * @return void + */ +void free_overlay_property_planes(dsp_overlay_properties_t *overlay_properties) +{ + free_image_property_planes(&(overlay_properties->overlay)); +} + +/** + * Free DSP struct resources + * + * @param[in] image_properties pointer to the properties to free + * @return void + */ +void free_image_property_planes(dsp_image_properties_t *image_properties) +{ + free(image_properties->planes); +} + +/** + * Creates and populates a dsp_image_properties_t + * struct with data of a given GstVideoFrame + * + * @param[in] video_frame Gst video frame with plane data + * @return populated dsp_image_properties_t of the video frame + */ +dsp_image_properties_t create_image_properties_from_video_frame(GstVideoFrame *video_frame) +{ + GstVideoFormat format = GST_VIDEO_FRAME_FORMAT(video_frame); + dsp_image_properties_t image_properties; + size_t image_width = GST_VIDEO_FRAME_WIDTH(video_frame); + size_t image_height = GST_VIDEO_FRAME_HEIGHT(video_frame); + size_t n_planes = GST_VIDEO_FRAME_N_PLANES(video_frame); + + switch (format) + { + case GST_VIDEO_FORMAT_RGB: + { + // RGB is non-planar, since all channels are interleaved, we treat the whole image as 1 plane + void *data = (void *)GST_VIDEO_FRAME_PLANE_DATA(video_frame, 0); + size_t input_line_stride = GST_VIDEO_FRAME_PLANE_STRIDE(video_frame, 0); + size_t input_size = GST_VIDEO_FRAME_SIZE(video_frame); + + // Allocate memory for the plane + dsp_data_plane_t *plane = malloc(sizeof(*plane)); + plane->ptr = data; + plane->line_stride = input_line_stride; + plane->plane_size = input_size; + + // Fill in dsp_image_properties_t values + image_properties = (dsp_image_properties_t){ + .width = image_width, + .height = image_height, + .planes = plane, + .planes_count = n_planes, + .format = DSP_IMAGE_FORMAT_RGB}; + break; + } + case GST_VIDEO_FORMAT_YUY2: + { + GST_CAT_ERROR(GST_CAT_DEFAULT, "DSP image properties from GstVideoFrame failed: YUY2 not yet supported."); + break; + } + case GST_VIDEO_FORMAT_NV12: + { + // NV12 is semi-planar, where the Y channel is a seprate plane from the UV channels + // Gather y channel info + void *y_channel_data = (void *)GST_VIDEO_FRAME_PLANE_DATA(video_frame, 0); + size_t y_channel_stride = GST_VIDEO_FRAME_PLANE_STRIDE(video_frame, 0); + size_t y_channel_size = y_channel_stride * image_height; + dsp_data_plane_t y_plane_data = { + .ptr = y_channel_data, + .line_stride = y_channel_stride, + .plane_size = y_channel_size, + }; + // Gather uv channel info + void *uv_channel_data = (void *)GST_VIDEO_FRAME_PLANE_DATA(video_frame, 1); + size_t uv_channel_stride = GST_VIDEO_FRAME_PLANE_STRIDE(video_frame, 1); + size_t uv_channel_size = uv_channel_stride * image_height / 2; + dsp_data_plane_t uv_plane_data = { + .ptr = uv_channel_data, + .line_stride = uv_channel_stride, + .plane_size = uv_channel_size, + }; + dsp_data_plane_t *yuv_planes = malloc(sizeof(*yuv_planes) * 2); + yuv_planes[0] = y_plane_data; + yuv_planes[1] = uv_plane_data; + + GST_CAT_DEBUG(GST_CAT_DEFAULT, "DSP image properties from GstVideoFrame: buffer offset = %zu", GST_BUFFER_OFFSET(video_frame->buffer)); + GST_CAT_DEBUG(GST_CAT_DEFAULT, "DSP image properties from GstVideoFrame: NV12, y ptr %p, y stride %zu, y size %zu, uv ptr %p, uv stride %zu, uv size %zu", + y_channel_data, y_channel_stride, y_channel_size, uv_channel_data, uv_channel_stride, uv_channel_size); + // Fill in dsp_image_properties_t values + image_properties = (dsp_image_properties_t){ + .width = image_width, + .height = image_height, + .planes = yuv_planes, + .planes_count = n_planes, + .format = DSP_IMAGE_FORMAT_NV12}; + break; + } + case GST_VIDEO_FORMAT_A420: + { + // A420 is fully planar (4:4:2:0), essentially I420 YUV with an extra alpha channel at full size + // Gather channel info + dsp_data_plane_t *a420_planes = malloc(sizeof(*a420_planes) * n_planes); + for (guint i = 0; i < n_planes; ++i) + { + void *channel_data = (void *)GST_VIDEO_FRAME_PLANE_DATA(video_frame, i); + size_t channel_stride = GST_VIDEO_FRAME_PLANE_STRIDE(video_frame, i); + size_t channel_size = channel_stride * image_height; + if (i == 1 || i == 2) + channel_size /= 2; + dsp_data_plane_t plane_data = { + .ptr = channel_data, + .line_stride = channel_stride, + .plane_size = channel_size, + }; + a420_planes[i] = plane_data; + } + + // Fill in dsp_image_properties_t values + image_properties = (dsp_image_properties_t){ + .width = image_width, + .height = image_height, + .planes = a420_planes, + .planes_count = n_planes, + .format = DSP_IMAGE_FORMAT_A420}; + break; + } + default: + break; + } + + return image_properties; +} diff --git a/core/hailo/plugins/dsp/gsthailodsp.h b/core/hailo/plugins/dsp/gsthailodsp.h new file mode 100644 index 0000000..28854b5 --- /dev/null +++ b/core/hailo/plugins/dsp/gsthailodsp.h @@ -0,0 +1,48 @@ +#pragma once + +#include "hailo/hailodsp.h" +#include +#include + +#define ARRAY_LENGTH(x) (sizeof(x) / sizeof((x)[0])) + +#undef __BEGIN_DECLS +#undef __END_DECLS +#ifdef __cplusplus +#define __BEGIN_DECLS \ + extern "C" \ + { +#define __END_DECLS } +#else +#define __BEGIN_DECLS /* empty */ +#define __END_DECLS /* empty */ +#endif + + +__BEGIN_DECLS + +typedef struct { + int perform_crop; + size_t crop_start_x; + size_t crop_end_x; + size_t crop_start_y; + size_t crop_end_y; + size_t destination_width; + size_t destination_height; +} crop_resize_dims_t; + +dsp_status create_hailo_dsp_buffer(size_t size, void **buffer); +dsp_status release_hailo_dsp_buffer(void *buffer); +dsp_status release_device(); +dsp_status acquire_device(); +dsp_image_properties_t create_image_properties_from_video_frame(GstVideoFrame *video_frame); +dsp_status perform_dsp_resize(dsp_image_properties_t *input_image_properties, dsp_image_properties_t *output_image_properties, dsp_interpolation_type_t dsp_interpolation_type); +dsp_status perform_dsp_crop_and_resize(dsp_image_properties_t *input_image_properties, dsp_image_properties_t *output_image_properties, + crop_resize_dims_t args, dsp_interpolation_type_t dsp_interpolation_type); +dsp_status perform_dsp_multiblend(dsp_image_properties_t *image_frame, dsp_overlay_properties_t *overlay, size_t overlays_count); +void free_overlay_property_planes(dsp_overlay_properties_t *overlay_properties); +void free_image_property_planes(dsp_image_properties_t *image_properties); + +extern dsp_interpolation_type_t dsp_interpolation_type; +extern dsp_status dsp_command_status; +__END_DECLS diff --git a/core/hailo/plugins/dsp/gsthailodspbasetransform.cpp b/core/hailo/plugins/dsp/gsthailodspbasetransform.cpp new file mode 100644 index 0000000..c15e3a9 --- /dev/null +++ b/core/hailo/plugins/dsp/gsthailodspbasetransform.cpp @@ -0,0 +1,206 @@ +#include "dsp/gsthailodspbasetransform.hpp" +#include "dsp/gsthailodspbufferpool.hpp" +#include "dsp/gsthailodspbufferpoolutils.hpp" +#include + +GST_DEBUG_CATEGORY_STATIC(gst_hailo_dsp_base_transform_debug); +#define GST_CAT_DEFAULT gst_hailo_dsp_base_transform_debug + +enum +{ + PROP_0, + PROP_POOL_SIZE, +}; + +#define _debug_init \ + GST_DEBUG_CATEGORY_INIT(gst_hailo_dsp_base_transform_debug, "hailodspbasetransform", 0, "Hailo DSP base transform element"); + +#define gst_hailo_dsp_base_transform_parent_class parent_class +G_DEFINE_ABSTRACT_TYPE_WITH_CODE(GstHailoDspBaseTransform, gst_hailo_dsp_base_transform, GST_TYPE_BASE_TRANSFORM, _debug_init); + +static gboolean +gst_hailo_dsp_base_transform_propose_allocation(GstBaseTransform *trans, + GstQuery *decide_query, GstQuery *query); +static void gst_hailo_dsp_base_transform_set_property(GObject *object, + guint property_id, const GValue *value, GParamSpec *pspec); +static void gst_hailo_dsp_base_transform_get_property(GObject *object, + guint property_id, GValue *value, GParamSpec *pspec); +static gboolean gst_hailo_dsp_base_transform_decide_allocation(GstBaseTransform *trans, GstQuery *query); +static GstFlowReturn gst_hailo_dsp_base_prepare_output_buffer(GstBaseTransform *trans, + GstBuffer *inbuf, GstBuffer **outbuf); + +static void +gst_hailo_dsp_base_transform_class_init(GstHailoDspBaseTransformClass *klass) +{ + GObjectClass *const object_class = G_OBJECT_CLASS(klass); + GstBaseTransformClass *const base_transform_class = GST_BASE_TRANSFORM_CLASS(klass); + + object_class->set_property = gst_hailo_dsp_base_transform_set_property; + object_class->get_property = gst_hailo_dsp_base_transform_get_property; + + base_transform_class->propose_allocation = GST_DEBUG_FUNCPTR(gst_hailo_dsp_base_transform_propose_allocation); + base_transform_class->decide_allocation = GST_DEBUG_FUNCPTR(gst_hailo_dsp_base_transform_decide_allocation); + base_transform_class->prepare_output_buffer = GST_DEBUG_FUNCPTR(gst_hailo_dsp_base_prepare_output_buffer); + + gst_element_class_set_static_metadata(GST_ELEMENT_CLASS(klass), + "Hailo DSP base transform", + "Hailo/Tools", + "Base class for Hailo15 DSP transformations", + "hailo.ai "); + + g_object_class_install_property(object_class, PROP_POOL_SIZE, + g_param_spec_uint("pool-size", "Pool Size", + "Size of the pool of buffers to use for cropping. Default 10", + 1, G_MAXINT, 10, + (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS | GST_PARAM_MUTABLE_READY))); +} + +static void +gst_hailo_dsp_base_transform_init(GstHailoDspBaseTransform *self) +{ + self->bufferpool_max_size = 10; + self->bufferpool_min_size = 1; + self->pool_is_active = FALSE; + self->bufferpool_padding = 0; +} + +static gboolean +gst_hailo_dsp_base_transform_propose_allocation(GstBaseTransform *trans, + GstQuery *decide_query, GstQuery *query) +{ + gboolean ret; + + gst_query_add_allocation_meta(query, GST_VIDEO_META_API_TYPE, NULL); + ret = GST_BASE_TRANSFORM_CLASS(parent_class)->propose_allocation(trans, decide_query, query); + return ret; +} + +static GstFlowReturn +gst_hailo_dsp_base_prepare_output_buffer(GstBaseTransform *trans, + GstBuffer *inbuf, GstBuffer **outbuf) +{ + GstFlowReturn ret = GST_FLOW_OK; + GstBaseTransformClass *base_transform_class; + + base_transform_class = GST_BASE_TRANSFORM_GET_CLASS(trans); + GstHailoDspBaseTransform *hailo_dsp_transform = GST_HAILO_DSP_BASE_TRANSFORM(trans); + + GST_DEBUG_OBJECT(hailo_dsp_transform, "Preparing DSP output buffer"); + + if (gst_base_transform_is_passthrough(trans)) + { + // Passthrough mdoe - modify the incoming buffer, output = input + GST_DEBUG_OBJECT(trans, "Passthrough: reusing input buffer"); + *outbuf = inbuf; + return GST_FLOW_OK; + } + + GST_DEBUG_OBJECT(hailo_dsp_transform, "NON passthrough: allocating new buffer"); + + GstBufferPool *buffer_pool = gst_base_transform_get_buffer_pool(trans); + + if (!buffer_pool) + { + GST_ERROR_OBJECT(hailo_dsp_transform, "DSP buffer allocation requested form pool - but buffer pool is not initialized"); + return GST_FLOW_ERROR; + } + + if (!hailo_dsp_transform->pool_is_active) + { + GST_DEBUG_OBJECT(hailo_dsp_transform, "Setting pool %p active", buffer_pool); + if (!gst_buffer_pool_set_active(buffer_pool, TRUE)) + { + GST_ERROR_OBJECT(hailo_dsp_transform, "Failed to activate buffer pool"); + return GST_FLOW_ERROR; + } + hailo_dsp_transform->pool_is_active = TRUE; + } + + GST_DEBUG_OBJECT(hailo_dsp_transform, "Buffer pool is active, Acquiring buffer from pool %p", buffer_pool); + + ret = gst_buffer_pool_acquire_buffer(GST_BUFFER_POOL(buffer_pool), outbuf, NULL); + if (ret != GST_FLOW_OK) + { + GST_ERROR_OBJECT(hailo_dsp_transform, "Failed to acquire buffer from pool"); + return ret; + } + + GST_DEBUG_OBJECT(hailo_dsp_transform, "Acquired buffer %p from pool", *outbuf); + + // copy the metadata + if (base_transform_class->copy_metadata) + { + if (!base_transform_class->copy_metadata(trans, inbuf, *outbuf)) + { + GST_ELEMENT_WARNING(trans, STREAM, NOT_IMPLEMENTED, + ("could not copy metadata"), (NULL)); + } + } + + return ret; +} + +static gboolean +gst_hailo_dsp_base_transform_decide_allocation(GstBaseTransform *trans, + GstQuery *query) +{ + GstHailoDspBaseTransform *self = GST_HAILO_DSP_BASE_TRANSFORM(trans); + GstElement *element = GST_ELEMENT_CAST(self); + + GST_DEBUG_OBJECT(self, "Performing decide allocation"); + + GstBufferPool *pool = gst_create_hailo_dsp_bufferpool_from_allocation_query(element, query, self->bufferpool_min_size, self->bufferpool_max_size, self->bufferpool_padding); + GST_DEBUG_OBJECT(self, "Created hailo buffer pool %p", pool); + if (pool == NULL) + { + GST_ERROR_OBJECT(self, "Decide Allocation - Failed to create buffer pool"); + return FALSE; + } + GstBufferPool *existing_pool = gst_base_transform_get_buffer_pool(trans); + GST_DEBUG_OBJECT(self, "Existing pool %p", existing_pool); + + // Replace existing pool with the new one + if (existing_pool) + { + gst_object_unref(existing_pool); + } + existing_pool = pool; + GST_DEBUG_OBJECT(self, "Existing pool moved to %p", existing_pool); + + self->pool_is_active = TRUE; + gst_object_unref(pool); + GST_INFO_OBJECT(self, "Decide allocation done - hailo buffer pool created"); + return TRUE; +} + +static void +gst_hailo_dsp_base_transform_set_property(GObject *object, guint property_id, const GValue *value, GParamSpec *pspec) +{ + GstHailoDspBaseTransform *self = GST_HAILO_DSP_BASE_TRANSFORM(object); + + switch (property_id) + { + case PROP_POOL_SIZE: + self->bufferpool_max_size = g_value_get_uint(value); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID(object, property_id, pspec); + break; + } +} + +static void +gst_hailo_dsp_base_transform_get_property(GObject *object, guint property_id, GValue *value, GParamSpec *pspec) +{ + GstHailoDspBaseTransform *self = GST_HAILO_DSP_BASE_TRANSFORM(object); + + switch (property_id) + { + case PROP_POOL_SIZE: + g_value_set_uint(value, self->bufferpool_max_size); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID(object, property_id, pspec); + break; + } +} \ No newline at end of file diff --git a/core/hailo/plugins/dsp/gsthailodspbasetransform.hpp b/core/hailo/plugins/dsp/gsthailodspbasetransform.hpp new file mode 100644 index 0000000..6d1915d --- /dev/null +++ b/core/hailo/plugins/dsp/gsthailodspbasetransform.hpp @@ -0,0 +1,44 @@ +#pragma once + +#include +#include + +G_BEGIN_DECLS + +#define GST_TYPE_HAILO_DSP_BASE_TRANSFORM \ + (gst_hailo_dsp_base_transform_get_type()) +#define GST_HAILO_DSP_BASE_TRANSFORM(obj) \ + (G_TYPE_CHECK_INSTANCE_CAST((obj), GST_TYPE_HAILO_DSP_BASE_TRANSFORM, GstHailoDspBaseTransform)) +#define GST_HAILO_DSP_BASE_TRANSFORM_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_CAST((klass), GST_TYPE_HAILO_DSP_BASE_TRANSFORM, GstHailoDspBaseTransformClass)) +#define GST_IS_HAILO_DSP_BASE_TRANSFORM(obj) \ + (G_TYPE_CHECK_INSTANCE_TYPE((obj), GST_TYPE_HAILO_DSP_BASE_TRANSFORM)) +#define GST_IS_HAILO_DSP_BASE_TRANSFORM_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_TYPE((klass), GST_TYPE_HAILO_DSP_BASE_TRANSFORM)) +#define GST_HAILO_DSP_BASE_TRANSFORM_CAST(obj) \ + ((GstHailoDspBaseTransform *)(obj)) +#define GST_HAILO_DSP_BASE_TRANSFORM_GET_CLASS(obj) \ + (G_TYPE_INSTANCE_GET_CLASS((obj), GST_TYPE_HAILO_DSP_BASE_TRANSFORM, GstHailoDspBaseTransformClass)) + +typedef struct _GstHailoDspBaseTransform GstHailoDspBaseTransform; +typedef struct _GstHailoDspBaseTransformClass GstHailoDspBaseTransformClass; + +struct _GstHailoDspBaseTransform +{ + GstBaseTransform parent; + gboolean pool_is_active; + guint bufferpool_max_size; + guint bufferpool_min_size; + guint bufferpool_padding; + // add any additional instance variables here +}; + +struct _GstHailoDspBaseTransformClass +{ + GstBaseTransformClass parent_class; + // add any additional class variables here +}; + +G_GNUC_INTERNAL GType gst_hailo_dsp_base_transform_get_type(void); + +G_END_DECLS \ No newline at end of file diff --git a/core/hailo/plugins/dsp/gsthailodspbufferpool.cpp b/core/hailo/plugins/dsp/gsthailodspbufferpool.cpp new file mode 100644 index 0000000..4274b77 --- /dev/null +++ b/core/hailo/plugins/dsp/gsthailodspbufferpool.cpp @@ -0,0 +1,123 @@ +#include + +#include "dsp/gsthailodspbufferpool.hpp" +#include "dsp/gsthailodsp.h" + +G_DEFINE_TYPE(GstHailoDspBufferPool, gst_hailo_dsp_buffer_pool, GST_TYPE_BUFFER_POOL) + +static GstFlowReturn gst_hailo_dsp_buffer_pool_alloc_buffer(GstBufferPool *pool, GstBuffer **output_buffer_ptr, GstBufferPoolAcquireParams *params); +static void gst_hailo_dsp_buffer_pool_free_buffer(GstBufferPool *pool, GstBuffer *buffer); +static void gst_hailo_dsp_buffer_pool_dispose(GObject *object); + +static void +gst_hailo_dsp_buffer_pool_class_init(GstHailoDspBufferPoolClass *klass) +{ + GObjectClass *const object_class = G_OBJECT_CLASS(klass); + GstBufferPoolClass *const pool_class = GST_BUFFER_POOL_CLASS(klass); + + GST_INFO_OBJECT(object_class, "Hailo DSP buffer pool class init"); + + pool_class->alloc_buffer = GST_DEBUG_FUNCPTR(gst_hailo_dsp_buffer_pool_alloc_buffer); + pool_class->free_buffer = GST_DEBUG_FUNCPTR(gst_hailo_dsp_buffer_pool_free_buffer); + object_class->dispose = GST_DEBUG_FUNCPTR(gst_hailo_dsp_buffer_pool_dispose); +} + +static void +gst_hailo_dsp_buffer_pool_dispose(GObject *object) +{ + G_OBJECT_CLASS(gst_hailo_dsp_buffer_pool_parent_class)->dispose(object); + GstHailoDspBufferPool *pool = GST_HAILO_DSP_BUFFER_POOL(object); + GST_INFO_OBJECT(pool, "Hailo DSP buffer pool dispose"); + // Release DSP device + dsp_status result = release_device(); + if (result != DSP_SUCCESS) + { + GST_ERROR_OBJECT(pool, "Release DSP device failed with status code %d", result); + } +} + +static void +gst_hailo_dsp_buffer_pool_init(GstHailoDspBufferPool *pool) +{ + GST_INFO_OBJECT(pool, "New Hailo DSP buffer pool"); + // Acquire DSP device + dsp_status status = acquire_device(); + if (status != DSP_SUCCESS) + { + GST_ERROR_OBJECT(pool, "Accuire DSP device failed with status code %d", status); + } +} + +GstBufferPool * +gst_hailo_dsp_buffer_pool_new(guint padding) +{ + GstHailoDspBufferPool *pool = GST_HAILO_DSP_BUFFER_POOL(g_object_new(GST_TYPE_HAILO_DSP_BUFFER_POOL, NULL)); + pool->padding = padding; + return GST_BUFFER_POOL_CAST(pool); +} + +static GstFlowReturn +gst_hailo_dsp_buffer_pool_alloc_buffer(GstBufferPool *pool, GstBuffer **output_buffer_ptr, GstBufferPoolAcquireParams *params) +{ + GstHailoDspBufferPool *hailo_dsp_pool = GST_HAILO_DSP_BUFFER_POOL(pool); + + // Get the size of a buffer from the config of the pool + GstStructure *config = gst_buffer_pool_get_config(pool); + guint buffer_size = 0; + gst_buffer_pool_config_get_params(config, NULL, &buffer_size, NULL, NULL); + + // Validate the size of the buffer + if (buffer_size == 0) + { + GST_ERROR_OBJECT(hailo_dsp_pool, "Invalid buffer size"); + return GST_FLOW_ERROR; + } + GST_INFO_OBJECT(hailo_dsp_pool, "Allocating buffer of size %d with padding %d", buffer_size, hailo_dsp_pool->padding); + + void *buffer_ptr = NULL; + dsp_status result = create_hailo_dsp_buffer((size_t)buffer_size+hailo_dsp_pool->padding, &buffer_ptr); + if (result != DSP_SUCCESS) + { + GST_ERROR_OBJECT(pool, "Failed to create buffer with status code %d", result); + return GST_FLOW_ERROR; + } + GST_INFO_OBJECT(hailo_dsp_pool, "Allocated buffer of size %d from dsp memory", buffer_size); + + void *aligned_buffer_ptr = (void *)(((size_t)buffer_ptr + hailo_dsp_pool->padding)); + *output_buffer_ptr = gst_buffer_new_wrapped_full(GST_MEMORY_FLAG_PHYSICALLY_CONTIGUOUS, + aligned_buffer_ptr, (size_t)buffer_size, 0, (size_t)buffer_size, NULL, NULL); + + GST_INFO_OBJECT(hailo_dsp_pool, "Allocated buffer memory wrapped"); + + return GST_FLOW_OK; +} + +static void +gst_hailo_dsp_buffer_pool_free_buffer(GstBufferPool *pool, GstBuffer *buffer) +{ + GstHailoDspBufferPool *hailo_dsp_pool = GST_HAILO_DSP_BUFFER_POOL(pool); + GST_DEBUG_OBJECT(hailo_dsp_pool, "Freeing buffer %p with padding %d", buffer, hailo_dsp_pool->padding); + guint memory_count = gst_buffer_n_memory(buffer); + for (guint i = 0; i < memory_count; i++) + { + GstMemory *memory = gst_buffer_peek_memory(buffer, i); + GstMapInfo memory_map_info; + gst_memory_map(memory, &memory_map_info, GST_MAP_READ); + void *buffer_ptr = memory_map_info.data; + + void *aligned_buffer_ptr = (void *)(((size_t)buffer_ptr - hailo_dsp_pool->padding)); + dsp_status result = release_hailo_dsp_buffer(aligned_buffer_ptr); + if (result != DSP_SUCCESS) + { + GST_ERROR_OBJECT(hailo_dsp_pool, "Failed to release dsp buffer %p number %d out of %d", buffer_ptr, (i+1), memory_count); + } + else + { + GST_INFO_OBJECT(hailo_dsp_pool, "Released dsp buffer %p number %d out of %d", buffer_ptr, (i+1), memory_count); + } + + gst_memory_unmap(memory, &memory_map_info); + } + + gst_buffer_unref(buffer); +} diff --git a/core/hailo/plugins/dsp/gsthailodspbufferpool.hpp b/core/hailo/plugins/dsp/gsthailodspbufferpool.hpp new file mode 100644 index 0000000..727b895 --- /dev/null +++ b/core/hailo/plugins/dsp/gsthailodspbufferpool.hpp @@ -0,0 +1,35 @@ +#pragma once + +#include +#include +#include + +G_BEGIN_DECLS + +#define GST_TYPE_HAILO_DSP_BUFFER_POOL (gst_hailo_dsp_buffer_pool_get_type()) +#define GST_HAILO_DSP_BUFFER_POOL(obj) (G_TYPE_CHECK_INSTANCE_CAST((obj), GST_TYPE_HAILO_DSP_BUFFER_POOL, GstHailoDspBufferPool)) +#define GST_HAILO_DSP_BUFFER_POOL_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST((klass), GST_TYPE_HAILO_DSP_BUFFER_POOL, GstHailoDspBufferPoolClass)) +#define GST_IS_HAILO_DSP_BUFFER_POOL(obj) (G_TYPE_CHECK_INSTANCE_TYPE((obj), GST_TYPE_HAILO_DSP_BUFFER_POOL)) +#define GST_IS_HAILO_DSP_BUFFER_POOL_CLASS(klass) (G_TYPE_CHECK_CLASS_TYPE((klass), GST_TYPE_HAILO_DSP_BUFFER_POOL)) + +typedef struct _GstHailoDspBufferPool GstHailoDspBufferPool; +typedef struct _GstHailoDspBufferPoolClass GstHailoDspBufferPoolClass; + +struct _GstHailoDspBufferPool +{ + GstBufferPool parent; + guint padding; +}; + +struct _GstHailoDspBufferPoolClass +{ + GstBufferPoolClass parent_class; + +}; + +GstBufferPool * +gst_hailo_dsp_buffer_pool_new(guint padding); + +GType gst_hailo_dsp_buffer_pool_get_type (void); + +G_END_DECLS diff --git a/core/hailo/plugins/dsp/gsthailodspbufferpoolutils.cpp b/core/hailo/plugins/dsp/gsthailodspbufferpoolutils.cpp new file mode 100644 index 0000000..142c60f --- /dev/null +++ b/core/hailo/plugins/dsp/gsthailodspbufferpoolutils.cpp @@ -0,0 +1,154 @@ +#include "gsthailodspbufferpoolutils.hpp" +#include "dsp/gsthailodspbufferpool.hpp" + +gboolean +gst_is_hailo_dsp_pool_type(GstBufferPool *pool) +{ + return GST_IS_HAILO_DSP_BUFFER_POOL(pool); +} + +gboolean +gst_hailo_dsp_configure_pool(GstDebugCategory *category, GstBufferPool *pool, + GstCaps *caps, gsize size, guint min_buffers, guint max_buffers) +{ + GstStructure *config = NULL; + + g_return_val_if_fail(size > 0, FALSE); + g_return_val_if_fail(max_buffers > 0, FALSE); + + config = gst_buffer_pool_get_config(pool); + + gst_buffer_pool_config_set_params(config, caps, size, min_buffers, max_buffers); + + if (!gst_buffer_pool_set_config(pool, config)) + { + GST_ERROR_OBJECT(pool, "Unable to set pool configuration"); + gst_object_unref(pool); + return FALSE; + } + + if (!gst_buffer_pool_config_validate_params(config, caps, size, min_buffers, + max_buffers)) + { + GST_ERROR_OBJECT(pool, "Pool configuration validation failed"); + gst_object_unref(pool); + return FALSE; + } + + return TRUE; +} + +GstBufferPool * +gst_hailo_dsp_create_new_pool(GstDebugCategory *category, GstQuery *query, guint min_buffers, + guint max_buffers, gsize size, guint padding) +{ + GstCaps *caps = NULL; + + // Create a new bufferpool object + GstBufferPool *pool = gst_hailo_dsp_buffer_pool_new(padding); + if (pool == NULL) + { + GST_CAT_ERROR(category, "Create Hailo pool failed"); + return NULL; + } + + gst_query_parse_allocation(query, &caps, NULL); + + // Configure the bufferpool + gboolean res = gst_hailo_dsp_configure_pool(category, pool, caps, size, min_buffers, max_buffers); + if (res == FALSE) + { + GST_ERROR_OBJECT(pool, "Unable to configure pool"); + return NULL; + } + + gst_caps_unref(caps); + GST_DEBUG_OBJECT(pool, "Dsp Bufferpool created with buffer size: %ld min buffers: %d max buffers: %d and padding: %d", + size, min_buffers, max_buffers, padding); + + return pool; +} + +GstBufferPool * +gst_create_hailo_dsp_bufferpool_from_allocation_query(GstElement *element, GstQuery *query, guint bufferpool_min_size, guint bufferpool_max_size, guint padding) +{ + GstCaps *caps; + GstAllocator *allocator = NULL; + gboolean need_pool = FALSE; + + gst_query_parse_allocation(query, &caps, &need_pool); + + if (caps == NULL) + { + GST_ERROR_OBJECT(element, "Bufferpool creation from alloc query - Alloc query Caps is NULL"); + return NULL; + } + + if (!gst_caps_is_fixed(caps)) + { + GST_ERROR_OBJECT(element, "Bufferpool creation from alloc query - Caps is not fixed"); + gst_caps_unref(caps); + return NULL; + } + + GST_INFO_OBJECT(element, "Bufferpool creation from alloc query - caps are fixed"); + + // Get the width and height of the caps + GstVideoInfo *video_info = gst_video_info_new(); + gst_video_info_from_caps(video_info, caps); + gst_caps_unref(caps); + + guint buffer_size = video_info->size; + + GST_INFO_OBJECT(element, "Bufferpool creation from alloc query - Creating new pool with buffer size: %d", buffer_size); + + gst_video_info_free(video_info); + GstBufferPool *pool = gst_hailo_dsp_create_new_pool(GST_CAT_DEFAULT, query, bufferpool_min_size, bufferpool_max_size, buffer_size, padding); + + if (pool == NULL) + { + GST_ERROR_OBJECT(element, "Bufferpool creation from alloc query - Pool creation failed"); + return NULL; + } + + GstAllocationParams params; + + gst_allocation_params_init(¶ms); + + // Get how many pools are in the query + guint n_pools = gst_query_get_n_allocation_pools(query); + if (n_pools > 0) + { + gst_query_set_nth_allocation_param(query, 0, allocator, ¶ms); + gst_query_set_nth_allocation_pool(query, 0, pool, buffer_size, bufferpool_min_size, bufferpool_max_size); + } + gst_query_add_allocation_param(query, allocator, ¶ms); + gst_query_add_allocation_pool(query, pool, buffer_size, bufferpool_min_size, bufferpool_max_size); + + if (!gst_buffer_pool_set_active(pool, TRUE)) + { + GST_ERROR_OBJECT(element, "Bufferpool creation from alloc query - Unable to set pool active"); + if (allocator) + { + gst_object_unref(allocator); + } + return NULL; + } + + if (gst_query_get_n_allocation_pools(query) == 0) + { + GST_ERROR_OBJECT(element, "Bufferpool creation from alloc query - No pools in query"); + if (allocator) + { + gst_object_unref(allocator); + } + return NULL; + } + + if (allocator) + { + gst_object_unref(allocator); + } + + return pool; +} \ No newline at end of file diff --git a/core/hailo/plugins/dsp/gsthailodspbufferpoolutils.hpp b/core/hailo/plugins/dsp/gsthailodspbufferpoolutils.hpp new file mode 100644 index 0000000..cb5ec49 --- /dev/null +++ b/core/hailo/plugins/dsp/gsthailodspbufferpoolutils.hpp @@ -0,0 +1,61 @@ +#pragma once + +#include +#include +#include + +G_BEGIN_DECLS + + +/** + * Adds a new hailo buffer pool to the query + * @param[in] category: Category to use for debug messages + * @param[in] query: Query to parse caps from + * @param[in] min_buffers: Minimum size buffers in the pool + * @param[in] max_buffers: Maximum size buffers in the pool + * @param[in] size: Maximum size buffers in the pool + * + * @return GstBufferPool + * + */ +GstBufferPool * +gst_hailo_dsp_create_new_pool(GstDebugCategory *category, GstQuery *query, guint min_buffers, + guint max_buffers, gsize size); + +/** + * Configures a hailo buffer pool + * @param[in] category: Category to use for debug messages + * @param[in] pool: Pool to configure + * @param[in] caps: Caps to use for the pool + * @param[in] size: size of buffers in the pool + * @param[in] min_buffers: Minimum number of buffers in the pool + * @param[in] max_buffers: Maximum number of buffers in the pool + * + * @return boolean - pool was successfully configured + * + */ +gboolean +gst_hailo_dsp_configure_pool(GstDebugCategory *category, GstBufferPool *pool, + GstCaps *caps, gsize size, guint min_buffers, guint max_buffers); + +/** + * Checks if the pool is a hailo pool + * @param[in] pool: bufferpool to check + * + * @return boolean - pool is a hailo pool + * + */ +gboolean +gst_is_hailo_dsp_pool_type(GstBufferPool *pool); + +/** + * Creates a hailo bufferpool from an allocation query, configure it, activate, and add it to the query + * @param[in] element: Element to use for debug messages + * @param[in] query: Query to check, get caps from, and add the pool to + * @param[in] bufferpool_max_size: Maximum size of the bufferpool + * @return GstBufferPool + */ +GstBufferPool * +gst_create_hailo_dsp_bufferpool_from_allocation_query(GstElement *element, GstQuery *query, guint bufferpool_min_size, guint bufferpool_max_size, guint padding); + +G_END_DECLS \ No newline at end of file diff --git a/core/hailo/plugins/dsp/gsthailovideoscale.cpp b/core/hailo/plugins/dsp/gsthailovideoscale.cpp new file mode 100644 index 0000000..86412e1 --- /dev/null +++ b/core/hailo/plugins/dsp/gsthailovideoscale.cpp @@ -0,0 +1,170 @@ +#include +#include +#include +#include "dsp/gsthailodspbasetransform.hpp" +#include "dsp/gsthailovideoscale.hpp" +#include "dsp/gsthailodsp.h" + +GST_DEBUG_CATEGORY_STATIC(gst_hailo_videoscale_debug); +#define GST_CAT_DEFAULT gst_hailo_videoscale_debug + +#define _do_init \ + GST_DEBUG_CATEGORY_INIT(gst_hailo_videoscale_debug, "hailovideoscale", 0, "Hailo Videoscale"); +#define gst_hailo_videoscale_parent_class parent_class +G_DEFINE_TYPE_WITH_CODE(GstHailoVideoScale, gst_hailo_videoscale, GST_TYPE_HAILO_DSP_BASE_TRANSFORM, _do_init); + +static GstFlowReturn +gst_hailo_videoscale_transform(GstBaseTransform *base_transform, GstBuffer *inbuf, GstBuffer *outbuf); + +static void gst_hailo_videoscale_set_property(GObject *object, guint prop_id, const GValue *value, GParamSpec *pspec); +static void gst_hailo_videoscale_get_property(GObject *object, guint prop_id, GValue *value, GParamSpec *pspec); +static GstCaps *gst_hailo_videoscale_transform_caps(GstBaseTransform *trans, + GstPadDirection direction, GstCaps *caps, GstCaps *filter); + +static void +gst_hailo_videoscale_class_init(GstHailoVideoScaleClass *klass) +{ + GObjectClass *const object_class = G_OBJECT_CLASS(klass); + GstBaseTransformClass *const base_transform_class = GST_BASE_TRANSFORM_CLASS(klass); + + object_class->set_property = gst_hailo_videoscale_set_property; + object_class->get_property = gst_hailo_videoscale_get_property; + + base_transform_class->transform = GST_DEBUG_FUNCPTR(gst_hailo_videoscale_transform); + + base_transform_class->transform_caps = GST_DEBUG_FUNCPTR(gst_hailo_videoscale_transform_caps); + + gst_element_class_set_static_metadata(GST_ELEMENT_CLASS(klass), + "Hailo Videoscale", + "Hailo/Tools", + "Perform resize on a buffer using Hailo15 DSP", + "hailo.ai "); + + gst_element_class_add_pad_template(GST_ELEMENT_CLASS(klass), + gst_pad_template_new("src", GST_PAD_SRC, GST_PAD_ALWAYS, + gst_caps_from_string(HAILO_VIDEOSCALE_VIDEO_CAPS))); + gst_element_class_add_pad_template(GST_ELEMENT_CLASS(klass), + gst_pad_template_new("sink", GST_PAD_SINK, GST_PAD_ALWAYS, + gst_caps_from_string(HAILO_VIDEOSCALE_VIDEO_CAPS))); +} + +static void +gst_hailo_videoscale_init(GstHailoVideoScale *self) +{ + gst_base_transform_set_in_place(GST_BASE_TRANSFORM(self), FALSE); +} + +static void +gst_hailo_videoscale_set_property(GObject *object, guint prop_id, const GValue *value, GParamSpec *pspec) +{ + switch (prop_id) + { + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec); + break; + } +} + +static void +gst_hailo_videoscale_get_property(GObject *object, guint prop_id, GValue *value, GParamSpec *pspec) +{ + switch (prop_id) + { + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec); + break; + } +} + +static GstCaps * +gst_hailo_videoscale_transform_caps(GstBaseTransform *trans, + GstPadDirection direction, GstCaps *caps, GstCaps *filter) +{ + GstCaps *res_caps, *tmp_caps; + GstStructure *structure; + guint i, caps_size; + + // Remove width and height from caps - to allow basetransform negotiate properly + res_caps = gst_caps_copy(caps); + caps_size = gst_caps_get_size(res_caps); + for (i = 0; i < caps_size; i++) + { + structure = gst_caps_get_structure(res_caps, i); + gst_structure_remove_field(structure, "width"); + gst_structure_remove_field(structure, "height"); + } + + if (filter) + { + tmp_caps = res_caps; + res_caps = gst_caps_intersect_full(filter, tmp_caps, GST_CAPS_INTERSECT_FIRST); + gst_caps_unref(tmp_caps); + } + return res_caps; +} + +static GstFlowReturn +gst_hailo_videoscale_transform(GstBaseTransform *base_transform, GstBuffer *inbuf, GstBuffer *outbuf) +{ + GstHailoVideoScale *hailovideoscale = GST_HAILO_VIDEOSCALE(base_transform); + + GstFlowReturn ret = GST_FLOW_OK; + + GST_DEBUG_OBJECT(hailovideoscale, "Hailo Videoscale transform"); + + // Get input and output caps + GstCaps *incaps, *outcaps; + incaps = gst_pad_get_current_caps(base_transform->sinkpad); + outcaps = gst_pad_get_current_caps(base_transform->srcpad); + + GstVideoInfo input_video_info; + gst_video_info_from_caps(&input_video_info, incaps); + + GstVideoInfo output_video_info; + gst_video_info_from_caps(&output_video_info, outcaps); + + // Map input and output buffers to GstVideoFrame + GstVideoFrame input_video_frame; + if (!gst_video_frame_map(&input_video_frame, &input_video_info, inbuf, GST_MAP_READ)) + { + GST_ERROR_OBJECT(hailovideoscale, "Cannot map input buffer to frame"); + throw std::runtime_error("Cannot map input buffer to frame"); + } + + GstVideoFrame output_video_frame; + if (!gst_video_frame_map(&output_video_frame, &output_video_info, outbuf, GST_MAP_READWRITE)) + { + GST_ERROR_OBJECT(hailovideoscale, "Cannot map output buffer to frame"); + throw std::runtime_error("Cannot map output buffer to frame"); + } + + // Create dsp image properties from both input and output video frame objects + dsp_image_properties_t input_image_properties = create_image_properties_from_video_frame(&input_video_frame); + dsp_image_properties_t output_image_properties = create_image_properties_from_video_frame(&output_video_frame); + + GST_DEBUG_OBJECT(hailovideoscale, "DSP Resize: Input Width: %ld, Height: %ld. \ + Resize target Width: %ld Height: %ld", + input_image_properties.width, input_image_properties.height, + output_image_properties.width, output_image_properties.height); + + // Perform the resize operation on the DSP + dsp_status result = perform_dsp_resize(&input_image_properties, &output_image_properties, INTERPOLATION_TYPE_BILINEAR); + + if (result != DSP_SUCCESS) + { + GST_ERROR_OBJECT(hailovideoscale, "Failed to perform dsp resize. return status: %d", result); + ret = GST_FLOW_ERROR; + } + + // Free resources + free_image_property_planes(&input_image_properties); + free_image_property_planes(&output_image_properties); + + gst_video_frame_unmap(&input_video_frame); + gst_video_frame_unmap(&output_video_frame); + + gst_caps_unref(incaps); + gst_caps_unref(outcaps); + + return ret; +} \ No newline at end of file diff --git a/core/hailo/plugins/dsp/gsthailovideoscale.hpp b/core/hailo/plugins/dsp/gsthailovideoscale.hpp new file mode 100644 index 0000000..766e832 --- /dev/null +++ b/core/hailo/plugins/dsp/gsthailovideoscale.hpp @@ -0,0 +1,38 @@ +#pragma once + +#include "dsp/gsthailodspbasetransform.hpp" +#include + +G_BEGIN_DECLS + +#define GST_TYPE_HAILO_VIDEOSCALE \ + (gst_hailo_videoscale_get_type()) +#define GST_HAILO_VIDEOSCALE(obj) \ + (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_HAILO_VIDEOSCALE,GstHailoVideoScale)) +#define GST_HAILO_VIDEOSCALE_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_HAILO_VIDEOSCALE,GstHailoVideoScaleClass)) +#define GST_IS_HAILO_VIDEOSCALE(obj) \ + (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_HAILO_VIDEOSCALE)) +#define GST_IS_HAILO_VIDEOSCALE_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_HAILO_VIDEOSCALE)) +#define GST_HAILO_VIDEOSCALE_GET_CLASS(obj) \ + (G_TYPE_INSTANCE_GET_CLASS((obj),GST_TYPE_HAILO_VIDEOSCALE,GstHailoVideoScaleClass)) + +#define HAILO_VIDEOSCALE_SUPPORTED_FORMATS "{ RGB, NV12 }" +#define HAILO_VIDEOSCALE_VIDEO_CAPS \ + GST_VIDEO_CAPS_MAKE(HAILO_VIDEOSCALE_SUPPORTED_FORMATS) + +typedef struct _GstHailoVideoScale GstHailoVideoScale; +typedef struct _GstHailoVideoScaleClass GstHailoVideoScaleClass; + +struct _GstHailoVideoScale { + GstHailoDspBaseTransform parent; +}; + +struct _GstHailoVideoScaleClass { + GstHailoDspBaseTransformClass parent_class; +}; + +GType gst_hailo_videoscale_get_type (void); + +G_END_DECLS \ No newline at end of file diff --git a/core/hailo/plugins/dsp/osd/gsthailoosd.cpp b/core/hailo/plugins/dsp/osd/gsthailoosd.cpp new file mode 100644 index 0000000..be93072 --- /dev/null +++ b/core/hailo/plugins/dsp/osd/gsthailoosd.cpp @@ -0,0 +1,291 @@ +/** + * Copyright (c) 2021-2022 Hailo Technologies Ltd. All rights reserved. + * Distributed under the LGPL license (https://www.gnu.org/licenses/old-licenses/lgpl-2.1.txt) + **/ +#include +#include +#include +#include +#include + +#include "gsthailoosd.hpp" +#include "gst_hailo_meta.hpp" +#include "hailo/hailort.h" + +GST_DEBUG_CATEGORY_STATIC(gst_hailoosd_debug_category); +#define GST_CAT_DEFAULT gst_hailoosd_debug_category +#define gst_hailoosd_parent_class parent_class + +static void gst_hailoosd_set_property(GObject *object, + guint property_id, const GValue *value, GParamSpec *pspec); +static void gst_hailoosd_get_property(GObject *object, + guint property_id, GValue *value, GParamSpec *pspec); +static void gst_hailoosd_dispose(GObject *object); +static void gst_hailoosd_finalize(GObject *object); + +static gboolean gst_hailoosd_start(GstBaseTransform *trans); +static gboolean gst_hailoosd_stop(GstBaseTransform *trans); +static gboolean gst_hailoosd_set_caps(GstBaseTransform *trans, GstCaps *incaps, GstCaps *outcaps); +static gboolean gst_hailoosd_propose_allocation (GstBaseTransform *trans, GstQuery *decide_query, GstQuery *query); +static GstFlowReturn gst_hailoosd_transform_ip(GstBaseTransform *trans, + GstBuffer *buffer); +static void gst_hailoosd_before_transform(GstBaseTransform *trans, GstBuffer *buffer); + +enum +{ + PROP_0, + PROP_CONFIG_FILE_PATH, + PROP_WAIT_FOR_WRITABLE_BUFFER, +}; + +G_DEFINE_TYPE_WITH_CODE(GstHailoOsd, gst_hailoosd, GST_TYPE_BASE_TRANSFORM, + GST_DEBUG_CATEGORY_INIT(gst_hailoosd_debug_category, "hailoosd", 0, + "debug category for hailoosd element")); + +static void +gst_hailoosd_class_init(GstHailoOsdClass *klass) +{ + GObjectClass *gobject_class = G_OBJECT_CLASS(klass); + GstBaseTransformClass *base_transform_class = + GST_BASE_TRANSFORM_CLASS(klass); + + gst_element_class_add_pad_template(GST_ELEMENT_CLASS(klass), + gst_pad_template_new("src", GST_PAD_SRC, GST_PAD_ALWAYS, + gst_caps_from_string(GST_VIDEO_CAPS_MAKE("{ NV12 }")))); + gst_element_class_add_pad_template(GST_ELEMENT_CLASS(klass), + gst_pad_template_new("sink", GST_PAD_SINK, GST_PAD_ALWAYS, + gst_caps_from_string(GST_VIDEO_CAPS_MAKE("{ NV12 }")))); + + gst_element_class_set_static_metadata(GST_ELEMENT_CLASS(klass), + "hailoosd - on-screen-display element", + "Hailo/Tools", + "Draws on-screen-display telemetry on frame.", + "hailo.ai "); + + gobject_class->set_property = gst_hailoosd_set_property; + gobject_class->get_property = gst_hailoosd_get_property; + g_object_class_install_property(gobject_class, PROP_CONFIG_FILE_PATH, + g_param_spec_string("config-path", "config-path", + "json config file path", NULL, + (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS | GST_PARAM_MUTABLE_READY))); + + g_object_class_install_property(gobject_class, PROP_WAIT_FOR_WRITABLE_BUFFER, + g_param_spec_boolean("wait-for-writable-buffer", "wait-for-writable-buffer", + "enables the element thread to wait until incomming buffer is writable", FALSE, + (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS | GST_PARAM_MUTABLE_READY))); + + gobject_class->dispose = gst_hailoosd_dispose; + gobject_class->finalize = gst_hailoosd_finalize; + base_transform_class->start = GST_DEBUG_FUNCPTR(gst_hailoosd_start); + base_transform_class->stop = GST_DEBUG_FUNCPTR(gst_hailoosd_stop); + base_transform_class->set_caps = GST_DEBUG_FUNCPTR(gst_hailoosd_set_caps); + base_transform_class->propose_allocation = GST_DEBUG_FUNCPTR(gst_hailoosd_propose_allocation); + base_transform_class->transform_ip = GST_DEBUG_FUNCPTR(gst_hailoosd_transform_ip); + base_transform_class->before_transform = GST_DEBUG_FUNCPTR(gst_hailoosd_before_transform); +} + +static void +gst_hailoosd_init(GstHailoOsd *hailoosd) +{ + hailoosd->params = nullptr; + hailoosd->config_path = g_strdup("NULL"); + hailoosd->wait_for_writable_buffer = false; +} + +void gst_hailoosd_set_property(GObject *object, guint property_id, + const GValue *value, GParamSpec *pspec) +{ + GstHailoOsd *hailoosd = GST_HAILO_OSD(object); + + GST_DEBUG_OBJECT(hailoosd, "set_property"); + + switch (property_id) + { + case PROP_CONFIG_FILE_PATH: + hailoosd->config_path = g_strdup(g_value_get_string(value)); + break; + case PROP_WAIT_FOR_WRITABLE_BUFFER: + hailoosd->wait_for_writable_buffer = g_value_get_boolean(value); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID(object, property_id, pspec); + break; + } +} + +void gst_hailoosd_get_property(GObject *object, guint property_id, + GValue *value, GParamSpec *pspec) +{ + GstHailoOsd *hailoosd = GST_HAILO_OSD(object); + + GST_DEBUG_OBJECT(hailoosd, "get_property"); + + switch (property_id) + { + case PROP_CONFIG_FILE_PATH: + g_value_set_string(value, hailoosd->config_path); + break; + case PROP_WAIT_FOR_WRITABLE_BUFFER: + g_value_set_boolean(value, hailoosd->wait_for_writable_buffer); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID(object, property_id, pspec); + break; + } +} +void gst_hailoosd_dispose(GObject *object) +{ + GstHailoOsd *hailoosd = GST_HAILO_OSD(object); + + GST_DEBUG_OBJECT(hailoosd, "dispose"); + + /* clean up as possible. may be called multiple times */ + + G_OBJECT_CLASS(gst_hailoosd_parent_class)->dispose(object); +} + +void gst_hailoosd_finalize(GObject *object) +{ + GstHailoOsd *hailoosd = GST_HAILO_OSD(object); + + GST_DEBUG_OBJECT(hailoosd, "finalize"); + + /* clean up object here */ + + G_OBJECT_CLASS(gst_hailoosd_parent_class)->finalize(object); +} + +static gboolean gst_hailoosd_start(GstBaseTransform *trans) +{ + GstHailoOsd *hailoosd = GST_HAILO_OSD(trans); + + // Load params + hailoosd->params = load_json_config(hailoosd->config_path); + + // Acquire DSP device + dsp_status status = acquire_device(); + if (status != DSP_SUCCESS) + { + GST_ERROR_OBJECT(hailoosd, "Accuire DSP device failed with status code %d", status); + } + + GST_DEBUG_OBJECT(hailoosd, "start"); + + return TRUE; +} + +static gboolean +gst_hailoosd_stop(GstBaseTransform *trans) +{ + GstHailoOsd *hailoosd = GST_HAILO_OSD(trans); + + // Release param resources + free_param_resources(hailoosd->params); + + // Release DSP device + dsp_status result = release_device(); + if (result != DSP_SUCCESS) + { + GST_ERROR_OBJECT(hailoosd, "Release DSP device failed with status code %d", result); + } + + GST_DEBUG_OBJECT(hailoosd, "stop"); + + return TRUE; +} + +static void +gst_hailoosd_wait_for_writable_buffer(GstHailoOsd *hailoosd, GstBuffer *buffer) +{ + GST_DEBUG_OBJECT(hailoosd, "Buffer (offset: %ld) is not writable, refcount: %d. waiting... ", GST_BUFFER_OFFSET(buffer), GST_OBJECT_REFCOUNT(buffer)); + while(gst_buffer_is_writable(buffer) == FALSE) + { + // Wait for buffer to be writable + g_usleep (100); + } +} + +static void +gst_hailoosd_before_transform(GstBaseTransform *trans, GstBuffer *buffer) +{ + GstHailoOsd *hailoosd = GST_HAILO_OSD(trans); + + if (gst_buffer_is_writable(buffer) == FALSE) + { + if(hailoosd->wait_for_writable_buffer) + { + gst_hailoosd_wait_for_writable_buffer(hailoosd, buffer); + } + else + { + GST_ERROR_OBJECT(hailoosd, "Buffer (offset: %ld) is not writable!, buffer refcount is %d. Aborting...", GST_BUFFER_OFFSET(buffer), GST_OBJECT_REFCOUNT(buffer)); + throw std::runtime_error("HailoOSD -> Buffer is not writable! please validate that that the pipline is sharing the buffer properly, or use wait-for-writable-buffer parameter"); + } + } + + GST_DEBUG_OBJECT(hailoosd, "Buffer is writable, refcount: %d, continuing...", GST_OBJECT_REFCOUNT(buffer)); +} + +static gboolean +gst_hailoosd_set_caps(GstBaseTransform *trans, GstCaps *incaps, GstCaps *outcaps) +{ + GstHailoOsd *hailoosd = GST_HAILO_OSD(trans); + osd_status_t ret = OSD_STATUS_UNINITIALIZED; + + // now that caps are negotiated, get the size of the image for relative scaling + GstVideoInfo *full_image_info = gst_video_info_new(); + gst_video_info_from_caps(full_image_info, incaps); + + // init overlay images from the json params + ret = initialize_overlay_images(hailoosd->params, full_image_info->width, full_image_info->height); + + // cleanup + gst_video_info_free(full_image_info); + + // check success status + if (ret != OSD_STATUS_OK) + return FALSE; + + return TRUE; +} + +static gboolean +gst_hailoosd_propose_allocation (GstBaseTransform *trans, + GstQuery *decide_query, GstQuery *query) +{ + GstHailoOsd *hailoosd = GST_HAILO_OSD(trans); + GST_DEBUG_OBJECT(hailoosd, "hailoosd propose allocation callback"); + gboolean ret; + gst_query_add_allocation_meta(query, GST_VIDEO_META_API_TYPE, NULL); + ret = GST_BASE_TRANSFORM_CLASS(parent_class)->propose_allocation(trans, decide_query, query); + return ret; +} + +static GstFlowReturn gst_hailoosd_transform_ip(GstBaseTransform *trans, + GstBuffer *buffer) +{ + osd_status_t ret = OSD_STATUS_UNINITIALIZED; + GstHailoOsd *hailoosd = GST_HAILO_OSD(trans); + GST_DEBUG_OBJECT(hailoosd, "transform_ip"); + + // acquire caps and metas + GstCaps *caps; + caps = gst_pad_get_current_caps(trans->sinkpad); + GstVideoInfo *info = gst_video_info_new(); + gst_video_info_from_caps(info, caps); + GstVideoFrame video_frame; + gst_video_frame_map(&video_frame, info, buffer, GST_MAP_READ); + + // perform blending + ret = blend_all(&video_frame, hailoosd->params); + + // cleanup + gst_video_frame_unmap (&video_frame); + gst_video_info_free(info); + gst_caps_unref(caps); + + // check success status + if (ret != OSD_STATUS_OK) + return GST_FLOW_ERROR; + return GST_FLOW_OK; +} diff --git a/core/hailo/plugins/dsp/osd/gsthailoosd.hpp b/core/hailo/plugins/dsp/osd/gsthailoosd.hpp new file mode 100644 index 0000000..65d917f --- /dev/null +++ b/core/hailo/plugins/dsp/osd/gsthailoosd.hpp @@ -0,0 +1,41 @@ +/** + * Copyright (c) 2021-2022 Hailo Technologies Ltd. All rights reserved. + * Distributed under the LGPL license (https://www.gnu.org/licenses/old-licenses/lgpl-2.1.txt) + **/ +#pragma once + +#include +#include +#include +#include +#include "hailo_objects.hpp" +#include "osd.hpp" +#include "dsp/gsthailodsp.h" + +G_BEGIN_DECLS + +#define GST_TYPE_HAILO_OSD (gst_hailoosd_get_type()) +#define GST_HAILO_OSD(obj) (G_TYPE_CHECK_INSTANCE_CAST((obj), GST_TYPE_HAILO_OSD, GstHailoOsd)) +#define GST_HAILO_OSD_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST((klass), GST_TYPE_HAILO_OSD, GstHailoOsdClass)) +#define GST_IS_HAILO_OSD(obj) (G_TYPE_CHECK_INSTANCE_TYPE((obj), GST_TYPE_HAILO_OSD)) +#define GST_IS_HAILO_OSD_CLASS(obj) (G_TYPE_CHECK_CLASS_TYPE((klass), GST_TYPE_HAILO_OSD)) + +typedef struct _GstHailoOsd GstHailoOsd; +typedef struct _GstHailoOsdClass GstHailoOsdClass; + +struct _GstHailoOsd +{ + GstBaseTransform base_hailoosd; + gchar *config_path; + OsdParams * params; + gboolean wait_for_writable_buffer; +}; + +struct _GstHailoOsdClass +{ + GstBaseTransformClass parent_class; +}; + +GType gst_hailoosd_get_type(void); + +G_END_DECLS \ No newline at end of file diff --git a/core/hailo/plugins/dsp/osd/osd.cpp b/core/hailo/plugins/dsp/osd/osd.cpp new file mode 100644 index 0000000..77caeaf --- /dev/null +++ b/core/hailo/plugins/dsp/osd/osd.cpp @@ -0,0 +1,326 @@ +/** +* Copyright (c) 2021-2022 Hailo Technologies Ltd. All rights reserved. +* Distributed under the LGPL license (https://www.gnu.org/licenses/old-licenses/lgpl-2.1.txt) +**/ + +// General includes +#include +#include +#include + +// Hailo includes +#include "osd.hpp" +#include "hailo_common.hpp" +#include "json_config.hpp" + +// rapidjson includes +#include "rapidjson/document.h" +#include "rapidjson/stringbuffer.h" +#include "rapidjson/error/en.h" +#include "rapidjson/filereadstream.h" +#include "rapidjson/schema.h" + +#if __GNUC__ > 8 +#include +namespace fs = std::filesystem; +#else +#include +namespace fs = std::experimental::filesystem; +#endif + +/** +* Load json configuration to OsdParams. Use default values if necessary. +* +* @param[in] config_path path to json config file +* @return OsdParams* +*/ +OsdParams *load_json_config(const std::string config_path) +{ + std::vector static_texts; + std::vector static_images; + std::vector date_times; + + if (!fs::exists(config_path)) + { + std::cerr << "Config file doesn't exist, using default parameters" << std::endl; + // default static texts + static_texts.push_back({"Hailo", 1.0, 1, {255, 255, 255}, 0.7, 0.7}); + static_texts.push_back({"Stream 0", 1.0, 1, {255, 255, 255}, 0.1, 0.1}); + + // default date/time + date_times.push_back({1.0, 1, {255, 255, 255}, 0.6, 0.1}); + } else { + char config_buffer[4096]; + std::FILE *fp = fopen(config_path.c_str(), "r"); + if (fp == nullptr) + { + throw std::runtime_error("JSON config file is not valid"); + } + rapidjson::FileReadStream stream(fp, config_buffer, sizeof(config_buffer)); + bool valid_json = common::validate_json_with_schema(stream, osd::json_schema); + if (valid_json) + { + rapidjson::Document doc_config_json; + doc_config_json.ParseStream(stream); + + if (doc_config_json.HasMember("staticText") && doc_config_json["staticText"].IsArray()) + { + auto json_static_texts = doc_config_json["staticText"].GetArray(); + for (uint i = 0; i < json_static_texts.Size(); i++) + { + const rapidjson::Value& static_text_entry = json_static_texts[i]; + std::array color_rgb = {255, 255, 255}; + if (static_text_entry.HasMember("rgb")) + { + auto rgb_values = static_text_entry["rgb"].GetArray(); + color_rgb = {rgb_values[0].GetInt(), rgb_values[1].GetInt(), rgb_values[2].GetInt()}; + } + static_texts.push_back({static_text_entry["label"].GetString(), + static_text_entry["font_size"].GetFloat(), + static_text_entry["line_thickness"].GetInt(), + color_rgb, + static_text_entry["x"].GetFloat(), + static_text_entry["y"].GetFloat()}); + } + } + if (doc_config_json.HasMember("staticImage") && doc_config_json["staticImage"].IsArray()) + { + auto json_static_images = doc_config_json["staticImage"].GetArray(); + for (uint i = 0; i < json_static_images.Size(); i++) + { + const rapidjson::Value& static_image_entry = json_static_images[i]; + static_images.push_back({static_image_entry["image_path"].GetString(), + static_image_entry["width"].GetFloat(), + static_image_entry["height"].GetFloat(), + static_image_entry["x"].GetFloat(), + static_image_entry["y"].GetFloat()}); + } + } + if (doc_config_json.HasMember("dateTime") && doc_config_json["dateTime"].IsArray()) + { + auto json_date_times = doc_config_json["dateTime"].GetArray(); + for (uint i = 0; i < json_date_times.Size(); i++) + { + const rapidjson::Value& static_date_time_entry = json_date_times[i]; + std::array color_rgb = {255, 255, 255}; + if (static_date_time_entry.HasMember("rgb")) + { + auto rgb_values = static_date_time_entry["rgb"].GetArray(); + color_rgb = {rgb_values[0].GetInt(), rgb_values[1].GetInt(), rgb_values[2].GetInt()}; + } + date_times.push_back({static_date_time_entry["font_size"].GetFloat(), + static_date_time_entry["line_thickness"].GetInt(), + color_rgb, + static_date_time_entry["x"].GetFloat(), + static_date_time_entry["y"].GetFloat()}); + } + } + } + else + { + throw std::runtime_error("JSON config file is not valid"); + } + } + OsdParams *params = new OsdParams(static_texts, static_images, date_times); + return params; +} + +/** +* Deep free on OsdParams, freeing and unmaping all overlay data +* +* @param[in] params_ptr params to free +* @return void +*/ +void free_param_resources(OsdParams *params_ptr) +{ + if (nullptr == params_ptr) + return; + + // iterate over params, free all resources + for (osd::staticText static_text : params_ptr->static_texts) + { + gst_video_frame_unmap(&(static_text.video_frame)); + free_overlay_property_planes(&(static_text.dsp_overlay_properties)); + } + for (osd::staticImage static_image : params_ptr->static_images) + { + gst_video_frame_unmap(&(static_image.video_frame)); + free_overlay_property_planes(&(static_image.dsp_overlay_properties)); + } + for (osd::dateTime date_time : params_ptr->date_times) + { + for (uint index = 0; index < sizeof(osd::DATE_TIME_CHARS); ++index) + { + gst_video_frame_unmap(&(date_time.video_frames[index])); + free_image_property_planes(&(date_time.dsp_image_properties[index])); + } + } + delete params_ptr; +} + +/** +* Calculate x and y offsets for overlays +* DSP blending does not support out-of-bounds overlays +* so throw errors if any +* +* @param[in] x_norm normalized x for top-left corner +* @param[in] y_norm normalized y for top-left corner +* @param[in] overlay_width width of overlay in pixels +* @param[in] overlay_height height of overlay in pixels +* @param[in] image_width width of image in pixels +* @param[in] image_height height of image in pixels +* @param[in] overlay_type overlay type for error logs +* @return std::tuple +*/ +std::tuple calc_xy_offsets(float x_norm, float y_norm, + int overlay_width, int overlay_height, + int image_width, int image_height, const std::string &overlay_type) +{ + int x_offset = x_norm * image_width; + int y_offset = y_norm * image_height; + if (x_offset + overlay_width > image_width) + throw std::runtime_error(overlay_type + " overlay too wide to fit in frame! Adjust width or x offset in json."); + if (y_offset + overlay_height > image_height) + throw std::runtime_error(overlay_type + " overlay too tall to fit in frame! Adjust height or y offset in json."); + return {x_offset, y_offset}; +} + +/** +* Load all overlay images in A420 format. Reads static images from +* file and generates static texts & date/time chars. +* +* @param[in] params params to load +* @param[in] full_image_width width of image in pixels +* @param[in] full_image_height height of image in pixels +* @return osd_status_t +*/ +osd_status_t initialize_overlay_images(OsdParams *params, int full_image_width, int full_image_height) +{ + osd_status_t ret = OSD_STATUS_UNINITIALIZED; + + // Initialize static images + for (uint i = 0; i < params->static_images.size(); i++) + { + cv::Mat loaded_image = osd::load_image_from_file(params->static_images[i].image_path); + cv::Mat resized_image = osd::resize_mat(loaded_image, + params->static_images[i].width * full_image_width, + params->static_images[i].height * full_image_height); + GstVideoFrame gst_bgra_image = osd::gst_video_frame_from_mat_bgra(resized_image); + GstVideoFrame dest_frame; + params->static_images[i].video_frame = dest_frame; + osd::convert_2_dsp_video_frame(&gst_bgra_image, &(params->static_images[i].video_frame), GST_VIDEO_FORMAT_A420); + params->static_images[i].dsp_image_properties = create_image_properties_from_video_frame(&(params->static_images[i].video_frame)); + auto [x_offset, y_offset] = calc_xy_offsets(params->static_images[i].x, params->static_images[i].y, + params->static_images[i].dsp_image_properties.width, params->static_images[i].dsp_image_properties.height, + full_image_width, full_image_height, "Static Image"); + params->static_images[i].dsp_overlay_properties = (dsp_overlay_properties_t){params->static_images[i].dsp_image_properties, + (size_t)x_offset, (size_t)y_offset}; + gst_buffer_unref(gst_bgra_image.buffer); + gst_video_frame_unmap(&gst_bgra_image); + } + + // Initialize static texts + for (uint i = 0; i < params->static_texts.size(); i++) + { + cv::Mat bgra_text = osd::load_image_from_text(params->static_texts[i].label, + params->static_texts[i].font_size, + params->static_texts[i].line_thickness, + params->static_texts[i].color_rgb); + GstVideoFrame gst_bgra_image = osd::gst_video_frame_from_mat_bgra(bgra_text); + GstVideoFrame dest_frame; + params->static_texts[i].video_frame = dest_frame; + osd::convert_2_dsp_video_frame(&gst_bgra_image, &(params->static_texts[i].video_frame), GST_VIDEO_FORMAT_A420); + params->static_texts[i].dsp_image_properties = create_image_properties_from_video_frame(&(params->static_texts[i].video_frame)); + auto [x_offset, y_offset] = calc_xy_offsets(params->static_texts[i].x, params->static_texts[i].y, + params->static_texts[i].dsp_image_properties.width, params->static_texts[i].dsp_image_properties.height, + full_image_width, full_image_height, "Static Text"); + params->static_texts[i].dsp_overlay_properties = (dsp_overlay_properties_t){params->static_texts[i].dsp_image_properties, + (size_t)x_offset, (size_t)y_offset}; + gst_buffer_unref(gst_bgra_image.buffer); + gst_video_frame_unmap(&gst_bgra_image); + } + + // Initialize static texts + for (uint i = 0; i < params->date_times.size(); i++) + { + for (uint j = 0; j < sizeof(osd::DATE_TIME_CHARS); j++) + { + cv::Mat bgra_char = osd::load_image_from_text(std::string(&osd::DATE_TIME_CHARS[j], 1), + params->date_times[i].font_size, + params->date_times[i].line_thickness, + params->date_times[i].color_rgb); + GstVideoFrame gst_bgra_char = osd::gst_video_frame_from_mat_bgra(bgra_char); + GstVideoFrame dest_frame; + params->date_times[i].video_frames[j] = dest_frame; + osd::convert_2_dsp_video_frame(&gst_bgra_char, &(params->date_times[i].video_frames[j]), GST_VIDEO_FORMAT_A420); + params->date_times[i].dsp_image_properties[j] = create_image_properties_from_video_frame(&(params->date_times[i].video_frames[j])); + gst_buffer_unref(gst_bgra_char.buffer); + gst_video_frame_unmap(&gst_bgra_char); + } + auto [x_offset, y_offset] = calc_xy_offsets(params->date_times[i].x, params->date_times[i].y, + params->date_times[i].dsp_image_properties[0].width * 19, params->date_times[i].dsp_image_properties[0].height, + full_image_width, full_image_height, "Date/Time"); + } + + ret = OSD_STATUS_OK; + return ret; +} + +/** +* Call DSP blending on all loaded overlays to the incoming GstVideoFrame +* +* @param[in] video_frame the incoming image to blend onto +* @param[in] params params with loaded overlay images to blend with +* @return osd_status_t +*/ +osd_status_t blend_all(GstVideoFrame *video_frame, OsdParams *params) +{ + osd_status_t ret = OSD_STATUS_UNINITIALIZED; + + // build image_properties from the input image and overlay + dsp_image_properties_t input_image_properties = create_image_properties_from_video_frame(video_frame); + + // We prepare to blend all overlays at once + int num_overlays = params->static_texts.size() + params->static_images.size() + (params->date_times.size() * 19); + dsp_overlay_properties_t overlays[num_overlays]; + + int staged_overlays = 0; + // iterate over params + for (osd::staticText static_text : params->static_texts) + { + overlays[staged_overlays] = static_text.dsp_overlay_properties; + staged_overlays++; + } + for (osd::staticImage static_image : params->static_images) + { + overlays[staged_overlays] = static_image.dsp_overlay_properties; + staged_overlays++; + } + + size_t full_image_width = GST_VIDEO_FRAME_WIDTH(video_frame); + size_t full_image_height = GST_VIDEO_FRAME_HEIGHT(video_frame); + for (osd::dateTime date_time : params->date_times) + { + int x_offset = date_time.x * full_image_width; + int y_offset = date_time.y * full_image_height; + std::vector char_indices = osd::select_chars_for_timestamp(); + for (uint index = 0; index < char_indices.size(); ++index) + { + overlays[staged_overlays] = (dsp_overlay_properties_t){date_time.dsp_image_properties[char_indices[index]], (size_t)x_offset, (size_t)y_offset}; + staged_overlays++; + x_offset += date_time.dsp_image_properties[char_indices[index]].width; + } + } + + // Perform blending for all overlays + dsp_status status; + status = perform_dsp_multiblend(&input_image_properties, overlays, staged_overlays); + if (status != DSP_SUCCESS) + throw std::runtime_error("DSP BLEND FAILED: " + std::to_string(status)); + + // free the struct + free_image_property_planes(&input_image_properties); + + ret = OSD_STATUS_OK; + return ret; +} diff --git a/core/hailo/plugins/dsp/osd/osd.hpp b/core/hailo/plugins/dsp/osd/osd.hpp new file mode 100644 index 0000000..ef91841 --- /dev/null +++ b/core/hailo/plugins/dsp/osd/osd.hpp @@ -0,0 +1,41 @@ +/** +* Copyright (c) 2021-2022 Hailo Technologies Ltd. All rights reserved. +* Distributed under the LGPL license (https://www.gnu.org/licenses/old-licenses/lgpl-2.1.txt) +**/ +#pragma once + +#include +#include +#include +#include "hailo_objects.hpp" +#include "osd_utils.hpp" +#include "dsp/gsthailodsp.h" + +typedef enum +{ + OSD_STATUS_UNINITIALIZED = -1, + OSD_STATUS_OK, +} osd_status_t; + +class OsdParams +{ +public: + std::vector static_texts; + std::vector static_images; + std::vector date_times; + + OsdParams(std::vector static_texts, + std::vector static_images, + std::vector date_times) { + this->static_texts = static_texts; + this->static_images = static_images; + this->date_times = date_times; + } +}; + +__BEGIN_DECLS +OsdParams *load_json_config(const std::string config_path); +void free_param_resources(OsdParams *params_ptr); +osd_status_t initialize_overlay_images(OsdParams *params, int full_image_width, int full_image_height); +osd_status_t blend_all(GstVideoFrame *video_frame, OsdParams *params); +__END_DECLS \ No newline at end of file diff --git a/core/hailo/plugins/dsp/osd/osd_utils.hpp b/core/hailo/plugins/dsp/osd/osd_utils.hpp new file mode 100644 index 0000000..423ad63 --- /dev/null +++ b/core/hailo/plugins/dsp/osd/osd_utils.hpp @@ -0,0 +1,388 @@ + +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "dsp/gsthailodsp.h" + +namespace osd +{ + struct staticText { + std::string label; + float font_size; + int line_thickness; + std::array color_rgb; + float x; + float y; + GstVideoFrame video_frame; + dsp_image_properties_t dsp_image_properties; + dsp_overlay_properties_t dsp_overlay_properties; + }; + + struct staticImage { + std::string image_path; + float width; + float height; + float x; + float y; + GstVideoFrame video_frame; + dsp_image_properties_t dsp_image_properties; + dsp_overlay_properties_t dsp_overlay_properties; + }; + + struct dateTime { + float font_size; + int line_thickness; + std::array color_rgb; + float x; + float y; + std::array video_frames; // chars 0-9 - ' ' and : + std::array dsp_image_properties; // chars 0-9 - ' ' and : + }; + + inline char DATE_TIME_CHARS[13] = {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-', ' ', ':'}; + + /** + * Load an image file to an RGBA cv::Mat + * + * @param[in] image_path the path of the image to load + * @return a loaded cv::Mat + */ + inline cv::Mat load_image_from_file(std::string image_path) + { + // check if the file exists + if (!cv::utils::fs::exists(image_path)) + { + std::cerr << "Error: file " << image_path << " does not exist" << std::endl; + throw std::runtime_error("Image path in json config is not valid"); + } + // read the image from file, keeping alpha channel + cv::Mat image = cv::imread(image_path, cv::IMREAD_UNCHANGED); + // check if the image was read successfully + if (image.empty()) + { + std::cerr << "Error: failed to read image file " << image_path << std::endl; + throw std::runtime_error("Image path is valid but failed to load"); + } + // convert image to 4-channel RGBA format if necessary + if (image.channels() != 4) + { + GST_INFO("READ IMAGE THAT WAS NOT 4 channels"); + cv::cvtColor(image, image, cv::COLOR_BGR2BGRA); + } + return image; + } + + /** + * Create a BGRA cv::Mat of a string label + * + * @param[in] label the text to draw + * @param[in] font_size the font scaling factor + * @param[in] line_thickness the line thickness + * @return a drawn cv::Mat + */ + inline cv::Mat load_image_from_text(const std::string& label, float font_size, int line_thickness, std::array color_rgb) + { + // calculate the size of the text + int baseline=0; + cv::Size text_size = cv::getTextSize(label, cv::FONT_HERSHEY_SIMPLEX, font_size, line_thickness, &baseline); + + // ensure even dimensions, round up not to clip text + text_size.width += (text_size.width % 2); + text_size.height += (text_size.height % 2); + + // create a transparent BGRA image + cv::Mat bgra_text(text_size.height + baseline, text_size.width, CV_8UC4, cv::Scalar(0, 0, 0, 0)); + auto text_position = cv::Point(0, text_size.height); + cv::Scalar text_color(color_rgb[2], color_rgb[1], color_rgb[0], 255); // The input is expected RGB, but we draw as BGRA + + // draw the text + cv::putText(bgra_text, label, text_position, cv::FONT_HERSHEY_SIMPLEX, font_size, text_color, line_thickness); + + return bgra_text; + } + + /** + * Resize a cv::Mat, intended for RGB,BGR,RGBA,BGRA + * use only + * + * @param[in] mat the cv::Mat to resize + * @param[in] width the target width + * @param[in] height the target height + * @return a resized cv::Mat + */ + inline cv::Mat resize_mat(cv::Mat original, int width, int height) + { + // ensure even dimensions, round up not to clip image + width += (width % 2); + height += (height % 2); + + // resize the mat image to target size + cv::Mat resized_image; + cv::resize(original, resized_image, cv::Size(width, height), 0, 0, cv::INTER_AREA); + return resized_image; + } + + /** + * Converts an BGRA cv::Mat to GstVideoFrame + * + * @param[in] mat the cv::Mat to convert + * @return a GstVideoFrame of the given cv::Mat + */ + inline GstVideoFrame gst_video_frame_from_mat_bgra(cv::Mat mat) + { + // Create caps at BGRA format and required size + GstCaps *caps = gst_caps_new_simple("video/x-raw", "format", G_TYPE_STRING, "BGRA", + "width", G_TYPE_INT, mat.cols, + "height", G_TYPE_INT, mat.rows, + NULL); + // Create GstVideoInfo meta from those caps + GstVideoInfo *image_info = gst_video_info_new(); + gst_video_info_from_caps(image_info, caps); + // Create a GstBuffer from the cv::mat, allowing for contiguous memory + GstBuffer *buffer = gst_buffer_new_wrapped_full(GST_MEMORY_FLAG_READONLY, + mat.data, + mat.total() * mat.elemSize(), + 0, mat.total() * mat.elemSize(), + NULL, NULL ); + // Create and map a GstVideoFrame from the GstVideoInfo and GstBuffer + GstVideoFrame frame; + gst_video_frame_map(&frame, image_info, buffer, GST_MAP_READ); + gst_video_info_free(image_info); + gst_caps_unref(caps); + return frame; + } + + /** + * Converts an GstVideoFrame to a given format + * the new frame will have contiguous memory + * + * @param[in] src_frame the GstVideoFrame to convert + * @param[in] dest_format the format to convert to + * @return a GstVideoFrame* to the new GstVideoFrame + */ + inline void convert_2_dsp_video_frame(GstVideoFrame* src_frame, GstVideoFrame* dest_frame, GstVideoFormat dest_format) { + // Prepare the video info and set the new format + GstVideoInfo *dest_info = gst_video_info_copy(&src_frame->info); + gst_video_info_set_format(dest_info, dest_format, src_frame->info.width, src_frame->info.height); + // Prepare the GstVideoConverter that will facilitate the conversion + GstVideoConverter* converter = gst_video_converter_new(&src_frame->info, dest_info, NULL); + + void *buffer_ptr = NULL; + dsp_status buffer_status = create_hailo_dsp_buffer(dest_info->size, &buffer_ptr); + if (buffer_status != DSP_SUCCESS) + { + std::cerr << "Error: create_hailo_dsp_buffer - failed to create buffer" << std::endl; + throw std::runtime_error("Failed to create Hailo dsp buffer"); + } + GstBuffer *buffer = gst_buffer_new_wrapped_full(GST_MEMORY_FLAG_PHYSICALLY_CONTIGUOUS, + buffer_ptr, + dest_info->size, 0, dest_info->size, buffer_ptr, GDestroyNotify(release_hailo_dsp_buffer)); + + // Prepare the destination buffer and frame + gst_video_frame_map(dest_frame, dest_info, buffer, GST_MAP_WRITE); + // Make the conversion + gst_video_converter_frame(converter, src_frame, dest_frame); + gst_video_converter_free(converter); + } + + /** + * Converts a number of indeterminant length to indices + * and pushes them into char_indices + * + * @param[in] number the int to convert + * @param[in] char_indices where to push the converted indices + * @return an A420 cv::Mat + */ + inline void numbers_2_indices(int number, std::vector &char_indices) + { + std::string number_s = std::to_string(number); + // Traverse the string + for (auto &ch : number_s) { + // Get the char as an int + int x = ch - '0'; + char_indices.push_back(x); + } + } + + /** + * Curates a selection of indices representing chars to draw + * + * @return std::vector indices of the chars needed to draw + */ + inline std::vector select_chars_for_timestamp() + { + auto system_time = std::chrono::system_clock::now(); + std::time_t ttime = std::chrono::system_clock::to_time_t(system_time); + std::tm *time_now = std::gmtime(&ttime); + + std::vector char_indices; + numbers_2_indices(time_now->tm_mday, char_indices); // day + char_indices.push_back(10); // - + numbers_2_indices(time_now->tm_mon, char_indices); // month + char_indices.push_back(10); // - + numbers_2_indices(1900 + time_now->tm_year, char_indices); // year + char_indices.push_back(11); // ' ' + numbers_2_indices(time_now->tm_hour, char_indices); // hour + char_indices.push_back(12); // : + numbers_2_indices(time_now->tm_min, char_indices); // min + char_indices.push_back(12); // : + numbers_2_indices(time_now->tm_sec, char_indices); // sec + + return char_indices; + } + + /** + * Overloads << for printing the staticText struct info + * + * @return ostream + */ + inline std::ostream& operator<<(std::ostream& os, const staticText& sText) { + return os << "--- staticText ---" << std::endl + << "label: " << sText.label << std::endl + << "font_size: " << sText.font_size << std::endl + << "line_thickness: " << sText.line_thickness << std::endl + << "rgb: [" << sText.color_rgb[0] << ", " << sText.color_rgb[1] << ", " << sText.color_rgb[2] << "]" << std::endl + << "x: " << sText.x << std::endl + << "y: " << sText.y << std::endl; + } + + /** + * Overloads << for printing the staticImage struct info + * + * @return ostream + */ + inline std::ostream& operator<<(std::ostream& os, const staticImage& sImage) { + return os << "--- staticImage ---" << std::endl + << "image_path: " << sImage.image_path << std::endl + << "x: " << sImage.x << std::endl + << "y: " << sImage.y << std::endl + << "width: " << sImage.width << std::endl + << "height: " << sImage.height << std::endl; + } + + /** + * Overloads << for printing the dateTime struct info + * + * @return ostream + */ + inline std::ostream& operator<<(std::ostream& os, const dateTime& dTime) { + return os << "--- dateTime ---" << std::endl + << "font_size: " << dTime.font_size << std::endl + << "line_thickness: " << dTime.line_thickness << std::endl + << "rgb: [" << dTime.color_rgb[0] << ", " << dTime.color_rgb[1] << ", " << dTime.color_rgb[2] << "]" << std::endl + << "x: " << dTime.x << std::endl + << "y: " << dTime.y << std::endl; + } + + inline const char *json_schema = R""""({ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "properties": { + "staticImage": { + "type": "array", + "items":{ + "type": "object", + "properties": { + "image_path": { + "type": "string" + }, + "width": { + "type": "number" + }, + "height": { + "type": "number" + }, + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + }, + "required": ["image_path"] + } + }, + "dateTime": { + "type": "array", + "items":{ + "type": "object", + "properties": { + "font_size": { + "type": "number" + }, + "line_thickness": { + "type": "integer" + }, + "rgb": { + "type": "array", + "items": [ + { + "type": "integer" + }, + { + "type": "integer" + }, + { + "type": "integer" + } + ] + }, + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + } + } + }, + "staticText": { + "type": "array", + "default": [], + "items":{ + "type": "object", + "properties": { + "label": { + "type": "string" + }, + "font_size": { + "type": "number" + }, + "line_thickness": { + "type": "integer" + }, + "rgb": { + "type": "array", + "items": [ + { + "type": "integer" + }, + { + "type": "integer" + }, + { + "type": "integer" + } + ] + }, + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + }, + "required": ["label"] + } + } + } + })""""; +} \ No newline at end of file diff --git a/core/hailo/plugins/dsp/upload/gsthailoupload.cpp b/core/hailo/plugins/dsp/upload/gsthailoupload.cpp new file mode 100644 index 0000000..284aee7 --- /dev/null +++ b/core/hailo/plugins/dsp/upload/gsthailoupload.cpp @@ -0,0 +1,147 @@ +#include "dsp/gsthailodspbasetransform.hpp" +#include "dsp/upload/gsthailoupload.hpp" +#include +#include +#include "dsp/gsthailodsp.h" + +GST_DEBUG_CATEGORY_STATIC(gst_hailo_upload_debug); +#define GST_CAT_DEFAULT gst_hailo_upload_debug + +#define _do_init \ + GST_DEBUG_CATEGORY_INIT(gst_hailo_upload_debug, "hailoupload", 0, "Hailo Upload"); +#define gst_hailo_upload_parent_class parent_class +G_DEFINE_TYPE_WITH_CODE(GstHailoUpload, gst_hailo_upload, GST_TYPE_HAILO_DSP_BASE_TRANSFORM, _do_init); + +static GstFlowReturn +gst_hailo_upload_transform(GstBaseTransform *base_transform, GstBuffer *inbuf, GstBuffer *outbuf); + +static void gst_hailo_upload_set_property(GObject *object, guint prop_id, const GValue *value, GParamSpec *pspec); +static void gst_hailo_upload_get_property(GObject *object, guint prop_id, GValue *value, GParamSpec *pspec); + +static void +gst_hailo_upload_class_init(GstHailoUploadClass *klass) +{ + GObjectClass *const object_class = G_OBJECT_CLASS(klass); + GstBaseTransformClass *const base_transform_class = GST_BASE_TRANSFORM_CLASS(klass); + + object_class->set_property = gst_hailo_upload_set_property; + object_class->get_property = gst_hailo_upload_get_property; + + base_transform_class->transform = GST_DEBUG_FUNCPTR(gst_hailo_upload_transform); + + gst_element_class_set_static_metadata(GST_ELEMENT_CLASS(klass), + "Hailo Upload", + "Hailo/Tools", + "Uploads a buffer to the Hailo15 DSP memory", + "hailo.ai "); + + gst_element_class_add_pad_template(GST_ELEMENT_CLASS(klass), + gst_pad_template_new("src", GST_PAD_SRC, GST_PAD_ALWAYS, + GST_CAPS_ANY)); + gst_element_class_add_pad_template(GST_ELEMENT_CLASS(klass), + gst_pad_template_new("sink", GST_PAD_SINK, GST_PAD_ALWAYS, + GST_CAPS_ANY)); +} + +static void +gst_hailo_upload_init(GstHailoUpload *self) +{ +} + +static void gst_hailo_upload_set_property(GObject *object, guint prop_id, const GValue *value, GParamSpec *pspec) +{ + switch (prop_id) + { + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec); + break; + } +} + +static void gst_hailo_upload_get_property(GObject *object, guint prop_id, GValue *value, GParamSpec *pspec) +{ + switch (prop_id) + { + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec); + break; + } +} + +static GstFlowReturn +gst_hailo_upload_transform(GstBaseTransform *base_transform, GstBuffer *inbuf, GstBuffer *outbuf) +{ + GstHailoUpload *hailoupload = GST_HAILO_UPLOAD(base_transform); + + GST_DEBUG_OBJECT(hailoupload, "Transforming hailo DSP buffer"); + + GstCaps *incaps, *outcaps; + incaps = gst_pad_get_current_caps(base_transform->sinkpad); + outcaps = gst_pad_get_current_caps(base_transform->srcpad); + GstVideoInfo input_video_info; + gst_video_info_from_caps(&input_video_info, incaps); + + GST_DEBUG_OBJECT(hailoupload, "Performing memcopy to DSP contiguous memory"); + + GstVideoFrame video_frame; + gst_video_frame_map(&video_frame, &input_video_info, inbuf, GST_MAP_READ); + + GstVideoFormat format = GST_VIDEO_FRAME_FORMAT(&video_frame); + size_t image_height = GST_VIDEO_FRAME_HEIGHT(&video_frame); + size_t n_planes = GST_VIDEO_FRAME_N_PLANES(&video_frame); + GstFlowReturn ret = GST_FLOW_OK; + + switch (format) + { + case GST_VIDEO_FORMAT_NV12: + { + // Validate the number of planes + if (n_planes != 2) + { + GST_ERROR_OBJECT(hailoupload, "Invalid number of planes for NV12 format"); + ret = GST_FLOW_ERROR; + break; + } + + // Gather Y channel info + void *y_channel_data = (void *)GST_VIDEO_FRAME_PLANE_DATA(&video_frame, 0); + size_t y_channel_stride = GST_VIDEO_FRAME_PLANE_STRIDE(&video_frame, 0); + size_t y_channel_size = y_channel_stride * image_height; + + // Gather UV channel info + void *uv_channel_data = (void *)GST_VIDEO_FRAME_PLANE_DATA(&video_frame, 1); + size_t uv_channel_stride = GST_VIDEO_FRAME_PLANE_STRIDE(&video_frame, 1); + size_t uv_channel_size = uv_channel_stride * image_height / 2; + + // Copy Y channel to the contiguous memory + gsize size_copied = gst_buffer_fill(outbuf, 0, y_channel_data, y_channel_size); + GST_DEBUG_OBJECT(hailoupload, "NV12 format: Copied Y channel %d bytes from input buffer to output buffer", (int)size_copied); + + // Copy UV channel to the contiguous memory + size_copied = gst_buffer_fill(outbuf, y_channel_size, uv_channel_data, uv_channel_size); + GST_DEBUG_OBJECT(hailoupload, "NV12 format: Copied UV channel %d bytes from input buffer to output buffer", (int)size_copied); + break; + } + case GST_VIDEO_FORMAT_RGB: + { + void *date = (void *)GST_VIDEO_FRAME_PLANE_DATA(&video_frame, 0); + size_t stride = GST_VIDEO_FRAME_PLANE_STRIDE(&video_frame, 0); + size_t channel_size = stride * image_height; + gsize size_copied = gst_buffer_fill(outbuf, 0, date, channel_size); + GST_DEBUG_OBJECT(hailoupload, "RGB format: Copied %ld bytes to DSP memory", size_copied); + break; + } + default: + { + GST_ERROR_OBJECT(hailoupload, "Unsupported video format"); + ret = GST_FLOW_ERROR; + break; + } + } + + gst_video_frame_unmap(&video_frame); + gst_caps_unref(incaps); + gst_caps_unref(outcaps); + + return ret; +} \ No newline at end of file diff --git a/core/hailo/plugins/dsp/upload/gsthailoupload.hpp b/core/hailo/plugins/dsp/upload/gsthailoupload.hpp new file mode 100644 index 0000000..42590a0 --- /dev/null +++ b/core/hailo/plugins/dsp/upload/gsthailoupload.hpp @@ -0,0 +1,36 @@ +#pragma once + +#include "dsp/gsthailodspbasetransform.hpp" +#include + +G_BEGIN_DECLS + +#define GST_TYPE_HAILO_UPLOAD \ + (gst_hailo_upload_get_type()) +#define GST_HAILO_UPLOAD(obj) \ + (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_HAILO_UPLOAD,GstHailoUpload)) +#define GST_HAILO_UPLOAD_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_HAILO_UPLOAD,GstHailoUploadClass)) +#define GST_IS_HAILO_UPLOAD(obj) \ + (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_HAILO_UPLOAD)) +#define GST_IS_HAILO_UPLOAD_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_HAILO_UPLOAD)) +#define GST_HAILO_UPLOAD_GET_CLASS(obj) \ + (G_TYPE_INSTANCE_GET_CLASS((obj),GST_TYPE_HAILO_UPLOAD,GstHailoUploadClass)) + +typedef struct _GstHailoUpload GstHailoUpload; +typedef struct _GstHailoUploadClass GstHailoUploadClass; + +struct _GstHailoUpload { + GstHailoDspBaseTransform parent; + // add any additional instance variables here +}; + +struct _GstHailoUploadClass { + GstHailoDspBaseTransformClass parent_class; + // add any additional class variables here +}; + +GType gst_hailo_upload_get_type (void); + +G_END_DECLS \ No newline at end of file diff --git a/core/hailo/plugins/dsp/upload2/gsthailoupload2.cpp b/core/hailo/plugins/dsp/upload2/gsthailoupload2.cpp new file mode 100644 index 0000000..23bf7b1 --- /dev/null +++ b/core/hailo/plugins/dsp/upload2/gsthailoupload2.cpp @@ -0,0 +1,90 @@ +#include "dsp/gsthailodspbasetransform.hpp" +#include "dsp/upload2/gsthailoupload2.hpp" +#include +#include +#include "dsp/gsthailodsp.h" + +GST_DEBUG_CATEGORY_STATIC(gst_hailo_upload2_debug); +#define GST_CAT_DEFAULT gst_hailo_upload2_debug + +#define _do_init \ + GST_DEBUG_CATEGORY_INIT(gst_hailo_upload2_debug, "hailoupload2", 0, "Hailo Upload2"); +#define gst_hailo_upload2_parent_class parent_class +G_DEFINE_TYPE_WITH_CODE(GstHailoUpload2, gst_hailo_upload2, GST_TYPE_HAILO_DSP_BASE_TRANSFORM, _do_init); + +static void gst_hailo_upload2_set_property(GObject *object, guint prop_id, const GValue *value, GParamSpec *pspec); +static void gst_hailo_upload2_get_property(GObject *object, guint prop_id, GValue *value, GParamSpec *pspec); + +static gboolean gst_hailo_upload2_propose_allocation(GstBaseTransform *trans, GstQuery *decide_query, GstQuery *query); + +static void +gst_hailo_upload2_class_init(GstHailoUpload2Class *klass) +{ + GObjectClass *const object_class = G_OBJECT_CLASS(klass); + GstBaseTransformClass *const base_transform_class = GST_BASE_TRANSFORM_CLASS(klass); + + object_class->set_property = gst_hailo_upload2_set_property; + object_class->get_property = gst_hailo_upload2_get_property; + + base_transform_class->propose_allocation = GST_DEBUG_FUNCPTR(gst_hailo_upload2_propose_allocation); + + gst_element_class_set_static_metadata(GST_ELEMENT_CLASS(klass), + "Hailo Upload 2", + "Hailo/Tools", + "Manages a buffer pool using Hailo15 DSP memory, and propogates it upstream for pipeline usage.", + "hailo.ai "); + + gst_element_class_add_pad_template(GST_ELEMENT_CLASS(klass), + gst_pad_template_new("src", GST_PAD_SRC, GST_PAD_ALWAYS, + GST_CAPS_ANY)); + gst_element_class_add_pad_template(GST_ELEMENT_CLASS(klass), + gst_pad_template_new("sink", GST_PAD_SINK, GST_PAD_ALWAYS, + GST_CAPS_ANY)); +} + +static void +gst_hailo_upload2_init(GstHailoUpload2 *self) +{ +} + +static void gst_hailo_upload2_set_property(GObject *object, guint prop_id, const GValue *value, GParamSpec *pspec) +{ + switch (prop_id) + { + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec); + break; + } +} + +static void gst_hailo_upload2_get_property(GObject *object, guint prop_id, GValue *value, GParamSpec *pspec) +{ + switch (prop_id) + { + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec); + break; + } +} + +static gboolean +gst_hailo_upload2_propose_allocation (GstBaseTransform *trans, + GstQuery *decide_query, GstQuery *query) +{ + GstHailoUpload2 *hailoupload = GST_HAILO_UPLOAD2(trans); + GstHailoDspBaseTransform *dspbasetrans = GST_HAILO_DSP_BASE_TRANSFORM(trans); + GST_DEBUG_OBJECT(hailoupload, "hailoupload2 propose allocation callback"); + gboolean ret; + + GstBufferPool *buffer_pool = gst_base_transform_get_buffer_pool(trans); + // Get the size of a buffer from the config of the pool + GstStructure *config = gst_buffer_pool_get_config(buffer_pool); + guint buffer_size = 0; + gst_buffer_pool_config_get_params(config, NULL, &buffer_size, NULL, NULL); + + gst_query_add_allocation_pool(query, buffer_pool, buffer_size, dspbasetrans->bufferpool_min_size, dspbasetrans->bufferpool_max_size); + + gst_query_add_allocation_meta(query, GST_VIDEO_META_API_TYPE, NULL); + ret = GST_BASE_TRANSFORM_CLASS(parent_class)->propose_allocation(trans, decide_query, query); + return ret; +} \ No newline at end of file diff --git a/core/hailo/plugins/dsp/upload2/gsthailoupload2.hpp b/core/hailo/plugins/dsp/upload2/gsthailoupload2.hpp new file mode 100644 index 0000000..e368c11 --- /dev/null +++ b/core/hailo/plugins/dsp/upload2/gsthailoupload2.hpp @@ -0,0 +1,36 @@ +#pragma once + +#include "dsp/gsthailodspbasetransform.hpp" +#include + +G_BEGIN_DECLS + +#define GST_TYPE_HAILO_UPLOAD2 \ + (gst_hailo_upload2_get_type()) +#define GST_HAILO_UPLOAD2(obj) \ + (G_TYPE_CHECK_INSTANCE_CAST((obj),GST_TYPE_HAILO_UPLOAD2,GstHailoUpload2)) +#define GST_HAILO_UPLOAD2_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_CAST((klass),GST_TYPE_HAILO_UPLOAD2,GstHailoUpload2Class)) +#define GST_IS_HAILO_UPLOAD2(obj) \ + (G_TYPE_CHECK_INSTANCE_TYPE((obj),GST_TYPE_HAILO_UPLOAD2)) +#define GST_IS_HAILO_UPLOAD2_CLASS(klass) \ + (G_TYPE_CHECK_CLASS_TYPE((klass),GST_TYPE_HAILO_UPLOAD2)) +#define GST_HAILO_UPLOAD2_GET_CLASS(obj) \ + (G_TYPE_INSTANCE_GET_CLASS((obj),GST_TYPE_HAILO_UPLOAD2,GstHailoUpload2Class)) + +typedef struct _GstHailoUpload2 GstHailoUpload2; +typedef struct _GstHailoUpload2Class GstHailoUpload2Class; + +struct _GstHailoUpload2 { + GstHailoDspBaseTransform parent; + // add any additional instance variables here +}; + +struct _GstHailoUpload2Class { + GstHailoDspBaseTransformClass parent_class; + // add any additional class variables here +}; + +GType gst_hailo_upload2_get_type (void); + +G_END_DECLS \ No newline at end of file diff --git a/core/hailo/plugins/encoder/enc_common.h b/core/hailo/plugins/encoder/enc_common.h index dd8c34c..a7f3cd4 100644 --- a/core/hailo/plugins/encoder/enc_common.h +++ b/core/hailo/plugins/encoder/enc_common.h @@ -6,9 +6,59 @@ #include #include "video_encoder/base_type.h" #include "video_encoder/hevcencapi.h" -#include "video_encoder/encasiccontroller.h" -#include "video_encoder/instance.h" #include "video_encoder/encinputlinebuffer.h" +#include "video_encoder/ewl.h" + +#define MIN_QP_VALUE (0) +#define MAX_QP_VALUE (51) +#define MIN_BITRATE_VARIABLE_RANGE (0) +#define MAX_BITRATE_VARIABLE_RANGE (2000) + +#define MIN_BITRATE (10000) +#define MAX_BITRATE (40000000) +#define MIN_HRD_CPB_SIZE (10000) +#define MAX_HRD_CPB_SIZE (40000000) +#define MIN_MONITOR_FRAMES (10) +#define MAX_MONITOR_FRAMES (120) +#define MIN_INTRA_PIC_RATE (0) +#define MAX_INTRA_PIC_RATE (300) +#define MIN_GOP_LENGTH (1) +#define MAX_GOP_LENGTH (300) +#define MIN_GOP_SIZE (1) +// #define MAX_GOP_SIZE (8) - Defined in hevcencapi.h +#define MIN_QPHDR (-1) +#define MAX_QPHDR (MAX_QP_VALUE) +#define MIN_INTRA_QP_DELTA (-MAX_QP_VALUE) +#define MAX_INTRA_QP_DELTA (MAX_QP_VALUE) +#define MIN_FIXED_INTRA_QP (MIN_QP_VALUE) +#define MAX_FIXED_INTRA_QP (MAX_QP_VALUE) +#define MIN_BFRAME_QP_DELTA (-1) +#define MAX_BFRAME_QP_DELTA (MAX_QP_VALUE) + + +#define DEFAULT_UNCHANGED (-255) +#define DEFAULT_INPUT_FORMAT (VCENC_YUV420_SEMIPLANAR) +#define DEFAULT_HEVC_PROFILE (VCENC_HEVC_MAIN_PROFILE) +#define DEFAULT_HEVC_LEVEL (VCENC_HEVC_LEVEL_5_1) +#define DEFAULT_H264_PROFILE (VCENC_H264_HIGH_10_PROFILE) +#define DEFAULT_H264_LEVEL (VCENC_H264_LEVEL_5_2) +#define DEFAULT_INTRA_PIC_RATE (30) +#define DEFAULT_GOP_LENGTH (30) +#define DEFAULT_GOP_SIZE (MIN_GOP_SIZE) +#define DEFAULT_QPHDR (26) +#define DEFAULT_QPMIN (MIN_QP_VALUE) +#define DEFAULT_QPMAX (MAX_QP_VALUE) +#define DEFAULT_INTRA_QP_DELTA (-5) +#define DEFAULT_FIXED_INTRA_QP (MIN_QP_VALUE) +#define DEFAULT_BFRAME_QP_DELTA (MIN_BFRAME_QP_DELTA) +#define DEFAULT_BITRATE (MAX_BITRATE) +#define DEFAULT_TOL_MOVING_BITRATE (MAX_BITRATE_VARIABLE_RANGE) +#define DEFAULT_BITVAR_RANGE_I (MAX_BITRATE_VARIABLE_RANGE) +#define DEFAULT_BITVAR_RANGE_P (MAX_BITRATE_VARIABLE_RANGE) +#define DEFAULT_BITVAR_RANGE_B (MAX_BITRATE_VARIABLE_RANGE) +#define DEFAULT_MONITOR_FRAMES (30) +#define DEFAULT_HRD_CPB_SIZE (10000000) + typedef struct { i32 width; @@ -21,11 +71,8 @@ typedef struct { i32 frameRateDenom; /* Output frame rate denominator */ i32 picture_cnt; i32 picture_enc_cnt; - i32 idr_interval; + u32 idr_interval; i32 last_idr_picture_cnt; - u8 *lum; - u8 *cb; - u8 *cr; u32 validencodedframenumber; u32 alignment; @@ -35,15 +82,33 @@ typedef struct { i32 min_tr_size; /* Min transform size in pixels */ i32 tr_depth_intra; /* Max transform hierarchy depth */ i32 tr_depth_inter; /* Max transform hierarchy depth */ - i32 intraPicRate; /* IDR interval */ u32 outBufSizeMax; /* Max buf size in MB */ u32 roiMapDeltaQpBlockUnit; - u32 total_bits; + + // Rate Control Params + i32 qphdr; + u32 qpmin; + u32 qpmax; + i32 intra_qp_delta; + i32 bFrameQpDelta; + u32 fixed_intra_qp; + u32 bitrate; + u32 bitVarRangeI; + u32 bitVarRangeP; + u32 bitVarRangeB; + u32 tolMovingBitRate; + u32 monitorFrames; + u32 pictureRc; + u32 ctbRc; + u32 blockRcSize; /*size of block rate control : 2=16x16,1= 32x32, 0=64x64*/ + u32 pictureSkip; + u32 hrd; + u32 hrdCpbSize; - /* Moved from global space */ + u32 compressor; - /* SW/HW shared memories for input/output buffers */ - EWLLinearMem_t pictureMem; + /* SW/HW shared memories for output buffers */ + void * ewl; EWLLinearMem_t outbufMem; float sumsquareoferror; @@ -52,14 +117,17 @@ typedef struct { i32 maxerrorundertarget; long numbersquareoferror; + char * roiArea1; + char * roiArea2; + u32 gopSize; + u32 gopLength; VCEncIn encIn; VCEncOut encOut; bool codecH264; + u32 intraPicRate; u8 gopCfgOffset[MAX_GOP_SIZE + 1]; - inputLineBufferCfg inputCtbLineBuf; - // Slice data u8 *strmPtr; u32 multislice_encoding; @@ -74,7 +142,6 @@ typedef struct { int sum_costB; int last_gopsize; i32 nextGopSize; - u32 streamSize; VCEncPictureCodingType nextCodingType; -} EncoderParams; \ No newline at end of file +} EncoderParams; diff --git a/core/hailo/plugins/encoder/gopconfig.c b/core/hailo/plugins/encoder/gopconfig.c index 9977cce..3d3b023 100644 --- a/core/hailo/plugins/encoder/gopconfig.c +++ b/core/hailo/plugins/encoder/gopconfig.c @@ -284,12 +284,12 @@ static int HEVCReadGopConfig (char *fname, char **config, VCEncGopConfig *gopCfg return ret; } -int VCEncInitGopConfigs (int gopSize, char *gopCfgName, VCEncGopConfig *gopCfg, u8 *gopCfgOffset) +int VCEncInitGopConfigs (int gopSize, char *gopCfgName, VCEncGopConfig *gopCfg, u8 *gopCfgOffset, int bFrameQpDelta, bool codecH264) { int i, pre_load_num; char *fname = gopCfgName; char **default_configs[8] = { - RpsDefault_GOPSize_1, + codecH264?RpsDefault_H264_GOPSize_1:RpsDefault_GOPSize_1, RpsDefault_GOPSize_2, RpsDefault_GOPSize_3, RpsDefault_GOPSize_4, @@ -340,6 +340,7 @@ int VCEncInitGopConfigs (int gopSize, char *gopCfgName, VCEncGopConfig *gopCfg, } if (gopCfg->ltrInterval > 0) + { for(i = 0; i < (gopSize == 0 ? gopCfg->size : gopCfgOffset[gopSize]); i++) { // when use long-term, change P to B in default configs (used for last gop) @@ -347,6 +348,19 @@ int VCEncInitGopConfigs (int gopSize, char *gopCfgName, VCEncGopConfig *gopCfg, if (cfg->codingType == VCENC_PREDICTED_FRAME) cfg->codingType = VCENC_BIDIR_PREDICTED_FRAME; } + } + + //Compatible with old bFrameQpDelta setting + if (bFrameQpDelta >= 0) + { + for (i = 0; i < gopCfg->size; i++) + { + VCEncGopPicConfig *cfg = &(gopCfg->pGopPicCfg[i]); + if (cfg->codingType == VCENC_BIDIR_PREDICTED_FRAME) + cfg->QpOffset = bFrameQpDelta; + } + } + return 0; } @@ -388,38 +402,36 @@ VCEncPictureCodingType find_next_pic_internal(VCEncIn *encIn, EncoderParams *enc //next picture cnt enc_params->picture_cnt = picture_cnt_tmp + delta_poc_to_next; - //Handle Tail (seqence end or cut by an I frame) + //Handle Tail (cut by an I frame) { //just finished a GOP and will jump to a P frame if (encIn->gopPicIdx == 0 && delta_poc_to_next > 1) { - int gop_end_pic = enc_params->picture_cnt; - int gop_shorten = 0, gop_shorten_idr = 0, gop_shorten_tail = 0; - - //cut by an IDR - if ((enc_params->idr_interval) && ((gop_end_pic - enc_params->last_idr_picture_cnt) >= enc_params->idr_interval)) - gop_shorten_idr = 1 + ((gop_end_pic - enc_params->last_idr_picture_cnt) - enc_params->idr_interval); - - gop_shorten = gop_shorten_idr > gop_shorten_tail ? gop_shorten_idr : gop_shorten_tail; - - if (gop_shorten >= next_gop_size) - { - //for gopsize = 1 - enc_params->picture_cnt = picture_cnt_tmp + 1 - cur_poc; - } - else if (gop_shorten > 0) - { - //reduce gop size - const int max_reduced_gop_size = 4; - next_gop_size -= gop_shorten; - if (next_gop_size > max_reduced_gop_size) - next_gop_size = max_reduced_gop_size; - - idx = gopCfgOffset[next_gop_size]; - delta_poc_to_next = gopCfg->pGopPicCfg[idx].poc - cur_poc; - enc_params->picture_cnt = picture_cnt_tmp + delta_poc_to_next; - } - encIn->gopSize = next_gop_size; + int gop_end_pic = enc_params->picture_cnt; + int gop_shorten = 0; + + //cut by an IDR + if ((enc_params->idr_interval) && ((gop_end_pic - enc_params->last_idr_picture_cnt) >= enc_params->idr_interval)) + gop_shorten = 1 + ((gop_end_pic - enc_params->last_idr_picture_cnt) - enc_params->idr_interval); + + if (gop_shorten >= next_gop_size) + { + //for gopsize = 1 + enc_params->picture_cnt = picture_cnt_tmp + 1 - cur_poc; + } + else if (gop_shorten > 0) + { + //reduce gop size + const int max_reduced_gop_size = 4; + next_gop_size -= gop_shorten; + if (next_gop_size > max_reduced_gop_size) + next_gop_size = max_reduced_gop_size; + + idx = gopCfgOffset[next_gop_size]; + delta_poc_to_next = gopCfg->pGopPicCfg[idx].poc - cur_poc; + enc_params->picture_cnt = picture_cnt_tmp + delta_poc_to_next; + } + encIn->gopSize = next_gop_size; } encIn->poc += enc_params->picture_cnt - picture_cnt_tmp; diff --git a/core/hailo/plugins/encoder/gopconfig.h b/core/hailo/plugins/encoder/gopconfig.h index a49b4a1..88aac84 100644 --- a/core/hailo/plugins/encoder/gopconfig.h +++ b/core/hailo/plugins/encoder/gopconfig.h @@ -1,4 +1,4 @@ #pragma once #include "enc_common.h" -int VCEncInitGopConfigs (int gopSize, char *gopCfgName, VCEncGopConfig *gopCfg, u8 *gopCfgOffset); +int VCEncInitGopConfigs (int gopSize, char *gopCfgName, VCEncGopConfig *gopCfg, u8 *gopCfgOffset, int bFrameQpDelta, bool codecH264); VCEncPictureCodingType find_next_pic(EncoderParams *enc_params); \ No newline at end of file diff --git a/core/hailo/plugins/encoder/gsthailoenc.c b/core/hailo/plugins/encoder/gsthailoenc.c deleted file mode 100644 index 1d2bfd6..0000000 --- a/core/hailo/plugins/encoder/gsthailoenc.c +++ /dev/null @@ -1,641 +0,0 @@ -#ifdef HAVE_CONFIG_H -#include -#endif -#include -#include -/* for stats file handling */ -#include -// #include -#include -#include "gsthailoenc.h" - -GST_DEBUG_CATEGORY_STATIC(gst_hailoencoder_debug); -#define GST_CAT_DEFAULT gst_hailoencoder_debug - -enum -{ - NUM_OF_PROPS, -}; - -static void gst_hailoencoder_class_init (GstHailoEncClass * klass); -static void gst_hailoencoder_init (GstHailoEnc * hailoenc); -static void gst_hailoencoder_finalize (GObject * object); - -static gboolean gst_hailoencoder_start (GstVideoEncoder * encoder); -static gboolean gst_hailoencoder_stop (GstVideoEncoder * encoder); -static GstFlowReturn gst_hailoencoder_finish (GstVideoEncoder * encoder); -static gboolean gst_hailoencoder_set_format (GstVideoEncoder * encoder, - GstVideoCodecState * state); -static gboolean gst_hailoencoder_propose_allocation (GstVideoEncoder * encoder, - GstQuery * query); -static gboolean gst_hailoencoder_flush (GstVideoEncoder * encoder); - -static GstFlowReturn gst_hailoencoder_handle_frame (GstVideoEncoder * encoder, - GstVideoCodecFrame * frame); - -static void gst_hailoencoder_set_property (GObject * object, - guint prop_id, const GValue * value, GParamSpec * pspec); -static void gst_hailoencoder_get_property (GObject * object, - guint prop_id, GValue * value, GParamSpec * pspec); - - -static GstStaticPadTemplate sink_template = GST_STATIC_PAD_TEMPLATE("sink", - GST_PAD_SINK, - GST_PAD_ALWAYS, - GST_STATIC_CAPS("video/x-raw, " - "format=NV12, " - "width=(int)[16,MAX], " - "height=(int)[16,MAX], " - "framerate=(fraction)[0/1,MAX]")); - -static GstStaticPadTemplate src_template = GST_STATIC_PAD_TEMPLATE("src", - GST_PAD_SRC, - GST_PAD_ALWAYS, - GST_STATIC_CAPS("video/x-h265, " - // "framerate = (fraction) [0/1, MAX], " - // "width = (int) [ 16, MAX ], " "height = (int) [ 16, MAX ], " - "stream-format = (string) byte-stream, " - "alignment = (string) au, " - "profile = (string) { main, main-still-picture, " - "main-intra, main-10, main-10-intra }") - ); - -#define _do_init \ - GST_DEBUG_CATEGORY_INIT(gst_hailoencoder_debug, "hailoencoder", 0, "hailoencoder element"); -#define gst_hailoencoder_parent_class parent_class -G_DEFINE_TYPE_WITH_CODE(GstHailoEnc, gst_hailoencoder, GST_TYPE_VIDEO_ENCODER, _do_init); - -static void -gst_hailoencoder_class_init (GstHailoEncClass * klass) -{ - GObjectClass *gobject_class; - GstVideoEncoderClass *venc_class; - GstElementClass *element_class = GST_ELEMENT_CLASS(klass); - - gobject_class = (GObjectClass *) klass; - venc_class = (GstVideoEncoderClass *) klass; - - //parent_class = g_type_class_peek_parent (klass); - gst_element_class_add_pad_template (element_class, gst_static_pad_template_get(&sink_template)); - gst_element_class_add_pad_template (element_class, gst_static_pad_template_get(&src_template)); - gst_element_class_set_static_metadata(element_class, - "H265 Encoder", - "Encoder/Video", - "Encodes raw video into H265 format", - "hailo.ai "); - - gobject_class->set_property = gst_hailoencoder_set_property; - gobject_class->get_property = gst_hailoencoder_get_property; - - venc_class->start = gst_hailoencoder_start; - venc_class->stop = gst_hailoencoder_stop; - venc_class->finish = gst_hailoencoder_finish; - venc_class->handle_frame = gst_hailoencoder_handle_frame; - venc_class->set_format = gst_hailoencoder_set_format; - venc_class->propose_allocation = gst_hailoencoder_propose_allocation; - venc_class->flush = gst_hailoencoder_flush; - - gobject_class->finalize = gst_hailoencoder_finalize; - -} - -static void -gst_hailoencoder_init (GstHailoEnc * hailoenc) -{ - //GstHailoEncClass *klass = - // (GstHailoEncClass *) G_OBJECT_GET_CLASS (hailoenc); - EncoderParams *enc_params = &(hailoenc->enc_params); - GST_PAD_SET_ACCEPT_TEMPLATE (GST_VIDEO_ENCODER_SINK_PAD (hailoenc)); - hailoenc->apiVer = VCEncGetApiVersion(); - hailoenc->encBuild = VCEncGetBuild(); - memset(enc_params, 0, sizeof(EncoderParams)); - SetDefaultParameters(enc_params); - if(HW_ID_MAJOR_NUMBER(hailoenc->encBuild.hwBuild)<= 1) - { - //H2V1 only support 1. - enc_params->gopSize = 1; - } - memset(hailoenc->gopPicCfg, 0, sizeof(hailoenc->gopPicCfg)); - enc_params->encIn.gopConfig.pGopPicCfg = hailoenc->gopPicCfg; - hailoenc->opened = FALSE; - hailoenc->frameCntTotal = 0; -} - -static void -gst_hailoencoder_finalize (GObject * object) -{ - GstHailoEnc *hailoenc = (GstHailoEnc *) object; - GST_DEBUG_OBJECT(hailoenc, "hailoencoder finalize callback"); - - /* clean up remaining allocated data */ - - G_OBJECT_CLASS (parent_class)->finalize (object); -} - -static void update_params(GstHailoEnc *hailoenc, - GstVideoInfo * info) -{ - EncoderParams *enc_params = &(hailoenc->enc_params); - enc_params->width = GST_VIDEO_INFO_WIDTH (info); - enc_params->height = GST_VIDEO_INFO_HEIGHT (info); - - enc_params->frameRateNumer = GST_VIDEO_INFO_FPS_N (info); - enc_params->frameRateDenom = GST_VIDEO_INFO_FPS_D (info); - - switch (GST_VIDEO_INFO_FORMAT(info)) - { - case GST_VIDEO_FORMAT_NV12: - enc_params->inputFormat = VCENC_YUV420_SEMIPLANAR; - break; - case GST_VIDEO_FORMAT_NV21: - enc_params->inputFormat = VCENC_YUV420_SEMIPLANAR_VU; - break; - case GST_VIDEO_FORMAT_I420: - enc_params->inputFormat = VCENC_YUV420_PLANAR; - break; - default: - GST_ERROR_OBJECT(hailoenc, "Unsupported format %d", GST_VIDEO_INFO_FORMAT(info)); - break; - } -} - -static gboolean -gst_hailoencoder_set_format (GstVideoEncoder * encoder, - GstVideoCodecState * state) -{ - // GstCaps *other_caps; - GstCaps *allowed_caps; - GstCaps *icaps; - GstVideoCodecState *output_format; - GstHailoEnc *hailoenc = (GstHailoEnc *) encoder; - EncoderParams *enc_params = &(hailoenc->enc_params); - VCEncIn * pEncIn = &(enc_params->encIn); - - hailoenc->need_reopen = FALSE; - update_params(hailoenc, &(state->info)); - if (hailoenc->hevc_encoder == NULL) - { - /* Encoder initialization */ - if (OpenEncoder(&(hailoenc->hevc_encoder), enc_params) != 0) - { - return FALSE; - } - - /* Allocate input and output buffers */ - if (AllocRes(hailoenc->hevc_encoder, enc_params) != 0) - { - FreeRes(hailoenc->hevc_encoder, enc_params); - CloseEncoder(hailoenc->hevc_encoder); - // return -VCENC_MEMORY_ERROR; - return FALSE; - } - } - - pEncIn->timeIncrement = 0; - pEncIn->busOutBuf = enc_params->outbufMem.busAddress; - pEncIn->outBufSize = enc_params->outbufMem.size; - pEncIn->pOutBuf = enc_params->outbufMem.virtualAddress; - - - /* close old session */ - if (hailoenc->opened) { - } - - /* additional codec settings */ - - // GST_DEBUG_OBJECT (hailoenc, "Extracting common video information"); - /* fetch pix_fmt, fps, par, width, height... */ - - /* sanitize time base */ - //if (hailoenc->context->time_base.num <= 0 - // || hailoenc->context->time_base.den <= 0) - // goto insane_timebase; - - - //pix_fmt = hailoenc->context->pix_fmt; - - /* some codecs support more than one format, first auto-choose one */ - GST_DEBUG_OBJECT (hailoenc, "picking an output format ..."); - allowed_caps = gst_pad_get_allowed_caps (GST_VIDEO_ENCODER_SRC_PAD (encoder)); - if (!allowed_caps) { - GST_DEBUG_OBJECT (hailoenc, "... but no peer, using template caps"); - /* we need to copy because get_allowed_caps returns a ref, and - * get_pad_template_caps doesn't */ - allowed_caps = - gst_pad_get_pad_template_caps (GST_VIDEO_ENCODER_SRC_PAD (encoder)); - } - GST_DEBUG_OBJECT (hailoenc, "chose caps %" GST_PTR_FORMAT, allowed_caps); - - /* open codec */ - - /* is the colourspace correct? */ - //if (pix_fmt != hailoenc->context->pix_fmt) { - // gst_caps_unref (allowed_caps); - // goto pix_fmt_err; - //} - - // other_caps = gst_pad_get_allowed_caps (GST_VIDEO_ENCODER_SINK_PAD (encoder)); - /* try to set this caps on the other side */ - // if (!other_caps) { - // gst_caps_unref (allowed_caps); - // goto unsupported_codec; - // } - // GST_WARNING_OBJECT (hailoenc, "Other caps %" GST_PTR_FORMAT, allowed_caps); - - // icaps = gst_caps_intersect (allowed_caps, other_caps); - // gst_caps_unref (allowed_caps); - // gst_caps_unref (other_caps); - // if (gst_caps_is_empty (icaps)) { - // gst_caps_unref (icaps); - // goto unsupported_codec; - // } - icaps = gst_caps_fixate (allowed_caps); - - - //GST_DEBUG_OBJECT (hailoenc, "codec flags 0x%08x", hailoenc->context->flags); - - /* Store input state and set output state */ - if (hailoenc->input_state) - gst_video_codec_state_unref (hailoenc->input_state); - hailoenc->input_state = gst_video_codec_state_ref (state); - GST_DEBUG_OBJECT (hailoenc, "Setting output caps state %" GST_PTR_FORMAT, icaps); - - output_format = gst_video_encoder_set_output_state (encoder, icaps, state); - - gst_video_codec_state_unref (output_format); - - /* Store some tags */ - { - GstTagList *tags = gst_tag_list_new_empty (); - //const gchar *codec; - - //gst_tag_list_add (tags, GST_TAG_MERGE_REPLACE, GST_TAG_NOMINAL_BITRATE, - // (guint) hailoenc->context->bit_rate, NULL); - - gst_video_encoder_merge_tags (encoder, tags, GST_TAG_MERGE_REPLACE); - gst_tag_list_unref (tags); - } - gint max_delayed_frames = 5; - GstClockTime latency; - latency = gst_util_uint64_scale_ceil (GST_SECOND * 1, max_delayed_frames, 25); - gst_video_encoder_set_latency (GST_VIDEO_ENCODER (encoder), latency, latency); - - /* success! */ - hailoenc->opened = TRUE; - - return TRUE; - - /* ERRORS */ -// open_file_err: -// { -// GST_ELEMENT_ERROR (hailoenc, RESOURCE, OPEN_WRITE, -// (("Could not open file \"%s\" for writing."), hailoenc->filename), -// GST_ERROR_SYSTEM); -// return FALSE; -// } -// file_read_err: -// { -// GST_ELEMENT_ERROR (hailoenc, RESOURCE, READ, -// (("Could not get contents of file \"%s\"."), hailoenc->filename), -// GST_ERROR_SYSTEM); -// return FALSE; -// } - -//insane_timebase: -// { -// GST_ERROR_OBJECT (hailoenc, "Rejecting time base %d/%d", -// hailoenc->context->time_base.den, hailoenc->context->time_base.num); -// goto cleanup_stats_in; -// } -// unsupported_codec: -// { -// GST_ERROR ("Unsupported codec - no caps found"); -// return TRUE; -// } -// open_codec_fail: -// { -// //GST_DEBUG_OBJECT (hailoenc, "avenc_%s: Failed to open libav codec", -// // oclass->in_plugin->name); -// goto close_codec; -// } - -// pix_fmt_err: -// { -// //GST_DEBUG_OBJECT (hailoenc, -// // "avenc_%s: AV wants different colourspace (%d given, %d wanted)", -// // oclass->in_plugin->name, pix_fmt, hailoenc->context->pix_fmt); -// goto close_codec; -// } - -// bad_input_fmt: -// { -// //GST_DEBUG_OBJECT (hailoenc, "avenc_%s: Failed to determine input format", -// // oclass->in_plugin->name); -// goto close_codec; -// } -// close_codec: -// { -// goto cleanup_stats_in; -// } -// cleanup_stats_in: -// { -// //g_free (hailoenc->context->stats_in); -// return FALSE; -// } -} - - -static gboolean -gst_hailoencoder_propose_allocation (GstVideoEncoder * encoder, - GstQuery * query) -{ - GstHailoEnc *hailoenc = (GstHailoEnc *) encoder; - GST_DEBUG_OBJECT(hailoenc, "hailoencoder propose allocation callback"); - - gst_query_add_allocation_meta (query, GST_VIDEO_META_API_TYPE, NULL); - - return GST_VIDEO_ENCODER_CLASS (parent_class)->propose_allocation (encoder, - query); -} - -static GstFlowReturn -gst_hailoencoder_receive_packet (GstHailoEnc * hailoenc, u32 *address, u32 size, gboolean send) -{ - GstBuffer *outbuf; - GstVideoCodecFrame *frame; - - /* Get oldest frame */ - frame = gst_video_encoder_get_oldest_frame (GST_VIDEO_ENCODER(hailoenc)); - if (send) { - outbuf = gst_buffer_new_memdup(address, size); - frame->output_buffer = outbuf; - -// if (pkt->flags & AV_PKT_FLAG_KEY) -// GST_VIDEO_CODEC_FRAME_SET_SYNC_POINT (frame); -// else -// GST_VIDEO_CODEC_FRAME_UNSET_SYNC_POINT (frame); - } - -// // frame->dts = -// // gst_ffmpeg_time_ff_to_gst (pkt->dts, hailoenc->context->time_base); -// // /* This will lose some precision compared to setting the PTS from the input -// // * buffer directly, but that way we're sure PTS and DTS are consistent, in -// // * particular DTS should always be <= PTS -// // */ -// // frame->pts = -// // gst_ffmpeg_time_ff_to_gst (pkt->pts, hailoenc->context->time_base); - - return gst_video_encoder_finish_frame (GST_VIDEO_ENCODER (hailoenc), frame); -} - -static GstFlowReturn -gst_hailoencoder_first(GstVideoEncoder * encoder) -{ - GstHailoEnc *hailoenc = (GstHailoEnc *) encoder; - GstVideoCodecFrame *frame; - EncoderParams *enc_params = &(hailoenc->enc_params); - VCEncIn *pEncIn = &(enc_params->encIn); - VCEncOut *pEncOut = &(enc_params->encOut); - GstBuffer *outbuf; - - if (VCEncStrmStart(hailoenc->hevc_encoder, pEncIn, pEncOut) != VCENC_OK) - { - return GST_FLOW_ERROR; - } - frame = gst_video_encoder_get_oldest_frame (GST_VIDEO_ENCODER(hailoenc)); - outbuf = gst_buffer_new_memdup(enc_params->outbufMem.virtualAddress, pEncOut->streamSize); - - enc_params->total_bits += pEncOut->streamSize * 8; - enc_params->streamSize += pEncOut->streamSize; - - pEncIn->poc = 0; - - //default gop size as IPPP - pEncIn->gopSize = enc_params->nextGopSize = ((enc_params->gopSize == 0) ? 1 : enc_params->gopSize); - pEncIn->codingType = VCENC_INTRA_FRAME; - frame->output_buffer = outbuf; - - return gst_video_encoder_finish_subframe(encoder, frame); -} - -/* Callback function called by the encoder SW after "slice ready" - interrupt from HW. Note that this function is not necessarily called - after every slice i.e. it is possible that two or more slices are - completed between callbacks. -------------------------------------------------------------------------------*/ -void gst_hailoencoder_slice_ready(VCEncSliceReady *slice) -{ - u32 i; - u32 streamSize; - u8 *strmPtr; - GstHailoEnc *hailoenc = (GstHailoEnc *) slice->pAppData; - EncoderParams *enc_params = &(hailoenc->enc_params); - /* Here is possible to implement low-latency streaming by - * sending the complete slices before the whole frame is completed. */ - - if(enc_params->multislice_encoding&&(ENCH2_SLICE_READY_INTERRUPT)) - { - if (slice->slicesReadyPrev == 0) /* New frame */ - { - strmPtr = (u8 *)slice->pOutBuf; /* Pointer to beginning of frame */ - streamSize=0; - for(i=0;inalUnitInfoNum+slice->slicesReady;i++) - { - streamSize+=*(slice->sliceSizes+i); - } - gst_hailoencoder_receive_packet(hailoenc, (u32 *)strmPtr, streamSize, TRUE); - } - else - { - strmPtr = (u8 *)enc_params->strmPtr; /* Here we store the slice pointer */ - streamSize=0; - for(i=(slice->nalUnitInfoNum+slice->slicesReadyPrev);islicesReady+slice->nalUnitInfoNum;i++) - { - streamSize+=*(slice->sliceSizes+i); - } - gst_hailoencoder_receive_packet(hailoenc, (u32 *)strmPtr, streamSize, TRUE); - } - strmPtr+=streamSize; - /* Store the slice pointer for next callback */ - enc_params->strmPtr = strmPtr; - } -} - -static GstFlowReturn -gst_hailoencoder_handle_frame (GstVideoEncoder * encoder, - GstVideoCodecFrame * frame) -{ - GstHailoEnc *hailoenc = (GstHailoEnc *) encoder; - GstFlowReturn ret = GST_FLOW_ERROR; - VCEncRet enc_ret; - EncoderParams *enc_params = &(hailoenc->enc_params); - VCEncIn * pEncIn = &(enc_params->encIn); - VCEncOut * pEncOut = &(enc_params->encOut); - GstVideoFrame vframe; - - if (hailoenc->first) - { - hailoenc->first = false; - ret = gst_hailoencoder_first(encoder); - if (ret != GST_FLOW_OK) - { - GST_ERROR_OBJECT(hailoenc, "First buffer failed %d\n", ret); - return ret; - } - if (SetupInputBuffer(enc_params, pEncIn) == 0) - { - GST_ERROR_OBJECT(hailoenc, "Setup input buffer failed\n"); - return GST_FLOW_ERROR; - } - } - hailoenc->frameCntTotal++; - - enc_params->multislice_encoding=0; - enc_params->strmPtr = NULL; - gst_video_frame_map (&vframe, &(hailoenc->input_state->info), frame->input_buffer, GST_MAP_READ); - guint8 * luma = GST_VIDEO_FRAME_PLANE_DATA (&vframe, 0); - guint8 * chroma = GST_VIDEO_FRAME_PLANE_DATA (&vframe, 1); - guint src_offset, dst_offset; - for (int i = 0; i < enc_params->height; i++) - { - src_offset = i * GST_VIDEO_FRAME_PLANE_STRIDE(&vframe, 0); - dst_offset = i * enc_params->width; - memcpy(enc_params->lum + dst_offset, luma + src_offset, enc_params->width); - } - for (int i = 0; i < enc_params->height/2; i++) - { - src_offset = i * GST_VIDEO_FRAME_PLANE_STRIDE(&vframe, 1); - dst_offset = i * enc_params->width; - memcpy(enc_params->cb + dst_offset, chroma + src_offset, enc_params->width); - } - - enc_ret = EncodeFrame(enc_params, hailoenc->hevc_encoder, &gst_hailoencoder_slice_ready, hailoenc); - switch (enc_ret) - { - case VCENC_FRAME_READY: - enc_params->picture_enc_cnt++; - if (enc_params->encOut.streamSize == 0) - { - enc_params->picture_cnt++; - break; - } - else - { - if((enc_params->multislice_encoding==0)||(ENCH2_SLICE_READY_INTERRUPT==0)) - { - gst_hailoencoder_receive_packet (hailoenc, enc_params->outbufMem.virtualAddress, - pEncOut->streamSize, TRUE); - } - UpdateEncoder(enc_params, hailoenc->hevc_encoder); - } - break; - case VCENC_OUTPUT_BUFFER_OVERFLOW: - enc_params->picture_cnt++; - break; - default: - return ret; - break; - } - ret = GST_FLOW_OK; - return ret; -} - - -static void -gst_hailoencoder_set_property (GObject * object, - guint prop_id, const GValue * value, GParamSpec * pspec) -{ - GstHailoEnc *hailoenc = (GstHailoEnc *) (object); - - if (hailoenc->opened) { - GST_WARNING_OBJECT (hailoenc, - "Can't change properties once decoder is setup !"); - return; - } - - switch (prop_id) { - default: - G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); - break; - } -} - -static void -gst_hailoencoder_get_property (GObject * object, - guint prop_id, GValue * value, GParamSpec * pspec) -{ - // GstHailoEnc *hailoenc = (GstHailoEnc *) (object); - - switch (prop_id) { - default: - G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); - break; - } -} - -static gboolean -gst_hailoencoder_flush (GstVideoEncoder * encoder) -{ - return TRUE; -} - -static gboolean -gst_hailoencoder_start (GstVideoEncoder * encoder) -{ - GstHailoEnc *hailoenc = (GstHailoEnc *) encoder; - EncoderParams *enc_params = &(hailoenc->enc_params); - - if (VCEncInitGopConfigs (enc_params->gopSize, NULL, &(enc_params->encIn.gopConfig), enc_params->gopCfgOffset) != 0) - { - return FALSE; - } - - hailoenc->first = TRUE; - - gst_video_encoder_set_min_pts (encoder, GST_SECOND * 60 * 60 * 1000); - - return TRUE; -} - -static gboolean -gst_hailoencoder_stop (GstVideoEncoder * encoder) -{ - GstHailoEnc *hailoenc = (GstHailoEnc *) encoder; - EncoderParams *enc_params = &(hailoenc->enc_params); - FreeRes(hailoenc->hevc_encoder, enc_params); - CloseEncoder(hailoenc->hevc_encoder); - - return TRUE; -} - -static GstFlowReturn -gst_hailoencoder_finish (GstVideoEncoder * encoder) -{ - GstFlowReturn ret = GST_FLOW_OK; - GstHailoEnc *hailoenc = (GstHailoEnc *) encoder; - EncoderParams *enc_params = &(hailoenc->enc_params); - VCEncOut *pEncOut = &(enc_params->encOut); - VCEncIn *pEncIn = &(enc_params->encIn); - VCEncRet enc_ret; - GstBuffer *outbuf; - - /* End stream */ - enc_ret = VCEncStrmEnd(hailoenc->hevc_encoder, pEncIn, pEncOut); - if (enc_ret == VCENC_OK) - { - outbuf = gst_buffer_new_memdup(enc_params->outbufMem.virtualAddress, pEncOut->streamSize); - gst_pad_push(encoder->srcpad, outbuf); - enc_params->streamSize += pEncOut->streamSize; - } - - return ret; -} -static gboolean -plugin_init(GstPlugin *plugin) -{ - gst_element_register(plugin, "hailoencoder", GST_RANK_PRIMARY, GST_TYPE_HAILO_ENCODER); - return TRUE; -} - -GST_PLUGIN_DEFINE(GST_VERSION_MAJOR, GST_VERSION_MINOR, hailoencoder, "hailo encoder plugin", plugin_init, - VERSION, "unknown", PACKAGE, "https://hailo.ai/") diff --git a/core/hailo/plugins/encoder/gsthailoenc.h b/core/hailo/plugins/encoder/gsthailoenc.h deleted file mode 100644 index 6bbfce2..0000000 --- a/core/hailo/plugins/encoder/gsthailoenc.h +++ /dev/null @@ -1,42 +0,0 @@ -#pragma once - -#include -#include -#include "hailoencoder.h" - -G_BEGIN_DECLS - -typedef struct _GstHailoEnc GstHailoEnc; -typedef struct _GstHailoEncClass GstHailoEncClass; -#define GST_TYPE_HAILO_ENCODER (gst_hailoencoder_get_type()) -#define GST_HAILO_ENCODER(obj) (G_TYPE_CHECK_INSTANCE_CAST((obj), GST_TYPE_HAILO_ENCODER, GstHailoEnc)) -#define GST_HAILO_ENCODER_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST((klass), GST_TYPE_HAILO_ENCODER, GstHailoEncClass)) -#define GST_IS_HAILO_ENCODER(obj) (G_TYPE_CHECK_INSTANCE_TYPE((obj), GST_TYPE_HAILO_ENCODER)) -#define GST_IS_HAILO_ENCODER_CLASS(obj) (G_TYPE_CHECK_CLASS_TYPE((klass), GST_TYPE_HAILO_ENCODER)) -#define GST_HAILO_ENCODER_GET_CLASS(obj) (G_TYPE_INSTANCE_GET_CLASS ((obj), GST_TYPE_HAILO_ENCODER, GstHailoEncClass)) - -struct _GstHailoEnc -{ - GstVideoEncoder parent; - gchar* filename; - GstVideoCodecState *input_state; - VCEncApiVersion apiVer; - VCEncBuild encBuild; - VCEncGopPicConfig gopPicCfg[MAX_GOP_PIC_CONFIG_NUM]; - guint frameCntTotal; - EncoderParams enc_params; - VCEncInst hevc_encoder; - gboolean opened; - gboolean first; - gboolean need_reopen; - -}; - -struct _GstHailoEncClass -{ - GstVideoEncoderClass parent_class; -}; - -G_GNUC_INTERNAL GType gst_hailoencoder_get_type(void); - -G_END_DECLS diff --git a/core/hailo/plugins/encoder/gsthailoh264enc.c b/core/hailo/plugins/encoder/gsthailoh264enc.c new file mode 100644 index 0000000..6bf3b28 --- /dev/null +++ b/core/hailo/plugins/encoder/gsthailoh264enc.c @@ -0,0 +1,1369 @@ +#include +#include +/* for stats file handling */ +#include +#include +#include +#include "gsthailoh264enc.h" + + +/******************* +Property Definitions +*******************/ +enum +{ + PROP_0, + PROP_PROFILE, + PROP_LEVEL, + PROP_INTRA_PIC_RATE, + PROP_GOP_SIZE, + PROP_GOP_LENGTH, + PROP_QPHDR, + PROP_QPMIN, + PROP_QPMAX, + PROP_INTRA_QP_DELTA, + PROP_FIXED_INTRA_QP, + PROP_BFRAME_QP_DELTA, + PROP_BITRATE, + PROP_TOL_MOVING_BITRATE, + PROP_BITRATE_VAR_RANGE_I, + PROP_BITRATE_VAR_RANGE_P, + PROP_BITRATE_VAR_RANGE_B, + PROP_PICTURE_RC, + PROP_CTB_RC, + PROP_PICTURE_SKIP, + PROP_HRD, + PROP_MONITOR_FRAMES, + PROP_ROI_AREA_1, + PROP_ROI_AREA_2, + PROP_COMPRESSOR, + PROP_BLOCK_RC_SIZE, + PROP_HRD_CPB_SIZE, + NUM_OF_PROPS, +}; + +#define GST_TYPE_HAILOH264ENC_PROFILE (gst_hailoh264enc_profile_get_type()) +static GType +gst_hailoh264enc_profile_get_type(void) +{ + static GType hailoh264enc_profile_type = 0; + static const GEnumValue hailoh264enc_profiles[] = { + {VCENC_H264_BASE_PROFILE, "Base Profile", "base"}, + {VCENC_H264_MAIN_PROFILE, "Main Profile", "main"}, + {VCENC_H264_HIGH_PROFILE, "High Profile", "high"}, + {VCENC_H264_HIGH_10_PROFILE, "High 10 Profile", "high-10"}, + {0, NULL, NULL}, + }; + if (!hailoh264enc_profile_type) + { + hailoh264enc_profile_type = + g_enum_register_static("GstHailoH264EncProfile", hailoh264enc_profiles); + } + return hailoh264enc_profile_type; +} + +#define GST_TYPE_HAILOH264ENC_LEVEL (gst_hailoh264enc_level_get_type()) +static GType +gst_hailoh264enc_level_get_type(void) +{ + static GType hailoh264enc_level_type = 0; + static const GEnumValue hailoh264enc_levels[] = { + {VCENC_H264_LEVEL_1, "Level 1", "level-1"}, + {VCENC_H264_LEVEL_1_b, "Level 1b", "level-1-b"}, + {VCENC_H264_LEVEL_1_1, "Level 1.1", "level-1-1"}, + {VCENC_H264_LEVEL_1_2, "Level 1.2", "level-1-2"}, + {VCENC_H264_LEVEL_1_3, "Level 1.3", "level-1-3"}, + {VCENC_H264_LEVEL_2, "Level 2", "level-2"}, + {VCENC_H264_LEVEL_2_1, "Level 2.1", "level-2-1"}, + {VCENC_H264_LEVEL_2_2, "Level 2.2", "level-2-2"}, + {VCENC_H264_LEVEL_3, "Level 3", "level-3"}, + {VCENC_H264_LEVEL_3_1, "Level 3.1", "level-3-1"}, + {VCENC_H264_LEVEL_3_2, "Level 3.2", "level-3-2"}, + {VCENC_H264_LEVEL_4, "Level 4", "level-4"}, + {VCENC_H264_LEVEL_4_1, "Level 4.1", "level-4-1"}, + {VCENC_H264_LEVEL_4_2, "Level 4.2", "level-4-2"}, + {VCENC_H264_LEVEL_5, "Level 5", "level-5"}, + {VCENC_H264_LEVEL_5_1, "Level 5.1", "level-5-1"}, + {VCENC_H264_LEVEL_5_2, "Level 5.2", "level-5-2"}, + {0, NULL, NULL}, + }; + if (!hailoh264enc_level_type) + { + hailoh264enc_level_type = + g_enum_register_static("GstHailoH264EncLevel", hailoh264enc_levels); + } + return hailoh264enc_level_type; +} + +#define GST_TYPE_HAILOH264ENC_COMPRESSOR (gst_hailoh264enc_compressor_get_type()) +static GType +gst_hailoh264enc_compressor_get_type(void) +{ + /*Enable/Disable Embedded Compression + 0 = Disable Compression + 1 = Only Enable Luma Compression + 2 = Only Enable Chroma Compression + 3 = Enable Both Luma and Chroma Compression*/ + static GType hailoh264enc_compressor_type = 0; + static const GEnumValue hailoh264enc_compressors[] = { + {0, "Disable Compression", "disable"}, + {1, "Only Enable Luma Compression", "enable-luma"}, + {2, "Only Enable Chroma Compression", "enable-chroma"}, + {3, "Enable Both Luma and Chroma Compression", "enable-both"}, + {0, NULL, NULL}, + }; + if (!hailoh264enc_compressor_type) + { + hailoh264enc_compressor_type = + g_enum_register_static("GstHailoH264EncCompressor", hailoh264enc_compressors); + } + return hailoh264enc_compressor_type; +} + +#define GST_TYPE_HAILOH264ENC_BLOCK_RC_SIZE (gst_hailoh264enc_block_rc_size_get_type()) +static GType +gst_hailoh264enc_block_rc_size_get_type(void) +{ + static GType hailoh264enc_block_rc_size_type = 0; + static const GEnumValue hailoh264enc_block_rc_size_types[] = { + {0, "64X64", "64x64"}, + {1, "32X32", "32x32"}, + {2, "16X16", "16x16"}, + {0, NULL, NULL}, + }; + if (!hailoh264enc_block_rc_size_type) + { + hailoh264enc_block_rc_size_type = + g_enum_register_static("GstHailoH264EncBlockRcSize", hailoh264enc_block_rc_size_types); + } + return hailoh264enc_block_rc_size_type; +} + +/******************* +Function Definitions +*******************/ + +static void gst_hailoh264enc_class_init(GstHailoH264EncClass * klass); +static void gst_hailoh264enc_init(GstHailoH264Enc * hailoenc); +static void gst_hailoh264enc_set_property(GObject * object, + guint prop_id, const GValue * value, GParamSpec * pspec); +static void gst_hailoh264enc_get_property(GObject * object, + guint prop_id, GValue * value, GParamSpec * pspec); +static void gst_hailoh264enc_finalize(GObject * object); + +static gboolean gst_hailoh264enc_set_format(GstVideoEncoder * encoder, GstVideoCodecState * state); +static gboolean gst_hailoh264enc_propose_allocation(GstVideoEncoder * encoder, GstQuery * query); +static gboolean gst_hailoh264enc_flush(GstVideoEncoder * encoder); +static gboolean gst_hailoh264enc_start(GstVideoEncoder * encoder); +static gboolean gst_hailoh264enc_stop(GstVideoEncoder * encoder); +static GstFlowReturn gst_hailoh264enc_finish(GstVideoEncoder * encoder); +static GstFlowReturn gst_hailoh264enc_handle_frame(GstVideoEncoder * encoder, GstVideoCodecFrame * frame); + +/************ +Pad Templates +************/ + +static GstStaticPadTemplate sink_template = GST_STATIC_PAD_TEMPLATE("sink", + GST_PAD_SINK, + GST_PAD_ALWAYS, + GST_STATIC_CAPS("video/x-raw, " + "format=NV12, " + "width=(int)[16,MAX], " + "height=(int)[16,MAX], " + "framerate=(fraction)[0/1,MAX]")); + +static GstStaticPadTemplate src_template = GST_STATIC_PAD_TEMPLATE("src", + GST_PAD_SRC, + GST_PAD_ALWAYS, + GST_STATIC_CAPS("video/x-h264, " + "stream-format = (string) byte-stream, " + "alignment = (string) au, " + "profile = (string) { base, main, high, high-10 }")); + + +/************* +Init Functions +*************/ + +GST_DEBUG_CATEGORY_STATIC(gst_hailoh264enc_debug); +#define GST_CAT_DEFAULT gst_hailoh264enc_debug +#define _do_init \ + GST_DEBUG_CATEGORY_INIT(gst_hailoh264enc_debug, "hailoh264enc", 0, "hailoh264enc element"); +#define gst_hailoh264enc_parent_class parent_class +G_DEFINE_TYPE_WITH_CODE(GstHailoH264Enc, gst_hailoh264enc, GST_TYPE_VIDEO_ENCODER, _do_init); + +static void +gst_hailoh264enc_class_init(GstHailoH264EncClass * klass) +{ + GObjectClass *gobject_class; + GstVideoEncoderClass *venc_class; + GstElementClass *element_class = GST_ELEMENT_CLASS(klass); + + gobject_class = (GObjectClass *) klass; + venc_class = (GstVideoEncoderClass *) klass; + + //parent_class = g_type_class_peek_parent (klass); + gst_element_class_add_pad_template (element_class, gst_static_pad_template_get(&sink_template)); + gst_element_class_add_pad_template (element_class, gst_static_pad_template_get(&src_template)); + gst_element_class_set_static_metadata(element_class, + "H264 Encoder", + "Encoder/Video", + "Encodes raw video into H264 format", + "hailo.ai "); + + gobject_class->set_property = gst_hailoh264enc_set_property; + gobject_class->get_property = gst_hailoh264enc_get_property; + + + g_object_class_install_property(gobject_class, PROP_PROFILE, + g_param_spec_enum("profile", "encoder profile", "profile to encoder", + GST_TYPE_HAILOH264ENC_PROFILE, (gint)DEFAULT_H264_PROFILE, + (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); + g_object_class_install_property(gobject_class, PROP_LEVEL, + g_param_spec_enum("level", "encoder level", "level to encoder", + GST_TYPE_HAILOH264ENC_LEVEL, (gint)DEFAULT_H264_LEVEL, + (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); + g_object_class_install_property(gobject_class, PROP_INTRA_PIC_RATE, + g_param_spec_uint("intra-pic-rate", "IDR Interval", "I frames interval (0 - Dynamic IDR Interval)", + MIN_INTRA_PIC_RATE, MAX_INTRA_PIC_RATE, (guint)DEFAULT_INTRA_PIC_RATE, + (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS | GST_PARAM_MUTABLE_PLAYING))); + g_object_class_install_property(gobject_class, PROP_GOP_SIZE, + g_param_spec_uint("gop-size", "GOP Size", "GOP Size (1 - No B Frames)", + MIN_GOP_SIZE, MAX_GOP_SIZE, (guint)DEFAULT_GOP_SIZE, + (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS | GST_PARAM_MUTABLE_PLAYING))); + g_object_class_install_property(gobject_class, PROP_GOP_LENGTH, + g_param_spec_uint("gop-length", "GOP Length", "GOP Length", + MIN_GOP_LENGTH, MAX_GOP_LENGTH, (guint)DEFAULT_GOP_LENGTH, + (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS | GST_PARAM_MUTABLE_PLAYING))); + g_object_class_install_property(gobject_class, PROP_QPHDR, + g_param_spec_int("qp-hdr", "Initial target QP", "Initial target QP, -1 = Encoder calculates initial QP", + MIN_QPHDR, MAX_QPHDR, (gint)DEFAULT_QPHDR, + (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS | GST_PARAM_MUTABLE_PLAYING))); + g_object_class_install_property(gobject_class, PROP_QPMIN, + g_param_spec_uint("qp-min", "QP Min", "Minimum frame header QP", + MIN_QP_VALUE, MAX_QP_VALUE, (guint)DEFAULT_QPMIN, + (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS | GST_PARAM_MUTABLE_PLAYING))); + g_object_class_install_property(gobject_class, PROP_QPMAX, + g_param_spec_uint("qp-max", "QP Max", "Maximum frame header QP", + MIN_QP_VALUE, MAX_QP_VALUE, (guint)DEFAULT_QPMAX, + (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS | GST_PARAM_MUTABLE_PLAYING))); + g_object_class_install_property(gobject_class, PROP_INTRA_QP_DELTA, + g_param_spec_int("intra-qp-delta", "Intra QP delta", "QP difference between target QP and intra frame QP", + MIN_INTRA_QP_DELTA, MAX_INTRA_QP_DELTA, (gint)DEFAULT_INTRA_QP_DELTA, + (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS | GST_PARAM_MUTABLE_PLAYING))); + g_object_class_install_property(gobject_class, PROP_FIXED_INTRA_QP, + g_param_spec_uint("fixed-intra-qp", "Fixed Intra QP", "Use fixed QP value for every intra frame in stream, 0 = disabled", + MIN_FIXED_INTRA_QP, MAX_FIXED_INTRA_QP, (guint)DEFAULT_FIXED_INTRA_QP, + (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS | GST_PARAM_MUTABLE_PLAYING))); + g_object_class_install_property(gobject_class, PROP_BFRAME_QP_DELTA, + g_param_spec_int("bframe-qp-delta", "BFrame QP Delta", "QP difference between BFrame QP and target QP, -1 = Disabled", + MIN_BFRAME_QP_DELTA, MAX_BFRAME_QP_DELTA, (guint)DEFAULT_BFRAME_QP_DELTA, + (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS | GST_PARAM_MUTABLE_PLAYING))); + g_object_class_install_property(gobject_class, PROP_BITRATE, + g_param_spec_uint("bitrate", "Target bitrate", "Target bitrate for rate control in bits/second", + MIN_BITRATE, MAX_BITRATE, (guint)DEFAULT_BITRATE, + (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS | GST_PARAM_MUTABLE_PLAYING))); + g_object_class_install_property(gobject_class, PROP_TOL_MOVING_BITRATE, + g_param_spec_uint("tol-moving-bitrate", "Tolerance moving bitrate", "Percent tolerance over target bitrate of moving bit rate", + MIN_BITRATE_VARIABLE_RANGE, MAX_BITRATE_VARIABLE_RANGE, (guint)DEFAULT_TOL_MOVING_BITRATE, + (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS | GST_PARAM_MUTABLE_PLAYING))); + g_object_class_install_property(gobject_class, PROP_BITRATE_VAR_RANGE_I, + g_param_spec_uint("bitvar-range-i", "Bitrate percent variation I frame", "Percent variations over average bits per frame for I frame", + MIN_BITRATE_VARIABLE_RANGE, MAX_BITRATE_VARIABLE_RANGE, (guint)DEFAULT_BITVAR_RANGE_I, + (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS | GST_PARAM_MUTABLE_PLAYING))); + g_object_class_install_property(gobject_class, PROP_BITRATE_VAR_RANGE_P, + g_param_spec_uint("bitvar-range-p", "Bitrate percent variation P frame", "Percent variations over average bits per frame for P frame", + MIN_BITRATE_VARIABLE_RANGE, MAX_BITRATE_VARIABLE_RANGE, (guint)DEFAULT_BITVAR_RANGE_P, + (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS | GST_PARAM_MUTABLE_PLAYING))); + g_object_class_install_property(gobject_class, PROP_BITRATE_VAR_RANGE_B, + g_param_spec_uint("bitvar-range-b", "Bitrate percent variation B frame", "Percent variations over average bits per frame for B frame", + MIN_BITRATE_VARIABLE_RANGE, MAX_BITRATE_VARIABLE_RANGE, (guint)DEFAULT_BITVAR_RANGE_B, + (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS | GST_PARAM_MUTABLE_PLAYING))); + g_object_class_install_property(gobject_class, PROP_PICTURE_RC, + g_param_spec_boolean("picture-rc", "Picture Rate Control", "Adjust QP between pictures", true, + (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS | GST_PARAM_MUTABLE_PLAYING))); + g_object_class_install_property(gobject_class, PROP_CTB_RC, + g_param_spec_boolean("ctb-rc", "Block Rate Control", "Adaptive adjustment of QP inside frame", false, + (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS | GST_PARAM_MUTABLE_PLAYING))); + g_object_class_install_property(gobject_class, PROP_PICTURE_SKIP, + g_param_spec_boolean("picture-skip", "Picture Skip", "Allow rate control to skip pictures", false, + (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS | GST_PARAM_MUTABLE_PLAYING))); + g_object_class_install_property(gobject_class, PROP_HRD, + g_param_spec_boolean("hrd", "Picture Rate Control", "Restricts the instantaneous bitrate and total bit amount of every coded picture.", false, + (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS | GST_PARAM_MUTABLE_PLAYING))); + g_object_class_install_property(gobject_class, PROP_MONITOR_FRAMES, + g_param_spec_uint("monitor-frames", "Monitor Frames", "How many frames will be monitored for moving bit rate. Default is using framerate", + MIN_MONITOR_FRAMES, MAX_MONITOR_FRAMES, (gint)DEFAULT_MONITOR_FRAMES, + (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS | GST_PARAM_MUTABLE_PLAYING))); + g_object_class_install_property(gobject_class, PROP_ROI_AREA_1, + g_param_spec_string("roi-area1", "ROI Area and QP Delta", + "Specifying rectangular area of CTBs as Region Of Interest with lower QP, left:top:right:bottom:delta_qp format ", NULL, + (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS | GST_PARAM_MUTABLE_PLAYING))); + g_object_class_install_property(gobject_class, PROP_ROI_AREA_2, + g_param_spec_string("roi-area2", "ROI Area and QP Delta", + "Specifying rectangular area of CTBs as Region Of Interest with lower QP, left:top:right:bottom:delta_qp format ", NULL, + (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS | GST_PARAM_MUTABLE_PLAYING))); + g_object_class_install_property(gobject_class, PROP_COMPRESSOR, + g_param_spec_enum("compressor", "Compressor", "Enable/Disable Embedded Compression", + GST_TYPE_HAILOH264ENC_COMPRESSOR, (guint)3, + (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); + g_object_class_install_property(gobject_class, PROP_BLOCK_RC_SIZE, + g_param_spec_enum("block-rc-size", "Block Rate Control Size", "Size of block rate control", + GST_TYPE_HAILOH264ENC_BLOCK_RC_SIZE, (guint)0, + (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS | GST_PARAM_MUTABLE_PLAYING))); + g_object_class_install_property(gobject_class, PROP_HRD_CPB_SIZE, + g_param_spec_uint("hrd-cpb-size", "HRD Coded Picture Buffer size", "Buffer size used by the HRD model in bits", + MIN_HRD_CPB_SIZE, MAX_HRD_CPB_SIZE, (guint)DEFAULT_HRD_CPB_SIZE, + (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS | GST_PARAM_MUTABLE_PLAYING))); + + venc_class->start = gst_hailoh264enc_start; + venc_class->stop = gst_hailoh264enc_stop; + venc_class->finish = gst_hailoh264enc_finish; + venc_class->handle_frame = gst_hailoh264enc_handle_frame; + venc_class->set_format = gst_hailoh264enc_set_format; + venc_class->propose_allocation = gst_hailoh264enc_propose_allocation; + venc_class->flush = gst_hailoh264enc_flush; + + gobject_class->finalize = gst_hailoh264enc_finalize; + +} + +static void +gst_hailoh264enc_init(GstHailoH264Enc * hailoenc) +{ + //GstHailoH264EncClass *klass = + // (GstHailoH264EncClass *) G_OBJECT_GET_CLASS (hailoenc); + EncoderParams *enc_params = &(hailoenc->enc_params); + GST_PAD_SET_ACCEPT_TEMPLATE (GST_VIDEO_ENCODER_SINK_PAD (hailoenc)); + hailoenc->apiVer = VCEncGetApiVersion(); + hailoenc->encBuild = VCEncGetBuild(); + hailoenc->stream_restart = FALSE; + memset(enc_params, 0, sizeof(EncoderParams)); + SetDefaultParameters(enc_params, true); + if(HW_ID_MAJOR_NUMBER(hailoenc->encBuild.hwBuild)<= 1) + { + //H2V1 only support 1. + enc_params->gopSize = 1; + } + memset(hailoenc->gopPicCfg, 0, sizeof(hailoenc->gopPicCfg)); + hailoenc->h264_encoder = NULL; + enc_params->encIn.gopConfig.pGopPicCfg = hailoenc->gopPicCfg; +} + +/************************ +GObject Virtual Functions +************************/ + +static void +gst_hailoh264enc_get_property(GObject * object, + guint prop_id, GValue * value, GParamSpec * pspec) +{ + GstHailoH264Enc *hailoenc = (GstHailoH264Enc *) (object); + + switch (prop_id) { + case PROP_PROFILE: + GST_OBJECT_LOCK(hailoenc); + g_value_set_enum(value, (gint)hailoenc->enc_params.profile); + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_LEVEL: + GST_OBJECT_LOCK(hailoenc); + g_value_set_enum(value, (gint)hailoenc->enc_params.level); + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_INTRA_PIC_RATE: + GST_OBJECT_LOCK(hailoenc); + g_value_set_uint(value, (guint)hailoenc->enc_params.intraPicRate); + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_GOP_SIZE: + GST_OBJECT_LOCK(hailoenc); + g_value_set_uint(value, (guint)hailoenc->enc_params.gopSize); + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_GOP_LENGTH: + GST_OBJECT_LOCK(hailoenc); + g_value_set_uint(value, (guint)hailoenc->enc_params.gopLength); + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_QPHDR: + GST_OBJECT_LOCK(hailoenc); + g_value_set_int(value, (gint)hailoenc->enc_params.qphdr); + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_QPMIN: + GST_OBJECT_LOCK(hailoenc); + g_value_set_uint(value, (guint)hailoenc->enc_params.qpmin); + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_QPMAX: + GST_OBJECT_LOCK(hailoenc); + g_value_set_uint(value, (guint)hailoenc->enc_params.qpmax); + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_INTRA_QP_DELTA: + GST_OBJECT_LOCK(hailoenc); + g_value_set_int(value, (gint)hailoenc->enc_params.intra_qp_delta); + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_FIXED_INTRA_QP: + GST_OBJECT_LOCK(hailoenc); + g_value_set_uint(value, (guint)hailoenc->enc_params.fixed_intra_qp); + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_BFRAME_QP_DELTA: + GST_OBJECT_LOCK(hailoenc); + g_value_set_int(value, (gint)hailoenc->enc_params.bFrameQpDelta); + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_BITRATE: + GST_OBJECT_LOCK(hailoenc); + g_value_set_uint(value, (guint)hailoenc->enc_params.bitrate); + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_TOL_MOVING_BITRATE: + GST_OBJECT_LOCK(hailoenc); + g_value_set_uint(value, (guint)hailoenc->enc_params.tolMovingBitRate); + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_BITRATE_VAR_RANGE_I: + GST_OBJECT_LOCK(hailoenc); + g_value_set_uint(value, (guint)hailoenc->enc_params.bitVarRangeI); + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_BITRATE_VAR_RANGE_P: + GST_OBJECT_LOCK(hailoenc); + g_value_set_uint(value, (guint)hailoenc->enc_params.bitVarRangeP); + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_BITRATE_VAR_RANGE_B: + GST_OBJECT_LOCK(hailoenc); + g_value_set_uint(value, (guint)hailoenc->enc_params.bitVarRangeB); + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_MONITOR_FRAMES: + GST_OBJECT_LOCK(hailoenc); + g_value_set_uint(value, (guint)hailoenc->enc_params.monitorFrames); + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_PICTURE_RC: + GST_OBJECT_LOCK(hailoenc); + gboolean picture_rc = hailoenc->enc_params.pictureRc == 1 ? TRUE : FALSE; + g_value_set_boolean(value, picture_rc); + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_CTB_RC: + GST_OBJECT_LOCK(hailoenc); + gboolean ctb_rc = hailoenc->enc_params.ctbRc == 1 ? TRUE : FALSE; + g_value_set_boolean(value, ctb_rc); + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_PICTURE_SKIP: + GST_OBJECT_LOCK(hailoenc); + gboolean picture_skip = hailoenc->enc_params.pictureSkip == 1 ? TRUE : FALSE; + g_value_set_boolean(value, picture_skip); + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_HRD: + GST_OBJECT_LOCK(hailoenc); + gboolean hrd = hailoenc->enc_params.hrd == 1 ? TRUE : FALSE; + g_value_set_boolean(value, hrd); + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_ROI_AREA_1: + GST_OBJECT_LOCK(hailoenc); + g_value_set_string(value, (const gchar *)hailoenc->enc_params.roiArea1); + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_ROI_AREA_2: + GST_OBJECT_LOCK(hailoenc); + g_value_set_string(value, (const gchar *)hailoenc->enc_params.roiArea2); + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_COMPRESSOR: + GST_OBJECT_LOCK(hailoenc); + g_value_set_enum(value, (gint)hailoenc->enc_params.compressor); + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_BLOCK_RC_SIZE: + GST_OBJECT_LOCK(hailoenc); + g_value_set_enum(value, (gint)hailoenc->enc_params.blockRcSize); + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_HRD_CPB_SIZE: + GST_OBJECT_LOCK(hailoenc); + g_value_set_uint(value, (guint)hailoenc->enc_params.hrdCpbSize); + GST_OBJECT_UNLOCK(hailoenc); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +gst_hailoh264enc_set_property(GObject * object, + guint prop_id, const GValue * value, GParamSpec * pspec) +{ + GstHailoH264Enc *hailoenc = (GstHailoH264Enc *) (object); + hailoenc->update_config=FALSE; + + switch (prop_id) { + case PROP_PROFILE: + GST_OBJECT_LOCK(hailoenc); + hailoenc->enc_params.profile = (VCEncProfile)g_value_get_enum(value); + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_LEVEL: + GST_OBJECT_LOCK(hailoenc); + hailoenc->enc_params.level = (VCEncLevel)g_value_get_enum(value); + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_INTRA_PIC_RATE: + GST_OBJECT_LOCK(hailoenc); + hailoenc->enc_params.intraPicRate = g_value_get_uint(value); + hailoenc->update_config = TRUE; + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_GOP_SIZE: + GST_OBJECT_LOCK(hailoenc); + hailoenc->enc_params.gopSize = g_value_get_uint(value); + hailoenc->update_config = TRUE; + hailoenc->update_gop_size = TRUE; + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_GOP_LENGTH: + GST_OBJECT_LOCK(hailoenc); + hailoenc->enc_params.gopLength = g_value_get_uint(value); + hailoenc->update_config = TRUE; + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_QPHDR: + GST_OBJECT_LOCK(hailoenc); + hailoenc->enc_params.qphdr = g_value_get_int(value); + hailoenc->update_config = TRUE; + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_QPMIN: + GST_OBJECT_LOCK(hailoenc); + hailoenc->enc_params.qpmin = g_value_get_uint(value); + hailoenc->update_config = TRUE; + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_QPMAX: + GST_OBJECT_LOCK(hailoenc); + hailoenc->enc_params.qpmax = g_value_get_uint(value); + hailoenc->update_config = TRUE; + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_INTRA_QP_DELTA: + GST_OBJECT_LOCK(hailoenc); + hailoenc->enc_params.intra_qp_delta = g_value_get_int(value); + hailoenc->update_config = TRUE; + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_FIXED_INTRA_QP: + GST_OBJECT_LOCK(hailoenc); + hailoenc->enc_params.fixed_intra_qp = g_value_get_uint(value); + hailoenc->update_config = TRUE; + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_BFRAME_QP_DELTA: + GST_OBJECT_LOCK(hailoenc); + hailoenc->enc_params.bFrameQpDelta = g_value_get_int(value); + hailoenc->update_config = TRUE; + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_BITRATE: + GST_OBJECT_LOCK(hailoenc); + hailoenc->enc_params.bitrate = g_value_get_uint(value); + hailoenc->update_config = TRUE; + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_TOL_MOVING_BITRATE: + GST_OBJECT_LOCK(hailoenc); + hailoenc->enc_params.tolMovingBitRate = g_value_get_uint(value); + hailoenc->update_config = TRUE; + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_BITRATE_VAR_RANGE_I: + GST_OBJECT_LOCK(hailoenc); + hailoenc->enc_params.bitVarRangeI = g_value_get_uint(value); + hailoenc->update_config = TRUE; + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_BITRATE_VAR_RANGE_P: + GST_OBJECT_LOCK(hailoenc); + hailoenc->enc_params.bitVarRangeP = g_value_get_uint(value); + hailoenc->update_config = TRUE; + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_BITRATE_VAR_RANGE_B: + GST_OBJECT_LOCK(hailoenc); + hailoenc->enc_params.bitVarRangeB = g_value_get_uint(value); + hailoenc->update_config = TRUE; + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_MONITOR_FRAMES: + GST_OBJECT_LOCK(hailoenc); + hailoenc->enc_params.monitorFrames = g_value_get_uint(value); + hailoenc->update_config = TRUE; + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_PICTURE_RC: + GST_OBJECT_LOCK(hailoenc); + hailoenc->enc_params.pictureRc = g_value_get_boolean(value) ? 1 : 0; + hailoenc->update_config = TRUE; + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_CTB_RC: + GST_OBJECT_LOCK(hailoenc); + hailoenc->enc_params.ctbRc = g_value_get_boolean(value) ? 1 : 0; + hailoenc->update_config = TRUE; + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_PICTURE_SKIP: + GST_OBJECT_LOCK(hailoenc); + hailoenc->enc_params.pictureSkip = g_value_get_boolean(value) ? 1 : 0; + hailoenc->update_config = TRUE; + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_HRD: + GST_OBJECT_LOCK(hailoenc); + hailoenc->enc_params.hrd = g_value_get_boolean(value) ? 1 : 0; + hailoenc->update_config = TRUE; + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_ROI_AREA_1: + GST_OBJECT_LOCK(hailoenc); + hailoenc->enc_params.roiArea1 = (char *)g_value_get_string(value); + hailoenc->update_config = TRUE; + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_ROI_AREA_2: + GST_OBJECT_LOCK(hailoenc); + hailoenc->enc_params.roiArea2 = (char *)g_value_get_string(value); + hailoenc->update_config = TRUE; + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_COMPRESSOR: + GST_OBJECT_LOCK(hailoenc); + hailoenc->enc_params.compressor = (u32)g_value_get_enum(value); + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_BLOCK_RC_SIZE: + GST_OBJECT_LOCK(hailoenc); + hailoenc->enc_params.blockRcSize = (u32)g_value_get_enum(value); + hailoenc->update_config = TRUE; + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_HRD_CPB_SIZE: + GST_OBJECT_LOCK(hailoenc); + hailoenc->enc_params.hrdCpbSize = g_value_get_uint(value); + hailoenc->update_config = TRUE; + GST_OBJECT_UNLOCK(hailoenc); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +gst_hailoh264enc_finalize(GObject * object) +{ + GstHailoH264Enc *hailoenc = (GstHailoH264Enc *) object; + GST_DEBUG_OBJECT(hailoenc, "hailoh264enc finalize callback"); + + /* clean up remaining allocated data */ + + G_OBJECT_CLASS (parent_class)->finalize (object); +} + +/***************** +Internal Functions +*****************/ + + +/** + * Gets the time difference between 2 time specs in milliseconds. + * @param[in] after The second time spec. + * @param[in] before The first time spec.\ + * @returns The time differnece in milliseconds. + */ +int64_t gst_hailoh264enc_difftimespec_ms(const struct timespec after, const struct timespec before) +{ + return ((int64_t)after.tv_sec - (int64_t)before.tv_sec) * (int64_t)1000 + + ((int64_t)after.tv_nsec - (int64_t)before.tv_nsec) / 1000000; +} + +/** + * Updates the encoder with the input video info. + * + * @param[in] hailoenc The GstHailoH264Enc object. + * @param[in] info A GstVideoInfo object containing the input video info for this pipeline. + * @returns TRUE if the encoder parameters were updated, FALSE otherwise. + * @note The updated data is the resolution, framerate and input format. + */ +static gboolean +gst_hailoh264enc_update_params(GstHailoH264Enc *hailoenc, GstVideoInfo * info) +{ + gboolean updated_params = FALSE; + EncoderParams *enc_params = &(hailoenc->enc_params); + + if (enc_params->width != GST_VIDEO_INFO_WIDTH (info) || + enc_params->height != GST_VIDEO_INFO_HEIGHT (info)) + { + enc_params->width = GST_VIDEO_INFO_WIDTH (info); + enc_params->height = GST_VIDEO_INFO_HEIGHT (info); + updated_params = TRUE; + } + + if (enc_params->frameRateNumer != GST_VIDEO_INFO_FPS_N (info) || + enc_params->frameRateDenom != GST_VIDEO_INFO_FPS_D (info)) + { + enc_params->frameRateNumer = GST_VIDEO_INFO_FPS_N (info); + enc_params->frameRateDenom = GST_VIDEO_INFO_FPS_D (info); + updated_params = TRUE; + } + + switch (GST_VIDEO_INFO_FORMAT(info)) + { + case GST_VIDEO_FORMAT_NV12: + enc_params->inputFormat = VCENC_YUV420_SEMIPLANAR; + break; + case GST_VIDEO_FORMAT_NV21: + enc_params->inputFormat = VCENC_YUV420_SEMIPLANAR_VU; + break; + case GST_VIDEO_FORMAT_I420: + enc_params->inputFormat = VCENC_YUV420_PLANAR; + break; + default: + GST_ERROR_OBJECT(hailoenc, "Unsupported format %d", GST_VIDEO_INFO_FORMAT(info)); + break; + } + return updated_params; +} + +/** + * Updated the encoder parameters with the physical addresses of the current input buffer. + * + * @param[in] hailoenc The GstHailoH264Enc object. + * @param[in] frame The GstVideoCodecFrame object with the input GstBuffer inside. + * @return GST_FLOW_OK on success, GST_FLOW_ERROR on failure. + * @note The function will fail when it cannot get the physical address or the memory is non-continous. + */ +static GstFlowReturn gst_hailoh264enc_update_input_buffer(GstHailoH264Enc * hailoenc, + GstVideoCodecFrame * frame) +{ + EncoderParams *enc_params = &(hailoenc->enc_params); + GstVideoFrame vframe; + int ret; + + // Get the Virtal addresses of input buffer luma and chroma. + gst_video_frame_map(&vframe, &(hailoenc->input_state->info), frame->input_buffer, GST_MAP_READ); + guint8 * luma = GST_VIDEO_FRAME_PLANE_DATA (&vframe, 0); + guint8 * chroma = GST_VIDEO_FRAME_PLANE_DATA (&vframe, 1); + // Get the Size of input buffer luma and chroma. + size_t luma_size = GST_VIDEO_FRAME_HEIGHT (&vframe) * GST_VIDEO_FRAME_PLANE_STRIDE (&vframe, 0); + size_t chroma_size = GST_VIDEO_FRAME_HEIGHT (&vframe) * GST_VIDEO_FRAME_PLANE_STRIDE (&vframe, 1) / 2; + gst_video_frame_unmap(&vframe); + + // Get the physical Addresses of input buffer luma and chroma. + ret = EWLGetBusAddress(enc_params->ewl, (u32*)luma, (u32*)&(enc_params->encIn.busLuma), luma_size); + if (ret != EWL_OK) + { + GST_ERROR_OBJECT(hailoenc, "Could not get physical address of input picture luma"); + return GST_FLOW_ERROR; + } + ret = EWLGetBusAddress(enc_params->ewl, (u32*)chroma, (u32*)&(enc_params->encIn.busChromaU), chroma_size); + if (ret != EWL_OK) + { + GST_ERROR_OBJECT(hailoenc, "Could not get physical address of input picture chroma"); + return GST_FLOW_ERROR; + } + return GST_FLOW_OK; +} + +/** + * Creats a GstBuffer object with the encoded data as memory. + * + * @param[in] hailoenc The GstHailoH264Enc object. + * @return A GstBuffer object of the encoded data. + * @note It also modifies parameters containing the total streamed bytes and bits. + */ +static GstBuffer *gst_hailoh264enc_get_encoded_buffer(GstHailoH264Enc * hailoenc) +{ + GstBuffer *outbuf; + EncoderParams *enc_params = &(hailoenc->enc_params); + outbuf = gst_buffer_new_memdup(enc_params->outbufMem.virtualAddress, + enc_params->encOut.streamSize); + return outbuf; +} + +/** + * Set the headers of the encoded stream + * + * @param[in] encoder The GstVideoEncoder object. + */ +static void +gst_hailoh264enc_set_headers(GstVideoEncoder * encoder) +{ + GstHailoH264Enc *hailoenc = (GstHailoH264Enc *) encoder; + GList *headers = NULL; + headers = g_list_prepend(headers, hailoenc->header_buffer); + gst_video_encoder_set_headers(encoder, headers); +} + +/** + * Encode and set the header - Performed via VCEncStrmStart + * + * @param[in] encoder The GstVideoEncoder object. + * @return Upon success, returns VCENC_OK. Otherwise, returns another error value from VCEncRet. + */ +static VCEncRet +gst_hailoh264enc_encode_header(GstVideoEncoder * encoder) +{ + VCEncRet enc_ret; + GstHailoH264Enc *hailoenc = (GstHailoH264Enc *) encoder; + EncoderParams *enc_params = &(hailoenc->enc_params); + VCEncIn *pEncIn = &(enc_params->encIn); + VCEncOut *pEncOut = &(enc_params->encOut); + pEncIn->gopSize = enc_params->gopSize; + + if (hailoenc->h264_encoder == NULL) + { + GST_ERROR_OBJECT(hailoenc, "Encoder not initialized"); + return VCENC_ERROR; + } + + enc_ret = VCEncStrmStart(hailoenc->h264_encoder, pEncIn, pEncOut); + if (enc_ret != VCENC_OK) + { + return enc_ret; + } + + hailoenc->header_buffer = gst_hailoh264enc_get_encoded_buffer(hailoenc); + gst_hailoh264enc_set_headers(encoder); + + // Default gop size as IPPP + pEncIn->poc = 0; + pEncIn->gopSize = enc_params->nextGopSize = ((enc_params->gopSize == 0) ? 1 : enc_params->gopSize); + pEncIn->codingType = VCENC_INTRA_FRAME; + + return enc_ret; +} + + +/** + * Restart the encoder + * + * @param[in] encoder The GstVideoEncoder object. + * @param[in] frame A GstVideoCodecFrame used for sending stream_end data. + * @return Upon success, returns GST_FLOW_OK, GST_FLOW_ERROR on failure. + */ +static GstFlowReturn +gst_hailoh264enc_stream_restart(GstVideoEncoder * encoder, GstVideoCodecFrame * frame) +{ + VCEncRet enc_ret; + GstHailoH264Enc *hailoenc = (GstHailoH264Enc *) encoder; + EncoderParams *enc_params = &(hailoenc->enc_params); + VCEncIn * pEncIn = &(enc_params->encIn); + VCEncOut * pEncOut = &(enc_params->encOut); + GST_WARNING_OBJECT(hailoenc, "Restarting encoder"); + + if (hailoenc->h264_encoder == NULL) + { + GST_ERROR_OBJECT(hailoenc, "Encoder not initialized"); + return GST_FLOW_ERROR; + } + + enc_ret = VCEncStrmEnd(hailoenc->h264_encoder, pEncIn, pEncOut); + if (enc_ret != VCENC_OK) + { + GST_ERROR_OBJECT(hailoenc, "Encoder restart - Failed to end stream, returned %d", enc_ret); + return GST_FLOW_ERROR; + } + + if (enc_params->picture_cnt > 0) + { + frame->output_buffer = gst_hailoh264enc_get_encoded_buffer(hailoenc); + gst_video_encoder_finish_subframe(GST_VIDEO_ENCODER(hailoenc), frame); + } + + if (hailoenc->hard_restart) + { + CloseEncoder(hailoenc->h264_encoder); + } + + if (hailoenc->update_gop_size) + { + GST_DEBUG_OBJECT(hailoenc, "Updating gop size to %u", enc_params->gopSize); + memset(hailoenc->gopPicCfg, 0, sizeof(hailoenc->gopPicCfg)); + memset(enc_params->gopCfgOffset, 0, sizeof(enc_params->gopCfgOffset)); + memset(&(enc_params->encIn.gopConfig), 0, sizeof(enc_params->encIn.gopConfig)); + enc_params->encIn.gopConfig.pGopPicCfg = hailoenc->gopPicCfg; + if (VCEncInitGopConfigs(enc_params->gopSize, NULL, &(enc_params->encIn.gopConfig), enc_params->gopCfgOffset, enc_params->bFrameQpDelta, enc_params->codecH264) != 0) + { + GST_ERROR_OBJECT(hailoenc, "Encoder restart - Failed to update gop size"); + return GST_FLOW_ERROR; + } + hailoenc->update_gop_size = FALSE; + } + + if (hailoenc->hard_restart) + { + GST_INFO_OBJECT(hailoenc, "Reopening encoder"); + if (OpenEncoder(&(hailoenc->h264_encoder), enc_params) != 0) + { + GST_ERROR_OBJECT(hailoenc, "Encoder restart - Failed to open encoder"); + return GST_FLOW_ERROR; + } + hailoenc->hard_restart = FALSE; + } + else + { + if (UpdateEncoderConfig(&(hailoenc->h264_encoder), enc_params) != 0) + { + GST_ERROR_OBJECT(hailoenc, "Encoder restart - Failed to update configuration"); + return GST_FLOW_ERROR; + } + } + + enc_ret = gst_hailoh264enc_encode_header(encoder); + if (enc_ret != VCENC_OK) + { + GST_ERROR_OBJECT(hailoenc, "Encoder restart - Failed to encode headers, returned %d", enc_ret); + return GST_FLOW_ERROR; + } + + hailoenc->stream_restart = FALSE; + return GST_FLOW_OK; +} + +/* + * Send a slice to the downstream element + * + * @param[in] hailoenc The GstHailoH264Enc object. + * @param[in] address The address of the slice. + * @param[in] size The size of the slice. + * @return Upon success, returns GST_FLOW_OK, GST_FLOW_ERROR on failure. + */ +static GstFlowReturn +gst_hailoh264enc_send_slice(GstHailoH264Enc * hailoenc, u32 *address, u32 size) +{ + GstBuffer *outbuf; + GstVideoCodecFrame *frame; + + /* Get oldest frame */ + frame = gst_video_encoder_get_oldest_frame (GST_VIDEO_ENCODER(hailoenc)); + outbuf = gst_buffer_new_memdup(address, size); + frame->output_buffer = outbuf; + + return gst_video_encoder_finish_subframe (GST_VIDEO_ENCODER (hailoenc), frame); +} + +/* + * Callback function for slice ready event + * + * @param[in] slice The slice ready event. + * @return void + */ +static void gst_hailoh264enc_slice_ready(VCEncSliceReady *slice) +{ + u32 i; + u32 streamSize; + u8 *strmPtr; + GstHailoH264Enc *hailoenc = (GstHailoH264Enc *) slice->pAppData; + EncoderParams *enc_params = &(hailoenc->enc_params); + /* Here is possible to implement low-latency streaming by + * sending the complete slices before the whole frame is completed. */ + + if(enc_params->multislice_encoding) + { + if (slice->slicesReadyPrev == 0) /* New frame */ + { + strmPtr = (u8 *)slice->pOutBuf; /* Pointer to beginning of frame */ + streamSize=0; + for(i=0;inalUnitInfoNum+slice->slicesReady;i++) + { + streamSize+=*(slice->sliceSizes+i); + } + gst_hailoh264enc_send_slice(hailoenc, (u32 *)strmPtr, streamSize); + } + else + { + strmPtr = (u8 *)enc_params->strmPtr; /* Here we store the slice pointer */ + streamSize=0; + for(i=(slice->nalUnitInfoNum+slice->slicesReadyPrev);islicesReady+slice->nalUnitInfoNum;i++) + { + streamSize+=*(slice->sliceSizes+i); + } + gst_hailoh264enc_send_slice(hailoenc, (u32 *)strmPtr, streamSize); + } + strmPtr+=streamSize; + /* Store the slice pointer for next callback */ + enc_params->strmPtr = strmPtr; + } +} + +/** + * Encode a single frame + * + * @param[in] hailoenc The GstHailoH264Enc object. + * @param[in] frame A GstVideoCodecFrame containing the input to encode. + * @return Upon success, returns GST_FLOW_OK, GST_FLOW_ERROR on failure. + */ +static GstFlowReturn encode_single_frame(GstHailoH264Enc *hailoenc, GstVideoCodecFrame * frame) +{ + GstFlowReturn ret = GST_FLOW_ERROR; + VCEncRet enc_ret; + EncoderParams *enc_params = &(hailoenc->enc_params); + struct timespec start_encode, end_encode; + GST_DEBUG_OBJECT(hailoenc, "Encoding frame number %u in type %u", frame->system_frame_number, enc_params->nextCodingType); + + if (hailoenc->h264_encoder == NULL) + { + GST_ERROR_OBJECT(hailoenc, "Encoder not initialized"); + return GST_FLOW_ERROR; + } + + ret = gst_hailoh264enc_update_input_buffer(hailoenc, frame); + if (ret != GST_FLOW_OK) + { + GST_ERROR_OBJECT(hailoenc, "Could not update the input buffer"); + return ret; + } + + clock_gettime(CLOCK_MONOTONIC,&start_encode); + enc_ret = EncodeFrame(enc_params, hailoenc->h264_encoder, &gst_hailoh264enc_slice_ready, hailoenc); + clock_gettime(CLOCK_MONOTONIC,&end_encode); + GST_DEBUG_OBJECT(hailoenc, "Encode took %lu milliseconds", (long)gst_hailoh264enc_difftimespec_ms(end_encode,start_encode)); + + switch (enc_ret) + { + case VCENC_FRAME_READY: + enc_params->picture_enc_cnt++; + if (enc_params->encOut.streamSize == 0) + { + enc_params->picture_cnt++; + ret = GST_FLOW_OK; + GST_WARNING_OBJECT(hailoenc, "Encoder didn't return any output for frame %d", enc_params->picture_cnt); + } + else + { + if(enc_params->multislice_encoding==0) + { + // Get the encoded output buffer. + frame->output_buffer = gst_hailoh264enc_get_encoded_buffer(hailoenc); + ret = gst_video_encoder_finish_frame(GST_VIDEO_ENCODER (hailoenc), frame); + if (ret != GST_FLOW_OK) + { + GST_ERROR_OBJECT(hailoenc, "Could not send incoded buffer, reason %d", ret); + return ret; + } + } + UpdateEncoderGOP(enc_params, hailoenc->h264_encoder); + } + break; + default: + GST_ERROR_OBJECT(hailoenc, "Encoder failed with error %d", enc_ret); + ret = GST_FLOW_ERROR; + return ret; + break; + } + return ret; +} + +/** + * Encode multiple frames - encode 1 frame or more according to the GOP config order. + * + * @param[in] encoder The GstVideoEncoder object. + * @return Upon success, returns GST_FLOW_OK, GST_FLOW_ERROR on failure. + * @note All the frames that will be encoded are queued in the GstVideoEncoder object and retreived + * via the gst_video_encoder_get_frame function. + */ +static GstFlowReturn +gst_hailoh264enc_encode_frames(GstVideoEncoder * encoder) +{ + GstHailoH264Enc *hailoenc = (GstHailoH264Enc *) encoder; + EncoderParams *enc_params = &(hailoenc->enc_params); + GstVideoCodecFrame * current_frame; + GstFlowReturn ret = GST_FLOW_ERROR; + guint gop_size = enc_params->encIn.gopSize; + GST_DEBUG_OBJECT(hailoenc, "Encoding %u frames", gop_size); + + if (hailoenc->h264_encoder == NULL) + { + GST_ERROR_OBJECT(hailoenc, "Encoder not initialized"); + return GST_FLOW_ERROR; + } + + if (gop_size <= 0) + { + GST_ERROR_OBJECT(hailoenc, "Invalid current GOP size %d", gop_size); + return GST_FLOW_ERROR; + } + + // Assuming enc_params->encIn.gopSize is not 0. + for (int i=0;ipicture_cnt); + if (!current_frame) + { + GST_ERROR_OBJECT(hailoenc, "frame %u is missing", enc_params->picture_cnt); + break; + } + ret = encode_single_frame(hailoenc, current_frame); + gst_video_codec_frame_unref(current_frame); + if (ret != GST_FLOW_OK) + { + GST_ERROR_OBJECT(hailoenc, "Encoding frame %u failed.", enc_params->picture_cnt); + break; + } + } + + if (hailoenc->update_config && enc_params->nextCodingType == VCENC_INTRA_FRAME) + { + GST_INFO_OBJECT(hailoenc, "Finished GOP, restarting encoder in order to update config"); + hailoenc->stream_restart = TRUE; + hailoenc->update_config=FALSE; + } + + return ret; +} + + +/******************************** +GstVideoEncoder Virtual Functions +********************************/ + +static gboolean +gst_hailoh264enc_set_format(GstVideoEncoder * encoder, GstVideoCodecState * state) +{ + // GstCaps *other_caps; + GstCaps *allowed_caps; + GstCaps *icaps; + GstVideoCodecState *output_format; + GstHailoH264Enc *hailoenc = (GstHailoH264Enc *) encoder; + EncoderParams *enc_params = &(hailoenc->enc_params); + + gboolean updated_caps = gst_hailoh264enc_update_params(hailoenc, &(state->info)); + if (hailoenc->h264_encoder != NULL && updated_caps) + { + GST_INFO_OBJECT(hailoenc, "Encoder parameters changed, restarting encoder"); + hailoenc->stream_restart = TRUE; + hailoenc->hard_restart = TRUE; + } + else if (hailoenc->h264_encoder == NULL) + { + /* Encoder initialization */ + if (OpenEncoder(&(hailoenc->h264_encoder), enc_params) != 0) + { + return FALSE; + } + + VCEncRet ret = gst_hailoh264enc_encode_header(encoder); + if (ret != VCENC_OK) + { + GST_ERROR_OBJECT(hailoenc, "Failed to encode headers, returned %d", ret); + return FALSE; + } + } + + /* some codecs support more than one format, first auto-choose one */ + GST_DEBUG_OBJECT (hailoenc, "picking an output format ..."); + allowed_caps = gst_pad_get_allowed_caps (GST_VIDEO_ENCODER_SRC_PAD (encoder)); + if (!allowed_caps) { + GST_DEBUG_OBJECT (hailoenc, "... but no peer, using template caps"); + /* we need to copy because get_allowed_caps returns a ref, and + * get_pad_template_caps doesn't */ + allowed_caps = + gst_pad_get_pad_template_caps (GST_VIDEO_ENCODER_SRC_PAD (encoder)); + } + GST_DEBUG_OBJECT (hailoenc, "chose caps %" GST_PTR_FORMAT, allowed_caps); + + icaps = gst_caps_fixate (allowed_caps); + + /* Store input state and set output state */ + if (hailoenc->input_state) + gst_video_codec_state_unref (hailoenc->input_state); + hailoenc->input_state = gst_video_codec_state_ref (state); + GST_DEBUG_OBJECT (hailoenc, "Setting output caps state %" GST_PTR_FORMAT, icaps); + + output_format = gst_video_encoder_set_output_state (encoder, icaps, state); + + gst_video_codec_state_unref (output_format); + + /* Store some tags */ + { + GstTagList *tags = gst_tag_list_new_empty (); + gst_video_encoder_merge_tags (encoder, tags, GST_TAG_MERGE_REPLACE); + gst_tag_list_unref (tags); + } + gint max_delayed_frames = 5; + GstClockTime latency; + latency = gst_util_uint64_scale_ceil (GST_SECOND * 1, max_delayed_frames, 25); + gst_video_encoder_set_latency (GST_VIDEO_ENCODER (encoder), latency, latency); + + return TRUE; +} + +static gboolean +gst_hailoh264enc_propose_allocation(GstVideoEncoder * encoder, GstQuery * query) +{ + GstHailoH264Enc *hailoenc = (GstHailoH264Enc *) encoder; + GST_DEBUG_OBJECT(hailoenc, "hailoh264enc propose allocation callback"); + + gst_query_add_allocation_meta (query, GST_VIDEO_META_API_TYPE, NULL); + + return GST_VIDEO_ENCODER_CLASS (parent_class)->propose_allocation (encoder, + query); +} + +static gboolean +gst_hailoh264enc_flush(GstVideoEncoder * encoder) +{ + return TRUE; +} + +static gboolean +gst_hailoh264enc_start(GstVideoEncoder * encoder) +{ + GstHailoH264Enc *hailoenc = (GstHailoH264Enc *) encoder; + EncoderParams *enc_params = &(hailoenc->enc_params); + VCEncIn * pEncIn = &(enc_params->encIn); + + if (VCEncInitGopConfigs (enc_params->gopSize, NULL, &(enc_params->encIn.gopConfig), enc_params->gopCfgOffset, enc_params->bFrameQpDelta, enc_params->codecH264) != 0) + { + return FALSE; + } + + /* Allocate input and output buffers */ + if (AllocRes(enc_params) != 0) + { + FreeRes(enc_params); + return FALSE; + } + pEncIn->timeIncrement = 0; + pEncIn->busOutBuf = enc_params->outbufMem.busAddress; + pEncIn->outBufSize = enc_params->outbufMem.size; + pEncIn->pOutBuf = enc_params->outbufMem.virtualAddress; + + gst_video_encoder_set_min_pts (encoder, GST_SECOND * 60 * 60 * 1000); + + return TRUE; +} + +static gboolean +gst_hailoh264enc_stop(GstVideoEncoder * encoder) +{ + GstHailoH264Enc *hailoenc = (GstHailoH264Enc *) encoder; + EncoderParams *enc_params = &(hailoenc->enc_params); + CloseEncoder(hailoenc->h264_encoder); + hailoenc->h264_encoder = NULL; + FreeRes(enc_params); + + return TRUE; +} + +static GstFlowReturn +gst_hailoh264enc_finish(GstVideoEncoder * encoder) +{ + GstHailoH264Enc *hailoenc = (GstHailoH264Enc *) encoder; + VCEncOut *pEncOut = &(hailoenc->enc_params.encOut); + VCEncIn *pEncIn = &(hailoenc->enc_params.encIn); + VCEncRet enc_ret; + + /* End stream */ + enc_ret = VCEncStrmEnd(hailoenc->h264_encoder, pEncIn, pEncOut); + if (enc_ret != VCENC_OK) + { + GST_ERROR_OBJECT(hailoenc, "Failed to end stream, returned %d", enc_ret); + return GST_FLOW_ERROR; + } + return gst_pad_push(encoder->srcpad, gst_hailoh264enc_get_encoded_buffer(hailoenc)); +} + +static GstFlowReturn +gst_hailoh264enc_handle_frame(GstVideoEncoder * encoder, + GstVideoCodecFrame * frame) +{ + GstHailoH264Enc *hailoenc = (GstHailoH264Enc *) encoder; + GstFlowReturn ret = GST_FLOW_ERROR; + EncoderParams *enc_params = &(hailoenc->enc_params); + GList *frames; + guint delayed_frames; + GstVideoCodecFrame * oldest_frame; + struct timespec start_handle, end_handle; + clock_gettime(CLOCK_MONOTONIC,&start_handle); + GST_DEBUG_OBJECT(hailoenc, "Received frame number %u", frame->system_frame_number); + + if (hailoenc->stream_restart) + { + ret = gst_hailoh264enc_stream_restart(encoder, frame); + if (ret != GST_FLOW_OK) + { + GST_ERROR_OBJECT(hailoenc, "Failed to restart encoder"); + return ret; + } + } + + // Update Slice Encoding parameters + enc_params->multislice_encoding=0; + enc_params->strmPtr = NULL; + + if (GST_VIDEO_CODEC_FRAME_IS_FORCE_KEYFRAME(frame)) + { + GST_DEBUG_OBJECT(hailoenc, "Forcing keyframe"); + // Adding sync point in order to delete forced keyframe evnet from the queue. + GST_VIDEO_CODEC_FRAME_SET_SYNC_POINT(frame); + ForceKeyframe(enc_params, hailoenc->h264_encoder); + oldest_frame = gst_video_encoder_get_oldest_frame(encoder); + ret = encode_single_frame(hailoenc, oldest_frame); + if (ret != GST_FLOW_OK) + { + GST_ERROR_OBJECT(hailoenc, "Failed to encode forced keyframe"); + return ret; + } + if (frame == oldest_frame) + { + gst_video_codec_frame_unref(oldest_frame); + return ret; + } + } + + switch (enc_params->nextCodingType) + { + case VCENC_INTRA_FRAME: + ret = encode_single_frame(hailoenc, frame); + break; + case VCENC_PREDICTED_FRAME: + frames = gst_video_encoder_get_frames (encoder); + delayed_frames = g_list_length (frames); + g_list_free_full (frames, (GDestroyNotify) gst_video_codec_frame_unref); + guint gop_size = enc_params->encIn.gopSize; + if (delayed_frames == gop_size) + { + ret = gst_hailoh264enc_encode_frames(encoder); + } + else if (delayed_frames < gop_size) + { + ret = GST_FLOW_OK; + } + else + { + GST_ERROR_OBJECT(hailoenc, "Skipped too many frames"); + } + break; + case VCENC_BIDIR_PREDICTED_FRAME: + GST_ERROR_OBJECT(hailoenc,"Got B frame without pending P frame"); + break; + default: + GST_ERROR_OBJECT(hailoenc,"Unknown coding type %d", (int)enc_params->nextCodingType); + break; + } + clock_gettime(CLOCK_MONOTONIC,&end_handle); + GST_DEBUG_OBJECT(hailoenc, "handle_frame took %lu milliseconds", (long)gst_hailoh264enc_difftimespec_ms(end_handle,start_handle)); + return ret; +} diff --git a/core/hailo/plugins/encoder/gsthailoh264enc.h b/core/hailo/plugins/encoder/gsthailoh264enc.h new file mode 100644 index 0000000..e1b1f5f --- /dev/null +++ b/core/hailo/plugins/encoder/gsthailoh264enc.h @@ -0,0 +1,41 @@ +#pragma once + +#include +#include +#include "hailoencoder.h" + +G_BEGIN_DECLS + +typedef struct _GstHailoH264Enc GstHailoH264Enc; +typedef struct _GstHailoH264EncClass GstHailoH264EncClass; +#define GST_TYPE_HAILO_H264_ENC (gst_hailoh264enc_get_type()) +#define GST_HAILO_H264_ENC(obj) (G_TYPE_CHECK_INSTANCE_CAST((obj), GST_TYPE_HAILO_H264_ENC, GstHailoH264Enc)) +#define GST_HAILO_H264_ENC_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST((klass), GST_TYPE_HAILO_H264_ENC, GstHailoH264EncClass)) +#define GST_IS_HAILO_H264_ENC(obj) (G_TYPE_CHECK_INSTANCE_TYPE((obj), GST_TYPE_HAILO_H264_ENC)) +#define GST_IS_HAILO_H264_ENC_CLASS(obj) (G_TYPE_CHECK_CLASS_TYPE((klass), GST_TYPE_HAILO_H264_ENC)) +#define GST_HAILO_H264_ENC_GET_CLASS(obj) (G_TYPE_INSTANCE_GET_CLASS ((obj), GST_TYPE_HAILO_H264_ENC, GstHailoH264EncClass)) + +struct _GstHailoH264Enc +{ + GstVideoEncoder parent; + GstVideoCodecState *input_state; + GstBuffer *header_buffer; + VCEncApiVersion apiVer; + VCEncBuild encBuild; + VCEncGopPicConfig gopPicCfg[MAX_GOP_PIC_CONFIG_NUM]; + EncoderParams enc_params; + VCEncInst h264_encoder; + gboolean stream_restart; + gboolean hard_restart; + gboolean update_config; + gboolean update_gop_size; +}; + +struct _GstHailoH264EncClass +{ + GstVideoEncoderClass parent_class; +}; + +G_GNUC_INTERNAL GType gst_hailoh264enc_get_type(void); + +G_END_DECLS diff --git a/core/hailo/plugins/encoder/gsthailoh265enc.c b/core/hailo/plugins/encoder/gsthailoh265enc.c new file mode 100644 index 0000000..f2d8efd --- /dev/null +++ b/core/hailo/plugins/encoder/gsthailoh265enc.c @@ -0,0 +1,1361 @@ +#include +#include +/* for stats file handling */ +#include +#include +#include +#include "gsthailoh265enc.h" + + +/******************* +Property Definitions +*******************/ +enum +{ + PROP_0, + PROP_PROFILE, + PROP_LEVEL, + PROP_INTRA_PIC_RATE, + PROP_GOP_SIZE, + PROP_GOP_LENGTH, + PROP_QPHDR, + PROP_QPMIN, + PROP_QPMAX, + PROP_INTRA_QP_DELTA, + PROP_FIXED_INTRA_QP, + PROP_BFRAME_QP_DELTA, + PROP_BITRATE, + PROP_TOL_MOVING_BITRATE, + PROP_BITRATE_VAR_RANGE_I, + PROP_BITRATE_VAR_RANGE_P, + PROP_BITRATE_VAR_RANGE_B, + PROP_PICTURE_RC, + PROP_CTB_RC, + PROP_PICTURE_SKIP, + PROP_HRD, + PROP_MONITOR_FRAMES, + PROP_ROI_AREA_1, + PROP_ROI_AREA_2, + PROP_COMPRESSOR, + PROP_BLOCK_RC_SIZE, + PROP_HRD_CPB_SIZE, + NUM_OF_PROPS, +}; + +#define GST_TYPE_HAILOH265ENC_PROFILE (gst_hailoh265enc_profile_get_type()) +static GType +gst_hailoh265enc_profile_get_type(void) +{ + static GType hailoh265enc_profile_type = 0; + static const GEnumValue hailoh265enc_profiles[] = { + {VCENC_HEVC_MAIN_PROFILE, "Main Profile", "main"}, + {VCENC_HEVC_MAIN_STILL_PICTURE_PROFILE, "Main Still Picture Profile", "main-still-picture"}, + {VCENC_HEVC_MAIN_10_PROFILE, "Main 10 Profile", "main-10"}, + {0, NULL, NULL}, + }; + if (!hailoh265enc_profile_type) + { + hailoh265enc_profile_type = + g_enum_register_static("GstHailoH265EncProfile", hailoh265enc_profiles); + } + return hailoh265enc_profile_type; +} + +#define GST_TYPE_HAILOH265ENC_LEVEL (gst_hailoh265enc_level_get_type()) +static GType +gst_hailoh265enc_level_get_type(void) +{ + static GType hailoh265enc_level_type = 0; + static const GEnumValue hailoh265enc_levels[] = { + {VCENC_HEVC_LEVEL_1, "Level 1", "level-1"}, + {VCENC_HEVC_LEVEL_2, "Level 2", "level-2"}, + {VCENC_HEVC_LEVEL_2_1, "Level 2.1", "level-2-1"}, + {VCENC_HEVC_LEVEL_3, "Level 3", "level-3"}, + {VCENC_HEVC_LEVEL_3_1, "Level 3.1", "level-3-1"}, + {VCENC_HEVC_LEVEL_4, "Level 4", "level-4"}, + {VCENC_HEVC_LEVEL_4_1, "Level 4.1", "level-4-1"}, + {VCENC_HEVC_LEVEL_5, "Level 5", "level-5"}, + {VCENC_HEVC_LEVEL_5_1, "Level 5.1", "level-5-1"}, + {0, NULL, NULL}, + }; + if (!hailoh265enc_level_type) + { + hailoh265enc_level_type = + g_enum_register_static("GstHailoH265EncLevel", hailoh265enc_levels); + } + return hailoh265enc_level_type; +} + +#define GST_TYPE_HAILOH265ENC_COMPRESSOR (gst_hailoh265enc_compressor_get_type()) +static GType +gst_hailoh265enc_compressor_get_type(void) +{ + /*Enable/Disable Embedded Compression + 0 = Disable Compression + 1 = Only Enable Luma Compression + 2 = Only Enable Chroma Compression + 3 = Enable Both Luma and Chroma Compression*/ + static GType hailoh265enc_compressor_type = 0; + static const GEnumValue hailoh265enc_compressors[] = { + {0, "Disable Compression", "disable"}, + {1, "Only Enable Luma Compression", "enable-luma"}, + {2, "Only Enable Chroma Compression", "enable-chroma"}, + {3, "Enable Both Luma and Chroma Compression", "enable-both"}, + {0, NULL, NULL}, + }; + if (!hailoh265enc_compressor_type) + { + hailoh265enc_compressor_type = + g_enum_register_static("GstHailoH265EncCompressor", hailoh265enc_compressors); + } + return hailoh265enc_compressor_type; +} + +#define GST_TYPE_HAILOH265ENC_BLOCK_RC_SIZE (gst_hailoh265enc_block_rc_size_get_type()) +static GType +gst_hailoh265enc_block_rc_size_get_type(void) +{ + static GType hailoh265enc_block_rc_size_type = 0; + static const GEnumValue hailoh265enc_block_rc_size_types[] = { + {0, "64X64", "64x64"}, + {1, "32X32", "32x32"}, + {2, "16X16", "16x16"}, + {0, NULL, NULL}, + }; + if (!hailoh265enc_block_rc_size_type) + { + hailoh265enc_block_rc_size_type = + g_enum_register_static("GstHailoH265EncBlockRcSize", hailoh265enc_block_rc_size_types); + } + return hailoh265enc_block_rc_size_type; +} + +/******************* +Function Definitions +*******************/ + +static void gst_hailoh265enc_class_init(GstHailoH265EncClass * klass); +static void gst_hailoh265enc_init(GstHailoH265Enc * hailoenc); +static void gst_hailoh265enc_set_property(GObject * object, + guint prop_id, const GValue * value, GParamSpec * pspec); +static void gst_hailoh265enc_get_property(GObject * object, + guint prop_id, GValue * value, GParamSpec * pspec); +static void gst_hailoh265enc_finalize(GObject * object); + +static gboolean gst_hailoh265enc_set_format(GstVideoEncoder * encoder, GstVideoCodecState * state); +static gboolean gst_hailoh265enc_propose_allocation(GstVideoEncoder * encoder, GstQuery * query); +static gboolean gst_hailoh265enc_flush(GstVideoEncoder * encoder); +static gboolean gst_hailoh265enc_start(GstVideoEncoder * encoder); +static gboolean gst_hailoh265enc_stop(GstVideoEncoder * encoder); +static GstFlowReturn gst_hailoh265enc_finish(GstVideoEncoder * encoder); +static GstFlowReturn gst_hailoh265enc_handle_frame(GstVideoEncoder * encoder, GstVideoCodecFrame * frame); + +/************ +Pad Templates +************/ + +static GstStaticPadTemplate sink_template = GST_STATIC_PAD_TEMPLATE("sink", + GST_PAD_SINK, + GST_PAD_ALWAYS, + GST_STATIC_CAPS("video/x-raw, " + "format=NV12, " + "width=(int)[16,MAX], " + "height=(int)[16,MAX], " + "framerate=(fraction)[0/1,MAX]")); + +static GstStaticPadTemplate src_template = GST_STATIC_PAD_TEMPLATE("src", + GST_PAD_SRC, + GST_PAD_ALWAYS, + GST_STATIC_CAPS("video/x-h265, " + "stream-format = (string) byte-stream, " + "alignment = (string) au, " + "profile = (string) { main, main-still-picture, " + "main-intra, main-10, main-10-intra }") + ); + + +/************* +Init Functions +*************/ + +GST_DEBUG_CATEGORY_STATIC(gst_hailoh265enc_debug); +#define GST_CAT_DEFAULT gst_hailoh265enc_debug +#define _do_init \ + GST_DEBUG_CATEGORY_INIT(gst_hailoh265enc_debug, "hailoh265enc", 0, "hailoh265enc element"); +#define gst_hailoh265enc_parent_class parent_class +G_DEFINE_TYPE_WITH_CODE(GstHailoH265Enc, gst_hailoh265enc, GST_TYPE_VIDEO_ENCODER, _do_init); + +static void +gst_hailoh265enc_class_init(GstHailoH265EncClass * klass) +{ + GObjectClass *gobject_class; + GstVideoEncoderClass *venc_class; + GstElementClass *element_class = GST_ELEMENT_CLASS(klass); + + gobject_class = (GObjectClass *) klass; + venc_class = (GstVideoEncoderClass *) klass; + + //parent_class = g_type_class_peek_parent (klass); + gst_element_class_add_pad_template (element_class, gst_static_pad_template_get(&sink_template)); + gst_element_class_add_pad_template (element_class, gst_static_pad_template_get(&src_template)); + gst_element_class_set_static_metadata(element_class, + "H265 Encoder", + "Encoder/Video", + "Encodes raw video into H265 format", + "hailo.ai "); + + gobject_class->set_property = gst_hailoh265enc_set_property; + gobject_class->get_property = gst_hailoh265enc_get_property; + + + g_object_class_install_property(gobject_class, PROP_PROFILE, + g_param_spec_enum("profile", "encoder profile", "profile to encoder", + GST_TYPE_HAILOH265ENC_PROFILE, (gint)DEFAULT_HEVC_PROFILE, + (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); + g_object_class_install_property(gobject_class, PROP_LEVEL, + g_param_spec_enum("level", "encoder level", "level to encoder", + GST_TYPE_HAILOH265ENC_LEVEL, (gint)DEFAULT_HEVC_LEVEL, + (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); + g_object_class_install_property(gobject_class, PROP_INTRA_PIC_RATE, + g_param_spec_uint("intra-pic-rate", "IDR Interval", "I frames interval (0 - Dynamic IDR Interval)", + MIN_INTRA_PIC_RATE, MAX_INTRA_PIC_RATE, (guint)DEFAULT_INTRA_PIC_RATE, + (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS | GST_PARAM_MUTABLE_PLAYING))); + g_object_class_install_property(gobject_class, PROP_GOP_SIZE, + g_param_spec_uint("gop-size", "GOP Size", "GOP Size (1 - No B Frames)", + MIN_GOP_SIZE, MAX_GOP_SIZE, (guint)DEFAULT_GOP_SIZE, + (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS | GST_PARAM_MUTABLE_PLAYING))); + g_object_class_install_property(gobject_class, PROP_GOP_LENGTH, + g_param_spec_uint("gop-length", "GOP Length", "GOP Length", + MIN_GOP_LENGTH, MAX_GOP_LENGTH, (guint)DEFAULT_GOP_LENGTH, + (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS | GST_PARAM_MUTABLE_PLAYING))); + g_object_class_install_property(gobject_class, PROP_QPHDR, + g_param_spec_int("qp-hdr", "Initial target QP", "Initial target QP, -1 = Encoder calculates initial QP", + MIN_QPHDR, MAX_QPHDR, (gint)DEFAULT_QPHDR, + (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS | GST_PARAM_MUTABLE_PLAYING))); + g_object_class_install_property(gobject_class, PROP_QPMIN, + g_param_spec_uint("qp-min", "QP Min", "Minimum frame header QP", + MIN_QP_VALUE, MAX_QP_VALUE, (guint)DEFAULT_QPMIN, + (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS | GST_PARAM_MUTABLE_PLAYING))); + g_object_class_install_property(gobject_class, PROP_QPMAX, + g_param_spec_uint("qp-max", "QP Max", "Maximum frame header QP", + MIN_QP_VALUE, MAX_QP_VALUE, (guint)DEFAULT_QPMAX, + (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS | GST_PARAM_MUTABLE_PLAYING))); + g_object_class_install_property(gobject_class, PROP_INTRA_QP_DELTA, + g_param_spec_int("intra-qp-delta", "Intra QP delta", "QP difference between target QP and intra frame QP", + MIN_INTRA_QP_DELTA, MAX_INTRA_QP_DELTA, (gint)DEFAULT_INTRA_QP_DELTA, + (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS | GST_PARAM_MUTABLE_PLAYING))); + g_object_class_install_property(gobject_class, PROP_FIXED_INTRA_QP, + g_param_spec_uint("fixed-intra-qp", "Fixed Intra QP", "Use fixed QP value for every intra frame in stream, 0 = disabled", + MIN_FIXED_INTRA_QP, MAX_FIXED_INTRA_QP, (guint)DEFAULT_FIXED_INTRA_QP, + (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS | GST_PARAM_MUTABLE_PLAYING))); + g_object_class_install_property(gobject_class, PROP_BFRAME_QP_DELTA, + g_param_spec_int("bframe-qp-delta", "BFrame QP Delta", "QP difference between BFrame QP and target QP, -1 = Disabled", + MIN_BFRAME_QP_DELTA, MAX_BFRAME_QP_DELTA, (guint)DEFAULT_BFRAME_QP_DELTA, + (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS | GST_PARAM_MUTABLE_PLAYING))); + g_object_class_install_property(gobject_class, PROP_BITRATE, + g_param_spec_uint("bitrate", "Target bitrate", "Target bitrate for rate control in bits/second", + MIN_BITRATE, MAX_BITRATE, (guint)DEFAULT_BITRATE, + (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS | GST_PARAM_MUTABLE_PLAYING))); + g_object_class_install_property(gobject_class, PROP_TOL_MOVING_BITRATE, + g_param_spec_uint("tol-moving-bitrate", "Tolerance moving bitrate", "Percent tolerance over target bitrate of moving bit rate", + MIN_BITRATE_VARIABLE_RANGE, MAX_BITRATE_VARIABLE_RANGE, (guint)DEFAULT_TOL_MOVING_BITRATE, + (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS | GST_PARAM_MUTABLE_PLAYING))); + g_object_class_install_property(gobject_class, PROP_BITRATE_VAR_RANGE_I, + g_param_spec_uint("bitvar-range-i", "Bitrate percent variation I frame", "Percent variations over average bits per frame for I frame", + MIN_BITRATE_VARIABLE_RANGE, MAX_BITRATE_VARIABLE_RANGE, (guint)DEFAULT_BITVAR_RANGE_I, + (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS | GST_PARAM_MUTABLE_PLAYING))); + g_object_class_install_property(gobject_class, PROP_BITRATE_VAR_RANGE_P, + g_param_spec_uint("bitvar-range-p", "Bitrate percent variation P frame", "Percent variations over average bits per frame for P frame", + MIN_BITRATE_VARIABLE_RANGE, MAX_BITRATE_VARIABLE_RANGE, (guint)DEFAULT_BITVAR_RANGE_P, + (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS | GST_PARAM_MUTABLE_PLAYING))); + g_object_class_install_property(gobject_class, PROP_BITRATE_VAR_RANGE_B, + g_param_spec_uint("bitvar-range-b", "Bitrate percent variation B frame", "Percent variations over average bits per frame for B frame", + MIN_BITRATE_VARIABLE_RANGE, MAX_BITRATE_VARIABLE_RANGE, (guint)DEFAULT_BITVAR_RANGE_B, + (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS | GST_PARAM_MUTABLE_PLAYING))); + g_object_class_install_property(gobject_class, PROP_PICTURE_RC, + g_param_spec_boolean("picture-rc", "Picture Rate Control", "Adjust QP between pictures", true, + (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS | GST_PARAM_MUTABLE_PLAYING))); + g_object_class_install_property(gobject_class, PROP_CTB_RC, + g_param_spec_boolean("ctb-rc", "Block Rate Control", "Adaptive adjustment of QP inside frame", false, + (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS | GST_PARAM_MUTABLE_PLAYING))); + g_object_class_install_property(gobject_class, PROP_PICTURE_SKIP, + g_param_spec_boolean("picture-skip", "Picture Skip", "Allow rate control to skip pictures", false, + (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS | GST_PARAM_MUTABLE_PLAYING))); + g_object_class_install_property(gobject_class, PROP_HRD, + g_param_spec_boolean("hrd", "Picture Rate Control", "Restricts the instantaneous bitrate and total bit amount of every coded picture.", false, + (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS | GST_PARAM_MUTABLE_PLAYING))); + g_object_class_install_property(gobject_class, PROP_MONITOR_FRAMES, + g_param_spec_uint("monitor-frames", "Monitor Frames", "How many frames will be monitored for moving bit rate. Default is using framerate", + MIN_MONITOR_FRAMES, MAX_MONITOR_FRAMES, (gint)DEFAULT_MONITOR_FRAMES, + (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS | GST_PARAM_MUTABLE_PLAYING))); + g_object_class_install_property(gobject_class, PROP_ROI_AREA_1, + g_param_spec_string("roi-area1", "ROI Area and QP Delta", + "Specifying rectangular area of CTBs as Region Of Interest with lower QP, left:top:right:bottom:delta_qp format ", NULL, + (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS | GST_PARAM_MUTABLE_PLAYING))); + g_object_class_install_property(gobject_class, PROP_ROI_AREA_2, + g_param_spec_string("roi-area2", "ROI Area and QP Delta", + "Specifying rectangular area of CTBs as Region Of Interest with lower QP, left:top:right:bottom:delta_qp format ", NULL, + (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS | GST_PARAM_MUTABLE_PLAYING))); + g_object_class_install_property(gobject_class, PROP_COMPRESSOR, + g_param_spec_enum("compressor", "Compressor", "Enable/Disable Embedded Compression", + GST_TYPE_HAILOH265ENC_COMPRESSOR, (guint)3, + (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); + g_object_class_install_property(gobject_class, PROP_BLOCK_RC_SIZE, + g_param_spec_enum("block-rc-size", "Block Rate Control Size", "Size of block rate control", + GST_TYPE_HAILOH265ENC_BLOCK_RC_SIZE, (guint)0, + (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS | GST_PARAM_MUTABLE_PLAYING))); + g_object_class_install_property(gobject_class, PROP_HRD_CPB_SIZE, + g_param_spec_uint("hrd-cpb-size", "HRD Coded Picture Buffer size", "Buffer size used by the HRD model in bits", + MIN_HRD_CPB_SIZE, MAX_HRD_CPB_SIZE, (guint)DEFAULT_HRD_CPB_SIZE, + (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS | GST_PARAM_MUTABLE_PLAYING))); + + venc_class->start = gst_hailoh265enc_start; + venc_class->stop = gst_hailoh265enc_stop; + venc_class->finish = gst_hailoh265enc_finish; + venc_class->handle_frame = gst_hailoh265enc_handle_frame; + venc_class->set_format = gst_hailoh265enc_set_format; + venc_class->propose_allocation = gst_hailoh265enc_propose_allocation; + venc_class->flush = gst_hailoh265enc_flush; + + gobject_class->finalize = gst_hailoh265enc_finalize; + +} + +static void +gst_hailoh265enc_init(GstHailoH265Enc * hailoenc) +{ + //GstHailoH265EncClass *klass = + // (GstHailoH265EncClass *) G_OBJECT_GET_CLASS (hailoenc); + EncoderParams *enc_params = &(hailoenc->enc_params); + GST_PAD_SET_ACCEPT_TEMPLATE (GST_VIDEO_ENCODER_SINK_PAD (hailoenc)); + hailoenc->apiVer = VCEncGetApiVersion(); + hailoenc->encBuild = VCEncGetBuild(); + hailoenc->stream_restart = FALSE; + memset(enc_params, 0, sizeof(EncoderParams)); + SetDefaultParameters(enc_params, false); + if(HW_ID_MAJOR_NUMBER(hailoenc->encBuild.hwBuild)<= 1) + { + //H2V1 only support 1. + enc_params->gopSize = 1; + } + memset(hailoenc->gopPicCfg, 0, sizeof(hailoenc->gopPicCfg)); + hailoenc->hevc_encoder = NULL; + enc_params->encIn.gopConfig.pGopPicCfg = hailoenc->gopPicCfg; +} + +/************************ +GObject Virtual Functions +************************/ + +static void +gst_hailoh265enc_get_property(GObject * object, + guint prop_id, GValue * value, GParamSpec * pspec) +{ + GstHailoH265Enc *hailoenc = (GstHailoH265Enc *) (object); + + switch (prop_id) { + case PROP_PROFILE: + GST_OBJECT_LOCK(hailoenc); + g_value_set_enum(value, (gint)hailoenc->enc_params.profile); + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_LEVEL: + GST_OBJECT_LOCK(hailoenc); + g_value_set_enum(value, (gint)hailoenc->enc_params.level); + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_INTRA_PIC_RATE: + GST_OBJECT_LOCK(hailoenc); + g_value_set_uint(value, (guint)hailoenc->enc_params.intraPicRate); + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_GOP_SIZE: + GST_OBJECT_LOCK(hailoenc); + g_value_set_uint(value, (guint)hailoenc->enc_params.gopSize); + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_GOP_LENGTH: + GST_OBJECT_LOCK(hailoenc); + g_value_set_uint(value, (guint)hailoenc->enc_params.gopLength); + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_QPHDR: + GST_OBJECT_LOCK(hailoenc); + g_value_set_int(value, (gint)hailoenc->enc_params.qphdr); + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_QPMIN: + GST_OBJECT_LOCK(hailoenc); + g_value_set_uint(value, (guint)hailoenc->enc_params.qpmin); + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_QPMAX: + GST_OBJECT_LOCK(hailoenc); + g_value_set_uint(value, (guint)hailoenc->enc_params.qpmax); + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_INTRA_QP_DELTA: + GST_OBJECT_LOCK(hailoenc); + g_value_set_int(value, (gint)hailoenc->enc_params.intra_qp_delta); + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_FIXED_INTRA_QP: + GST_OBJECT_LOCK(hailoenc); + g_value_set_uint(value, (guint)hailoenc->enc_params.fixed_intra_qp); + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_BFRAME_QP_DELTA: + GST_OBJECT_LOCK(hailoenc); + g_value_set_int(value, (gint)hailoenc->enc_params.bFrameQpDelta); + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_BITRATE: + GST_OBJECT_LOCK(hailoenc); + g_value_set_uint(value, (guint)hailoenc->enc_params.bitrate); + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_TOL_MOVING_BITRATE: + GST_OBJECT_LOCK(hailoenc); + g_value_set_uint(value, (guint)hailoenc->enc_params.tolMovingBitRate); + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_BITRATE_VAR_RANGE_I: + GST_OBJECT_LOCK(hailoenc); + g_value_set_uint(value, (guint)hailoenc->enc_params.bitVarRangeI); + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_BITRATE_VAR_RANGE_P: + GST_OBJECT_LOCK(hailoenc); + g_value_set_uint(value, (guint)hailoenc->enc_params.bitVarRangeP); + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_BITRATE_VAR_RANGE_B: + GST_OBJECT_LOCK(hailoenc); + g_value_set_uint(value, (guint)hailoenc->enc_params.bitVarRangeB); + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_MONITOR_FRAMES: + GST_OBJECT_LOCK(hailoenc); + g_value_set_uint(value, (guint)hailoenc->enc_params.monitorFrames); + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_PICTURE_RC: + GST_OBJECT_LOCK(hailoenc); + gboolean picture_rc = hailoenc->enc_params.pictureRc == 1 ? TRUE : FALSE; + g_value_set_boolean(value, picture_rc); + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_CTB_RC: + GST_OBJECT_LOCK(hailoenc); + gboolean ctb_rc = hailoenc->enc_params.ctbRc == 1 ? TRUE : FALSE; + g_value_set_boolean(value, ctb_rc); + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_PICTURE_SKIP: + GST_OBJECT_LOCK(hailoenc); + gboolean picture_skip = hailoenc->enc_params.pictureSkip == 1 ? TRUE : FALSE; + g_value_set_boolean(value, picture_skip); + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_HRD: + GST_OBJECT_LOCK(hailoenc); + gboolean hrd = hailoenc->enc_params.hrd == 1 ? TRUE : FALSE; + g_value_set_boolean(value, hrd); + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_ROI_AREA_1: + GST_OBJECT_LOCK(hailoenc); + g_value_set_string(value, (const gchar *)hailoenc->enc_params.roiArea1); + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_ROI_AREA_2: + GST_OBJECT_LOCK(hailoenc); + g_value_set_string(value, (const gchar *)hailoenc->enc_params.roiArea2); + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_COMPRESSOR: + GST_OBJECT_LOCK(hailoenc); + g_value_set_enum(value, (gint)hailoenc->enc_params.compressor); + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_BLOCK_RC_SIZE: + GST_OBJECT_LOCK(hailoenc); + g_value_set_enum(value, (gint)hailoenc->enc_params.blockRcSize); + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_HRD_CPB_SIZE: + GST_OBJECT_LOCK(hailoenc); + g_value_set_uint(value, (guint)hailoenc->enc_params.hrdCpbSize); + GST_OBJECT_UNLOCK(hailoenc); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +gst_hailoh265enc_set_property(GObject * object, + guint prop_id, const GValue * value, GParamSpec * pspec) +{ + GstHailoH265Enc *hailoenc = (GstHailoH265Enc *) (object); + hailoenc->update_config=FALSE; + + switch (prop_id) { + case PROP_PROFILE: + GST_OBJECT_LOCK(hailoenc); + hailoenc->enc_params.profile = (VCEncProfile)g_value_get_enum(value); + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_LEVEL: + GST_OBJECT_LOCK(hailoenc); + hailoenc->enc_params.level = (VCEncLevel)g_value_get_enum(value); + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_INTRA_PIC_RATE: + GST_OBJECT_LOCK(hailoenc); + hailoenc->enc_params.intraPicRate = g_value_get_uint(value); + hailoenc->update_config = TRUE; + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_GOP_SIZE: + GST_OBJECT_LOCK(hailoenc); + hailoenc->enc_params.gopSize = g_value_get_uint(value); + hailoenc->update_config = TRUE; + hailoenc->update_gop_size = TRUE; + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_GOP_LENGTH: + GST_OBJECT_LOCK(hailoenc); + hailoenc->enc_params.gopLength = g_value_get_uint(value); + hailoenc->update_config = TRUE; + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_QPHDR: + GST_OBJECT_LOCK(hailoenc); + hailoenc->enc_params.qphdr = g_value_get_int(value); + hailoenc->update_config = TRUE; + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_QPMIN: + GST_OBJECT_LOCK(hailoenc); + hailoenc->enc_params.qpmin = g_value_get_uint(value); + hailoenc->update_config = TRUE; + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_QPMAX: + GST_OBJECT_LOCK(hailoenc); + hailoenc->enc_params.qpmax = g_value_get_uint(value); + hailoenc->update_config = TRUE; + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_INTRA_QP_DELTA: + GST_OBJECT_LOCK(hailoenc); + hailoenc->enc_params.intra_qp_delta = g_value_get_int(value); + hailoenc->update_config = TRUE; + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_FIXED_INTRA_QP: + GST_OBJECT_LOCK(hailoenc); + hailoenc->enc_params.fixed_intra_qp = g_value_get_uint(value); + hailoenc->update_config = TRUE; + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_BFRAME_QP_DELTA: + GST_OBJECT_LOCK(hailoenc); + hailoenc->enc_params.bFrameQpDelta = g_value_get_int(value); + hailoenc->update_config = TRUE; + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_BITRATE: + GST_OBJECT_LOCK(hailoenc); + hailoenc->enc_params.bitrate = g_value_get_uint(value); + hailoenc->update_config = TRUE; + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_TOL_MOVING_BITRATE: + GST_OBJECT_LOCK(hailoenc); + hailoenc->enc_params.tolMovingBitRate = g_value_get_uint(value); + hailoenc->update_config = TRUE; + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_BITRATE_VAR_RANGE_I: + GST_OBJECT_LOCK(hailoenc); + hailoenc->enc_params.bitVarRangeI = g_value_get_uint(value); + hailoenc->update_config = TRUE; + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_BITRATE_VAR_RANGE_P: + GST_OBJECT_LOCK(hailoenc); + hailoenc->enc_params.bitVarRangeP = g_value_get_uint(value); + hailoenc->update_config = TRUE; + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_BITRATE_VAR_RANGE_B: + GST_OBJECT_LOCK(hailoenc); + hailoenc->enc_params.bitVarRangeB = g_value_get_uint(value); + hailoenc->update_config = TRUE; + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_MONITOR_FRAMES: + GST_OBJECT_LOCK(hailoenc); + hailoenc->enc_params.monitorFrames = g_value_get_uint(value); + hailoenc->update_config = TRUE; + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_PICTURE_RC: + GST_OBJECT_LOCK(hailoenc); + hailoenc->enc_params.pictureRc = g_value_get_boolean(value) ? 1 : 0; + hailoenc->update_config = TRUE; + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_CTB_RC: + GST_OBJECT_LOCK(hailoenc); + hailoenc->enc_params.ctbRc = g_value_get_boolean(value) ? 1 : 0; + hailoenc->update_config = TRUE; + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_PICTURE_SKIP: + GST_OBJECT_LOCK(hailoenc); + hailoenc->enc_params.pictureSkip = g_value_get_boolean(value) ? 1 : 0; + hailoenc->update_config = TRUE; + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_HRD: + GST_OBJECT_LOCK(hailoenc); + hailoenc->enc_params.hrd = g_value_get_boolean(value) ? 1 : 0; + hailoenc->update_config = TRUE; + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_ROI_AREA_1: + GST_OBJECT_LOCK(hailoenc); + hailoenc->enc_params.roiArea1 = (char *)g_value_get_string(value); + hailoenc->update_config = TRUE; + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_ROI_AREA_2: + GST_OBJECT_LOCK(hailoenc); + hailoenc->enc_params.roiArea2 = (char *)g_value_get_string(value); + hailoenc->update_config = TRUE; + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_COMPRESSOR: + GST_OBJECT_LOCK(hailoenc); + hailoenc->enc_params.compressor = (u32)g_value_get_enum(value); + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_BLOCK_RC_SIZE: + GST_OBJECT_LOCK(hailoenc); + hailoenc->enc_params.blockRcSize = (u32)g_value_get_enum(value); + hailoenc->update_config = TRUE; + GST_OBJECT_UNLOCK(hailoenc); + break; + case PROP_HRD_CPB_SIZE: + GST_OBJECT_LOCK(hailoenc); + hailoenc->enc_params.hrdCpbSize = g_value_get_uint(value); + hailoenc->update_config = TRUE; + GST_OBJECT_UNLOCK(hailoenc); + break; + default: + G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec); + break; + } +} + +static void +gst_hailoh265enc_finalize(GObject * object) +{ + GstHailoH265Enc *hailoenc = (GstHailoH265Enc *) object; + GST_DEBUG_OBJECT(hailoenc, "hailoh265enc finalize callback"); + + /* clean up remaining allocated data */ + + G_OBJECT_CLASS (parent_class)->finalize (object); +} + +/***************** +Internal Functions +*****************/ + + +/** + * Gets the time difference between 2 time specs in milliseconds. + * @param[in] after The second time spec. + * @param[in] before The first time spec.\ + * @returns The time differnece in milliseconds. + */ +int64_t gst_hailoh265enc_difftimespec_ms(const struct timespec after, const struct timespec before) +{ + return ((int64_t)after.tv_sec - (int64_t)before.tv_sec) * (int64_t)1000 + + ((int64_t)after.tv_nsec - (int64_t)before.tv_nsec) / 1000000; +} + +/** + * Updates the encoder with the input video info. + * + * @param[in] hailoenc The GstHailoH265Enc object. + * @param[in] info A GstVideoInfo object containing the input video info for this pipeline. + * @returns TRUE if the encoder parameters were updated, FALSE otherwise. + * @note The updated data is the resolution, framerate and input format. + */ +static gboolean +gst_hailoh265enc_update_params(GstHailoH265Enc *hailoenc, GstVideoInfo * info) +{ + gboolean updated_params = FALSE; + EncoderParams *enc_params = &(hailoenc->enc_params); + + if (enc_params->width != GST_VIDEO_INFO_WIDTH (info) || + enc_params->height != GST_VIDEO_INFO_HEIGHT (info)) + { + enc_params->width = GST_VIDEO_INFO_WIDTH (info); + enc_params->height = GST_VIDEO_INFO_HEIGHT (info); + updated_params = TRUE; + } + + if (enc_params->frameRateNumer != GST_VIDEO_INFO_FPS_N (info) || + enc_params->frameRateDenom != GST_VIDEO_INFO_FPS_D (info)) + { + enc_params->frameRateNumer = GST_VIDEO_INFO_FPS_N (info); + enc_params->frameRateDenom = GST_VIDEO_INFO_FPS_D (info); + updated_params = TRUE; + } + + switch (GST_VIDEO_INFO_FORMAT(info)) + { + case GST_VIDEO_FORMAT_NV12: + enc_params->inputFormat = VCENC_YUV420_SEMIPLANAR; + break; + case GST_VIDEO_FORMAT_NV21: + enc_params->inputFormat = VCENC_YUV420_SEMIPLANAR_VU; + break; + case GST_VIDEO_FORMAT_I420: + enc_params->inputFormat = VCENC_YUV420_PLANAR; + break; + default: + GST_ERROR_OBJECT(hailoenc, "Unsupported format %d", GST_VIDEO_INFO_FORMAT(info)); + break; + } + return updated_params; +} + +/** + * Updated the encoder parameters with the physical addresses of the current input buffer. + * + * @param[in] hailoenc The GstHailoH265Enc object. + * @param[in] frame The GstVideoCodecFrame object with the input GstBuffer inside. + * @return GST_FLOW_OK on success, GST_FLOW_ERROR on failure. + * @note The function will fail when it cannot get the physical address or the memory is non-continous. + */ +static GstFlowReturn gst_hailoh265enc_update_input_buffer(GstHailoH265Enc * hailoenc, + GstVideoCodecFrame * frame) +{ + EncoderParams *enc_params = &(hailoenc->enc_params); + GstVideoFrame vframe; + int ret; + + // Get the Virtal addresses of input buffer luma and chroma. + gst_video_frame_map(&vframe, &(hailoenc->input_state->info), frame->input_buffer, GST_MAP_READ); + guint8 * luma = GST_VIDEO_FRAME_PLANE_DATA (&vframe, 0); + guint8 * chroma = GST_VIDEO_FRAME_PLANE_DATA (&vframe, 1); + // Get the Size of input buffer luma and chroma. + size_t luma_size = GST_VIDEO_FRAME_HEIGHT (&vframe) * GST_VIDEO_FRAME_PLANE_STRIDE (&vframe, 0); + size_t chroma_size = GST_VIDEO_FRAME_HEIGHT (&vframe) * GST_VIDEO_FRAME_PLANE_STRIDE (&vframe, 1) / 2; + gst_video_frame_unmap(&vframe); + + // Get the physical Addresses of input buffer luma and chroma. + ret = EWLGetBusAddress(enc_params->ewl, (u32*)luma, (u32*)&(enc_params->encIn.busLuma), luma_size); + if (ret != EWL_OK) + { + GST_ERROR_OBJECT(hailoenc, "Could not get physical address of input picture luma"); + return GST_FLOW_ERROR; + } + ret = EWLGetBusAddress(enc_params->ewl, (u32*)chroma, (u32*)&(enc_params->encIn.busChromaU), chroma_size); + if (ret != EWL_OK) + { + GST_ERROR_OBJECT(hailoenc, "Could not get physical address of input picture chroma"); + return GST_FLOW_ERROR; + } + return GST_FLOW_OK; +} + +/** + * Creats a GstBuffer object with the encoded data as memory. + * + * @param[in] hailoenc The GstHailoH265Enc object. + * @return A GstBuffer object of the encoded data. + * @note It also modifies parameters containing the total streamed bytes and bits. + */ +static GstBuffer *gst_hailoh265enc_get_encoded_buffer(GstHailoH265Enc * hailoenc) +{ + GstBuffer *outbuf; + EncoderParams *enc_params = &(hailoenc->enc_params); + outbuf = gst_buffer_new_memdup(enc_params->outbufMem.virtualAddress, + enc_params->encOut.streamSize); + return outbuf; +} + +/** + * Set the headers of the encoded stream + * + * @param[in] encoder The GstVideoEncoder object. + */ +static void +gst_hailoh265enc_set_headers(GstVideoEncoder * encoder) +{ + GstHailoH265Enc *hailoenc = (GstHailoH265Enc *) encoder; + GList *headers = NULL; + headers = g_list_prepend(headers, hailoenc->header_buffer); + gst_video_encoder_set_headers(encoder, headers); +} + +/** + * Encode and set the header - Performed via VCEncStrmStart + * + * @param[in] encoder The GstVideoEncoder object. + * @return Upon success, returns VCENC_OK. Otherwise, returns another error value from VCEncRet. + */ +static VCEncRet +gst_hailoh265enc_encode_header(GstVideoEncoder * encoder) +{ + VCEncRet enc_ret; + GstHailoH265Enc *hailoenc = (GstHailoH265Enc *) encoder; + EncoderParams *enc_params = &(hailoenc->enc_params); + VCEncIn *pEncIn = &(enc_params->encIn); + VCEncOut *pEncOut = &(enc_params->encOut); + pEncIn->gopSize = enc_params->gopSize; + + if (hailoenc->hevc_encoder == NULL) + { + GST_ERROR_OBJECT(hailoenc, "Encoder not initialized"); + return VCENC_ERROR; + } + enc_ret = VCEncStrmStart(hailoenc->hevc_encoder, pEncIn, pEncOut); + if (enc_ret != VCENC_OK) + { + return enc_ret; + } + hailoenc->header_buffer = gst_hailoh265enc_get_encoded_buffer(hailoenc); + gst_hailoh265enc_set_headers(encoder); + + // Default gop size as IPPP + pEncIn->poc = 0; + pEncIn->gopSize = enc_params->nextGopSize = ((enc_params->gopSize == 0) ? 1 : enc_params->gopSize); + pEncIn->codingType = VCENC_INTRA_FRAME; + + return enc_ret; +} + + +/** + * Restart the encoder + * + * @param[in] encoder The GstVideoEncoder object. + * @param[in] frame A GstVideoCodecFrame used for sending stream_end data. + * @return Upon success, returns GST_FLOW_OK, GST_FLOW_ERROR on failure. + */ +static GstFlowReturn +gst_hailoh265enc_stream_restart(GstVideoEncoder * encoder, GstVideoCodecFrame * frame) +{ + VCEncRet enc_ret; + GstHailoH265Enc *hailoenc = (GstHailoH265Enc *) encoder; + EncoderParams *enc_params = &(hailoenc->enc_params); + VCEncIn * pEncIn = &(enc_params->encIn); + VCEncOut * pEncOut = &(enc_params->encOut); + GST_WARNING_OBJECT(hailoenc, "Restarting encoder"); + + if (hailoenc->hevc_encoder == NULL) + { + GST_ERROR_OBJECT(hailoenc, "Encoder not initialized"); + return GST_FLOW_ERROR; + } + + enc_ret = VCEncStrmEnd(hailoenc->hevc_encoder, pEncIn, pEncOut); + if (enc_ret != VCENC_OK) + { + GST_ERROR_OBJECT(hailoenc, "Encoder restart - Failed to end stream, returned %d", enc_ret); + return GST_FLOW_ERROR; + } + + if (enc_params->picture_cnt > 0) + { + frame->output_buffer = gst_hailoh265enc_get_encoded_buffer(hailoenc); + gst_video_encoder_finish_subframe(GST_VIDEO_ENCODER(hailoenc), frame); + } + + if (hailoenc->hard_restart) + { + CloseEncoder(hailoenc->hevc_encoder); + } + + if (hailoenc->update_gop_size) + { + GST_DEBUG_OBJECT(hailoenc, "Updating gop size to %u", enc_params->gopSize); + memset(hailoenc->gopPicCfg, 0, sizeof(hailoenc->gopPicCfg)); + memset(enc_params->gopCfgOffset, 0, sizeof(enc_params->gopCfgOffset)); + memset(&(enc_params->encIn.gopConfig), 0, sizeof(enc_params->encIn.gopConfig)); + enc_params->encIn.gopConfig.pGopPicCfg = hailoenc->gopPicCfg; + if (VCEncInitGopConfigs(enc_params->gopSize, NULL, &(enc_params->encIn.gopConfig), enc_params->gopCfgOffset, enc_params->bFrameQpDelta, enc_params->codecH264) != 0) + { + GST_ERROR_OBJECT(hailoenc, "Encoder restart - Failed to update gop size"); + return GST_FLOW_ERROR; + } + hailoenc->update_gop_size = FALSE; + } + + if (hailoenc->hard_restart) + { + GST_INFO_OBJECT(hailoenc, "Reopening encoder"); + if (OpenEncoder(&(hailoenc->hevc_encoder), enc_params) != 0) + { + GST_ERROR_OBJECT(hailoenc, "Encoder restart - Failed to reopen encoder"); + return GST_FLOW_ERROR; + } + hailoenc->hard_restart = FALSE; + } + else + { + if (UpdateEncoderConfig(&(hailoenc->hevc_encoder), enc_params) != 0) + { + GST_ERROR_OBJECT(hailoenc, "Encoder restart - Failed to update configuration"); + return GST_FLOW_ERROR; + } + } + + enc_ret = gst_hailoh265enc_encode_header(encoder); + if (enc_ret != VCENC_OK) + { + GST_ERROR_OBJECT(hailoenc, "Encoder restart - Failed to encode headers, returned %d", enc_ret); + return GST_FLOW_ERROR; + } + + hailoenc->stream_restart = FALSE; + return GST_FLOW_OK; +} + +/* + * Send a slice to the downstream element + * + * @param[in] hailoenc The GstHailoH265Enc object. + * @param[in] address The address of the slice. + * @param[in] size The size of the slice. + * @return Upon success, returns GST_FLOW_OK, GST_FLOW_ERROR on failure. + */ +static GstFlowReturn +gst_hailoh265enc_send_slice(GstHailoH265Enc * hailoenc, u32 *address, u32 size) +{ + GstBuffer *outbuf; + GstVideoCodecFrame *frame; + + /* Get oldest frame */ + frame = gst_video_encoder_get_oldest_frame (GST_VIDEO_ENCODER(hailoenc)); + outbuf = gst_buffer_new_memdup(address, size); + frame->output_buffer = outbuf; + + return gst_video_encoder_finish_subframe (GST_VIDEO_ENCODER (hailoenc), frame); +} + +/* + * Callback function for slice ready event + * + * @param[in] slice The slice ready event. + * @return void + */ +static void gst_hailoh265enc_slice_ready(VCEncSliceReady *slice) +{ + u32 i; + u32 streamSize; + u8 *strmPtr; + GstHailoH265Enc *hailoenc = (GstHailoH265Enc *) slice->pAppData; + EncoderParams *enc_params = &(hailoenc->enc_params); + /* Here is possible to implement low-latency streaming by + * sending the complete slices before the whole frame is completed. */ + + if(enc_params->multislice_encoding) + { + if (slice->slicesReadyPrev == 0) /* New frame */ + { + strmPtr = (u8 *)slice->pOutBuf; /* Pointer to beginning of frame */ + streamSize=0; + for(i=0;inalUnitInfoNum+slice->slicesReady;i++) + { + streamSize+=*(slice->sliceSizes+i); + } + gst_hailoh265enc_send_slice(hailoenc, (u32 *)strmPtr, streamSize); + } + else + { + strmPtr = (u8 *)enc_params->strmPtr; /* Here we store the slice pointer */ + streamSize=0; + for(i=(slice->nalUnitInfoNum+slice->slicesReadyPrev);islicesReady+slice->nalUnitInfoNum;i++) + { + streamSize+=*(slice->sliceSizes+i); + } + gst_hailoh265enc_send_slice(hailoenc, (u32 *)strmPtr, streamSize); + } + strmPtr+=streamSize; + /* Store the slice pointer for next callback */ + enc_params->strmPtr = strmPtr; + } +} + +/** + * Encode a single frame + * + * @param[in] hailoenc The GstHailoH265Enc object. + * @param[in] frame A GstVideoCodecFrame containing the input to encode. + * @return Upon success, returns GST_FLOW_OK, GST_FLOW_ERROR on failure. + */ +static GstFlowReturn encode_single_frame(GstHailoH265Enc *hailoenc, GstVideoCodecFrame * frame) +{ + GstFlowReturn ret = GST_FLOW_ERROR; + VCEncRet enc_ret; + EncoderParams *enc_params = &(hailoenc->enc_params); + struct timespec start_encode, end_encode; + GST_DEBUG_OBJECT(hailoenc, "Encoding frame number %u in type %u", frame->system_frame_number, enc_params->nextCodingType); + + if (hailoenc->hevc_encoder == NULL) + { + GST_ERROR_OBJECT(hailoenc, "Encoder not initialized"); + return GST_FLOW_ERROR; + } + + ret = gst_hailoh265enc_update_input_buffer(hailoenc, frame); + if (ret != GST_FLOW_OK) + { + GST_ERROR_OBJECT(hailoenc, "Could not update the input buffer"); + return ret; + } + + clock_gettime(CLOCK_MONOTONIC,&start_encode); + enc_ret = EncodeFrame(enc_params, hailoenc->hevc_encoder, &gst_hailoh265enc_slice_ready, hailoenc); + clock_gettime(CLOCK_MONOTONIC,&end_encode); + GST_DEBUG_OBJECT(hailoenc, "Encode took %lu milliseconds", (long)gst_hailoh265enc_difftimespec_ms(end_encode,start_encode)); + + switch (enc_ret) + { + case VCENC_FRAME_READY: + enc_params->picture_enc_cnt++; + if (enc_params->encOut.streamSize == 0) + { + enc_params->picture_cnt++; + ret = GST_FLOW_OK; + GST_WARNING_OBJECT(hailoenc, "Encoder didn't return any output for frame %d", enc_params->picture_cnt); + } + else + { + if(enc_params->multislice_encoding==0) + { + // Get the encoded output buffer. + frame->output_buffer = gst_hailoh265enc_get_encoded_buffer(hailoenc); + + ret = gst_video_encoder_finish_frame(GST_VIDEO_ENCODER (hailoenc), frame); + if (ret != GST_FLOW_OK) + { + GST_ERROR_OBJECT(hailoenc, "Could not send incoded buffer, reason %d", ret); + return ret; + } + } + UpdateEncoderGOP(enc_params, hailoenc->hevc_encoder); + } + break; + default: + GST_ERROR_OBJECT(hailoenc, "Encoder failed with error %d", enc_ret); + ret = GST_FLOW_ERROR; + return ret; + break; + } + return ret; +} + +/** + * Encode multiple frames - encode 1 frame or more according to the GOP config order. + * + * @param[in] encoder The GstVideoEncoder object. + * @return Upon success, returns GST_FLOW_OK, GST_FLOW_ERROR on failure. + * @note All the frames that will be encoded are queued in the GstVideoEncoder object and retreived + * via the gst_video_encoder_get_frame function. + */ +static GstFlowReturn +gst_hailoh265enc_encode_frames(GstVideoEncoder * encoder) +{ + GstHailoH265Enc *hailoenc = (GstHailoH265Enc *) encoder; + EncoderParams *enc_params = &(hailoenc->enc_params); + GstVideoCodecFrame * current_frame; + GstFlowReturn ret = GST_FLOW_ERROR; + guint gop_size = enc_params->encIn.gopSize; + GST_DEBUG_OBJECT(hailoenc, "Encoding %u frames", gop_size); + + if (hailoenc->hevc_encoder == NULL) + { + GST_ERROR_OBJECT(hailoenc, "Encoder not initialized"); + return GST_FLOW_ERROR; + } + + if (gop_size <= 0) + { + GST_ERROR_OBJECT(hailoenc, "Invalid current GOP size %d", gop_size); + return GST_FLOW_ERROR; + } + + // Assuming enc_params->encIn.gopSize is not 0. + for (int i=0;ipicture_cnt); + if (!current_frame) + { + GST_ERROR_OBJECT(hailoenc, "frame %u is missing", enc_params->picture_cnt); + break; + } + ret = encode_single_frame(hailoenc, current_frame); + gst_video_codec_frame_unref(current_frame); + if (ret != GST_FLOW_OK) + { + GST_ERROR_OBJECT(hailoenc, "Encoding frame %u failed.", enc_params->picture_cnt); + break; + } + } + + if (hailoenc->update_config && enc_params->nextCodingType == VCENC_INTRA_FRAME) + { + GST_INFO_OBJECT(hailoenc, "Finished GOP, restarting encoder in order to update config"); + hailoenc->stream_restart = TRUE; + hailoenc->update_config=FALSE; + } + + return ret; +} + + +/******************************** +GstVideoEncoder Virtual Functions +********************************/ + +static gboolean +gst_hailoh265enc_set_format(GstVideoEncoder * encoder, GstVideoCodecState * state) +{ + // GstCaps *other_caps; + GstCaps *allowed_caps; + GstCaps *icaps; + GstVideoCodecState *output_format; + GstHailoH265Enc *hailoenc = (GstHailoH265Enc *) encoder; + EncoderParams *enc_params = &(hailoenc->enc_params); + + gboolean updated_caps = gst_hailoh265enc_update_params(hailoenc, &(state->info)); + if (hailoenc->hevc_encoder != NULL && updated_caps) + { + GST_INFO_OBJECT(hailoenc, "Encoder parameters changed, restarting encoder"); + hailoenc->stream_restart = TRUE; + hailoenc->hard_restart = TRUE; + } + else if (hailoenc->hevc_encoder == NULL) + { + /* Encoder initialization */ + if (OpenEncoder(&(hailoenc->hevc_encoder), enc_params) != 0) + { + return FALSE; + } + + VCEncRet ret = gst_hailoh265enc_encode_header(encoder); + if (ret != VCENC_OK) + { + GST_ERROR_OBJECT(hailoenc, "Failed to encode headers, returned %d", ret); + return FALSE; + } + } + + /* some codecs support more than one format, first auto-choose one */ + GST_DEBUG_OBJECT (hailoenc, "picking an output format ..."); + allowed_caps = gst_pad_get_allowed_caps (GST_VIDEO_ENCODER_SRC_PAD (encoder)); + if (!allowed_caps) { + GST_DEBUG_OBJECT (hailoenc, "... but no peer, using template caps"); + /* we need to copy because get_allowed_caps returns a ref, and + * get_pad_template_caps doesn't */ + allowed_caps = + gst_pad_get_pad_template_caps (GST_VIDEO_ENCODER_SRC_PAD (encoder)); + } + GST_DEBUG_OBJECT (hailoenc, "chose caps %" GST_PTR_FORMAT, allowed_caps); + + icaps = gst_caps_fixate (allowed_caps); + + /* Store input state and set output state */ + if (hailoenc->input_state) + gst_video_codec_state_unref (hailoenc->input_state); + hailoenc->input_state = gst_video_codec_state_ref (state); + GST_DEBUG_OBJECT (hailoenc, "Setting output caps state %" GST_PTR_FORMAT, icaps); + + output_format = gst_video_encoder_set_output_state (encoder, icaps, state); + + gst_video_codec_state_unref (output_format); + + /* Store some tags */ + { + GstTagList *tags = gst_tag_list_new_empty (); + gst_video_encoder_merge_tags (encoder, tags, GST_TAG_MERGE_REPLACE); + gst_tag_list_unref (tags); + } + gint max_delayed_frames = 5; + GstClockTime latency; + latency = gst_util_uint64_scale_ceil (GST_SECOND * 1, max_delayed_frames, 25); + gst_video_encoder_set_latency (GST_VIDEO_ENCODER (encoder), latency, latency); + + return TRUE; +} + +static gboolean +gst_hailoh265enc_propose_allocation(GstVideoEncoder * encoder, GstQuery * query) +{ + GstHailoH265Enc *hailoenc = (GstHailoH265Enc *) encoder; + GST_DEBUG_OBJECT(hailoenc, "hailoh265enc propose allocation callback"); + + gst_query_add_allocation_meta (query, GST_VIDEO_META_API_TYPE, NULL); + + return GST_VIDEO_ENCODER_CLASS (parent_class)->propose_allocation (encoder, + query); +} + +static gboolean +gst_hailoh265enc_flush(GstVideoEncoder * encoder) +{ + return TRUE; +} + +static gboolean +gst_hailoh265enc_start(GstVideoEncoder * encoder) +{ + GstHailoH265Enc *hailoenc = (GstHailoH265Enc *) encoder; + EncoderParams *enc_params = &(hailoenc->enc_params); + VCEncIn * pEncIn = &(enc_params->encIn); + + if (VCEncInitGopConfigs (enc_params->gopSize, NULL, &(enc_params->encIn.gopConfig), enc_params->gopCfgOffset, enc_params->bFrameQpDelta, enc_params->codecH264) != 0) + { + return FALSE; + } + + /* Allocate input and output buffers */ + if (AllocRes(enc_params) != 0) + { + FreeRes(enc_params); + return FALSE; + } + pEncIn->timeIncrement = 0; + pEncIn->busOutBuf = enc_params->outbufMem.busAddress; + pEncIn->outBufSize = enc_params->outbufMem.size; + pEncIn->pOutBuf = enc_params->outbufMem.virtualAddress; + + gst_video_encoder_set_min_pts (encoder, GST_SECOND * 60 * 60 * 1000); + + return TRUE; +} + +static gboolean +gst_hailoh265enc_stop(GstVideoEncoder * encoder) +{ + GstHailoH265Enc *hailoenc = (GstHailoH265Enc *) encoder; + EncoderParams *enc_params = &(hailoenc->enc_params); + CloseEncoder(hailoenc->hevc_encoder); + hailoenc->hevc_encoder = NULL; + FreeRes(enc_params); + + return TRUE; +} + +static GstFlowReturn +gst_hailoh265enc_finish(GstVideoEncoder * encoder) +{ + GstHailoH265Enc *hailoenc = (GstHailoH265Enc *) encoder; + VCEncOut *pEncOut = &(hailoenc->enc_params.encOut); + VCEncIn *pEncIn = &(hailoenc->enc_params.encIn); + VCEncRet enc_ret; + + /* End stream */ + enc_ret = VCEncStrmEnd(hailoenc->hevc_encoder, pEncIn, pEncOut); + if (enc_ret != VCENC_OK) + { + GST_ERROR_OBJECT(hailoenc, "Failed to end stream, returned %d", enc_ret); + return GST_FLOW_ERROR; + } + return gst_pad_push(encoder->srcpad, gst_hailoh265enc_get_encoded_buffer(hailoenc)); +} + +static GstFlowReturn +gst_hailoh265enc_handle_frame(GstVideoEncoder * encoder, + GstVideoCodecFrame * frame) +{ + GstHailoH265Enc *hailoenc = (GstHailoH265Enc *) encoder; + GstFlowReturn ret = GST_FLOW_ERROR; + EncoderParams *enc_params = &(hailoenc->enc_params); + GList *frames; + guint delayed_frames; + GstVideoCodecFrame * oldest_frame; + struct timespec start_handle, end_handle; + clock_gettime(CLOCK_MONOTONIC,&start_handle); + GST_DEBUG_OBJECT(hailoenc, "Received frame number %u", frame->system_frame_number); + + if (hailoenc->stream_restart) + { + ret = gst_hailoh265enc_stream_restart(encoder, frame); + if (ret != GST_FLOW_OK) + { + GST_ERROR_OBJECT(hailoenc, "Failed to restart encoder"); + return ret; + } + } + + // Update Slice Encoding parameters + enc_params->multislice_encoding=0; + enc_params->strmPtr = NULL; + + if (GST_VIDEO_CODEC_FRAME_IS_FORCE_KEYFRAME(frame)) + { + GST_DEBUG_OBJECT(hailoenc, "Forcing keyframe"); + // Adding sync point in order to delete forced keyframe evnet from the queue. + GST_VIDEO_CODEC_FRAME_SET_SYNC_POINT(frame); + ForceKeyframe(enc_params, hailoenc->hevc_encoder); + oldest_frame = gst_video_encoder_get_oldest_frame(encoder); + ret = encode_single_frame(hailoenc, oldest_frame); + if (ret != GST_FLOW_OK) + { + GST_ERROR_OBJECT(hailoenc, "Failed to encode forced keyframe"); + return ret; + } + if (frame == oldest_frame) + { + gst_video_codec_frame_unref(oldest_frame); + return ret; + } + } + + switch (enc_params->nextCodingType) + { + case VCENC_INTRA_FRAME: + ret = encode_single_frame(hailoenc, frame); + break; + case VCENC_PREDICTED_FRAME: + frames = gst_video_encoder_get_frames (encoder); + delayed_frames = g_list_length (frames); + g_list_free_full (frames, (GDestroyNotify) gst_video_codec_frame_unref); + guint gop_size = enc_params->encIn.gopSize; + if (delayed_frames == gop_size) + { + ret = gst_hailoh265enc_encode_frames(encoder); + } + else if (delayed_frames < gop_size) + { + ret = GST_FLOW_OK; + } + else + { + GST_ERROR_OBJECT(hailoenc, "Skipped too many frames"); + } + break; + case VCENC_BIDIR_PREDICTED_FRAME: + GST_ERROR_OBJECT(hailoenc,"Got B frame without pending P frame"); + break; + default: + GST_ERROR_OBJECT(hailoenc,"Unknown coding type %d", (int)enc_params->nextCodingType); + break; + } + clock_gettime(CLOCK_MONOTONIC,&end_handle); + GST_DEBUG_OBJECT(hailoenc, "handle_frame took %lu milliseconds", (long)gst_hailoh265enc_difftimespec_ms(end_handle,start_handle)); + return ret; +} diff --git a/core/hailo/plugins/encoder/gsthailoh265enc.h b/core/hailo/plugins/encoder/gsthailoh265enc.h new file mode 100644 index 0000000..29b913a --- /dev/null +++ b/core/hailo/plugins/encoder/gsthailoh265enc.h @@ -0,0 +1,41 @@ +#pragma once + +#include +#include +#include "hailoencoder.h" + +G_BEGIN_DECLS + +typedef struct _GstHailoH265Enc GstHailoH265Enc; +typedef struct _GstHailoH265EncClass GstHailoH265EncClass; +#define GST_TYPE_HAILO_H265_ENC (gst_hailoh265enc_get_type()) +#define GST_HAILO_H265_ENC(obj) (G_TYPE_CHECK_INSTANCE_CAST((obj), GST_TYPE_HAILO_H265_ENC, GstHailoH265Enc)) +#define GST_HAILO_H265_ENC_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST((klass), GST_TYPE_HAILO_H265_ENC, GstHailoH265EncClass)) +#define GST_IS_HAILO_H265_ENC(obj) (G_TYPE_CHECK_INSTANCE_TYPE((obj), GST_TYPE_HAILO_H265_ENC)) +#define GST_IS_HAILO_H265_ENC_CLASS(obj) (G_TYPE_CHECK_CLASS_TYPE((klass), GST_TYPE_HAILO_H265_ENC)) +#define GST_HAILO_H265_ENC_GET_CLASS(obj) (G_TYPE_INSTANCE_GET_CLASS ((obj), GST_TYPE_HAILO_H265_ENC, GstHailoH265EncClass)) + +struct _GstHailoH265Enc +{ + GstVideoEncoder parent; + GstVideoCodecState *input_state; + GstBuffer *header_buffer; + VCEncApiVersion apiVer; + VCEncBuild encBuild; + VCEncGopPicConfig gopPicCfg[MAX_GOP_PIC_CONFIG_NUM]; + EncoderParams enc_params; + VCEncInst hevc_encoder; + gboolean stream_restart; + gboolean hard_restart; + gboolean update_config; + gboolean update_gop_size; +}; + +struct _GstHailoH265EncClass +{ + GstVideoEncoderClass parent_class; +}; + +G_GNUC_INTERNAL GType gst_hailoh265enc_get_type(void); + +G_END_DECLS diff --git a/core/hailo/plugins/encoder/hailoencoder.c b/core/hailo/plugins/encoder/hailoencoder.c index 4f16a97..e84fbd3 100644 --- a/core/hailo/plugins/encoder/hailoencoder.c +++ b/core/hailo/plugins/encoder/hailoencoder.c @@ -1,33 +1,60 @@ #include "hailoencoder.h" -#include "error.h" -#define DEFAULT -255 -static void AdaptiveGOPSizeDecision(EncoderParams *enc_params, - VCEncInst encoder, - VCEncIn *pEncIn); -static void GetAlignedPicSizebyFormat(VCEncPictureType type, u32 width, u32 height, u32 alignment, - u32 *luma_Size, u32 *chroma_Size, u32 *picture_Size); - -void SetDefaultParameters(EncoderParams *enc_params) +void SetDefaultParameters(EncoderParams *enc_params, bool codecH264) { - enc_params->frameRateNumer = DEFAULT; - enc_params->frameRateDenom = DEFAULT; - - enc_params->width = DEFAULT; - enc_params->height = DEFAULT; - enc_params->inputFormat = VCENC_YUV420_SEMIPLANAR; - enc_params->profile = VCENC_HEVC_MAIN_10_PROFILE; - enc_params->level = VCENC_HEVC_LEVEL_5_2; - enc_params->streamType = VCENC_BYTE_STREAM; + enc_params->frameRateNumer = DEFAULT_UNCHANGED; + enc_params->frameRateDenom = DEFAULT_UNCHANGED; - enc_params->max_cu_size = 64; - enc_params->min_cu_size = 8; - enc_params->max_tr_size = 16; - enc_params->min_tr_size = 4; - enc_params->tr_depth_intra = 2; //mfu =>0 - enc_params->tr_depth_inter = (enc_params->max_cu_size == 64) ? 4 : 3; - enc_params->intraPicRate = 0; // only first is IDR. - enc_params->codecH264 = 0; + enc_params->width = DEFAULT_UNCHANGED; + enc_params->height = DEFAULT_UNCHANGED; + + enc_params->codecH264 = codecH264; + enc_params->inputFormat = DEFAULT_INPUT_FORMAT; + enc_params->profile = enc_params->codecH264 ? DEFAULT_H264_PROFILE : DEFAULT_HEVC_PROFILE; + enc_params->level = enc_params->codecH264 ? DEFAULT_H264_LEVEL : DEFAULT_HEVC_LEVEL; + enc_params->streamType = VCENC_BYTE_STREAM; + enc_params->gopSize = DEFAULT_GOP_SIZE; + enc_params->gopLength = DEFAULT_GOP_LENGTH; + enc_params->intraPicRate = DEFAULT_INTRA_PIC_RATE; + enc_params->qphdr = DEFAULT_QPHDR; + enc_params->qpmin = DEFAULT_QPMIN; + enc_params->qpmax = DEFAULT_QPMAX; + enc_params->intra_qp_delta = DEFAULT_INTRA_QP_DELTA; + enc_params->fixed_intra_qp = DEFAULT_FIXED_INTRA_QP; + enc_params->bitrate = DEFAULT_BITRATE; + enc_params->tolMovingBitRate = DEFAULT_TOL_MOVING_BITRATE; + enc_params->bitVarRangeI = DEFAULT_BITVAR_RANGE_I; + enc_params->bitVarRangeP = DEFAULT_BITVAR_RANGE_P; + enc_params->bitVarRangeB = DEFAULT_BITVAR_RANGE_B; + enc_params->pictureRc = 1; + enc_params->ctbRc = 0; + enc_params->blockRcSize = 0; + enc_params->pictureSkip = 0; + enc_params->hrd = 0; + enc_params->hrdCpbSize = DEFAULT_HRD_CPB_SIZE; + enc_params->monitorFrames = 0; + enc_params->roiArea1 = NULL; + enc_params->roiArea2 = NULL; + enc_params->compressor = 3; + + if (enc_params->codecH264) + { + enc_params->max_cu_size = 16; + enc_params->min_cu_size = 8; + enc_params->max_tr_size = 16; + enc_params->min_tr_size = 4; + enc_params->tr_depth_intra = 1; + enc_params->tr_depth_inter = 2; + } + else + { + enc_params->max_cu_size = 64; + enc_params->min_cu_size = 8; + enc_params->max_tr_size = 16; + enc_params->min_tr_size = 4; + enc_params->tr_depth_intra = 2; + enc_params->tr_depth_inter = 4; + } enc_params->outBufSizeMax = 12; enc_params->roiMapDeltaQpBlockUnit = 0; @@ -47,12 +74,20 @@ int InitEncoderConfig(EncoderParams *enc_params, VCEncInst *pEnc) cfg.strongIntraSmoothing = 1; cfg.streamType = enc_params->streamType; - cfg.codecH264 = 0; + cfg.codecH264 = enc_params->codecH264; cfg.profile = enc_params->profile; cfg.level = enc_params->level; - cfg.bitDepthLuma = DEFAULT_BIT_DEPTH; - cfg.bitDepthChroma = DEFAULT_BIT_DEPTH; + if (cfg.profile == VCENC_HEVC_MAIN_10_PROFILE || cfg.profile == VCENC_H264_HIGH_10_PROFILE) + { + cfg.bitDepthLuma = 10; + cfg.bitDepthChroma = 10; + } + else + { + cfg.bitDepthLuma = 8; + cfg.bitDepthChroma = 8; + } //default maxTLayer cfg.maxTLayers = 1; @@ -76,13 +111,13 @@ int InitEncoderConfig(EncoderParams *enc_params, VCEncInst *pEnc) } cfg.refFrameAmount = maxRefPics + cfg.interlacedFrame + (enc_params->encIn.gopConfig.ltrInterval > 0); cfg.maxTLayers = maxTemporalId +1; - cfg.compressor = 3; + cfg.compressor = enc_params->compressor; cfg.enableOutputCuInfo = 0; cfg.exp_of_alignment = 0; cfg.refAlignmentExp = 0; cfg.AXIAlignment = 0; - cfg.AXIreadOutstandingNum = ENCH2_ASIC_AXI_READ_OUTSTANDING_NUM; - cfg.AXIwriteOutstandingNum = ENCH2_ASIC_AXI_WRITE_OUTSTANDING_NUM; + cfg.AXIreadOutstandingNum = 64; //ENCH2_ASIC_AXI_READ_OUTSTANDING_NUM; + cfg.AXIwriteOutstandingNum = 64; //ENCH2_ASIC_AXI_WRITE_OUTSTANDING_NUM; if ((ret = VCEncInit(&cfg, pEnc)) != VCENC_OK) { //PrintErrorValue("VCEncInit() failed.", ret); @@ -91,8 +126,58 @@ int InitEncoderConfig(EncoderParams *enc_params, VCEncInst *pEnc) return 0; } +static void UpdateROIArea(EncoderParams *enc_params, VCEncCodingCtrl codingCfg) +{ + if (enc_params->roiArea1) + { + codingCfg.roi1Area.enable = 1; + sscanf(enc_params->roiArea1, + "%u:%u:%u:%u:%u", + &codingCfg.roi1Area.left, + &codingCfg.roi1Area.top, + &codingCfg.roi1Area.right, + &codingCfg.roi1Area.bottom, + &codingCfg.roi1DeltaQp); + } + if (enc_params->roiArea2) + { + codingCfg.roi2Area.enable = 1; + sscanf(enc_params->roiArea2, + "%u:%u:%u:%u:%u", + &codingCfg.roi2Area.left, + &codingCfg.roi2Area.top, + &codingCfg.roi2Area.right, + &codingCfg.roi2Area.bottom, + &codingCfg.roi2DeltaQp); + } +} + +int UpdateEncoderROIArea(EncoderParams *enc_params, VCEncInst *pEnc) +{ + VCEncInst encoder; + VCEncRet ret; + VCEncCodingCtrl codingCfg; + + encoder = *pEnc; + + /* Encoder setup: coding control */ + if ((ret = VCEncGetCodingCtrl(encoder, &codingCfg)) != VCENC_OK) + { + CloseEncoder(encoder); + return -1; + } + + UpdateROIArea(enc_params, codingCfg); -int InitEncoderCodingConfig(VCEncInst *pEnc) + if ((ret = VCEncSetCodingCtrl(encoder, &codingCfg)) != VCENC_OK) + { + CloseEncoder(encoder); + return -1; + } + return 0; +} + +int InitEncoderCodingConfig(EncoderParams *enc_params, VCEncInst *pEnc) { VCEncInst encoder; VCEncRet ret; @@ -145,17 +230,10 @@ int InitEncoderCodingConfig(VCEncInst *pEnc) codingCfg.ipcmMapEnable = 0; codingCfg.pcm_enabled_flag = 0; - codingCfg.roi1Area.enable = 0; - codingCfg.roi1Area.top = codingCfg.roi1Area.left = - codingCfg.roi1Area.bottom = - codingCfg.roi1Area.right = 0; - codingCfg.roi2Area.enable = 0; - codingCfg.roi2Area.top = codingCfg.roi2Area.left = - codingCfg.roi2Area.bottom = - codingCfg.roi2Area.right = 0; - codingCfg.roi1DeltaQp = 0; - codingCfg.roi2DeltaQp = 0; - codingCfg.codecH264 = 0; + + UpdateROIArea(enc_params, codingCfg); + + codingCfg.codecH264 = enc_params->codecH264; codingCfg.roiMapDeltaQpEnable = 0; codingCfg.roiMapDeltaQpBlockUnit = 0; @@ -185,7 +263,6 @@ int InitEncoderCodingConfig(VCEncInst *pEnc) return 0; } - int InitEncoderRateConfig(EncoderParams *enc_params, VCEncInst *pEnc) { VCEncInst encoder; @@ -202,27 +279,37 @@ int InitEncoderRateConfig(EncoderParams *enc_params, VCEncInst *pEnc) return -1; } - rcCfg.qpHdr = 26; - rcCfg.qpMin = 0; - rcCfg.qpMax = 51; - rcCfg.pictureSkip = 0; - rcCfg.pictureRc =0; - rcCfg.ctbRc = 0; - - rcCfg.blockRCSize = 0; - rcCfg.bitPerSecond = 60000000; - rcCfg.bitVarRangeI = 2000; - rcCfg.bitVarRangeP = 2000; - rcCfg.bitVarRangeB = 2000; - rcCfg.tolMovingBitRate = 2000; + rcCfg.qpHdr = enc_params->qphdr; + rcCfg.qpMin = enc_params->qpmin; + rcCfg.qpMax = enc_params->qpmax; + rcCfg.pictureSkip = enc_params->pictureSkip; + rcCfg.pictureRc = enc_params->pictureRc; + rcCfg.ctbRc = enc_params->ctbRc; + + rcCfg.blockRCSize = enc_params->blockRcSize; + + rcCfg.bitPerSecond = enc_params->bitrate; + rcCfg.bitVarRangeI = enc_params->bitVarRangeI; + rcCfg.bitVarRangeP = enc_params->bitVarRangeP; + rcCfg.bitVarRangeB = enc_params->bitVarRangeB; + rcCfg.tolMovingBitRate = enc_params->tolMovingBitRate; - rcCfg.monitorFrames = (enc_params->frameRateNumer+enc_params->frameRateDenom-1) / enc_params->frameRateDenom; - rcCfg.hrd = 0; - rcCfg.hrdCpbSize = 10000000; + if (enc_params->monitorFrames != 0) + rcCfg.monitorFrames = enc_params->monitorFrames; + else + rcCfg.monitorFrames = (enc_params->frameRateNumer+enc_params->frameRateDenom-1) / enc_params->frameRateDenom; - // rcCfg.gopLen = 0; // default is good. - rcCfg.intraQpDelta = -5; - rcCfg.fixedIntraQp = 0; + if(rcCfg.monitorFrames>MAX_MONITOR_FRAMES) + rcCfg.monitorFrames=MAX_MONITOR_FRAMES; + if(rcCfg.monitorFrameshrd; + rcCfg.hrdCpbSize = enc_params->hrdCpbSize; + + rcCfg.gopLen = enc_params->gopLength; + rcCfg.intraQpDelta = enc_params->intra_qp_delta; + rcCfg.fixedIntraQp = enc_params->fixed_intra_qp; if ((ret = VCEncSetRateCtrl(encoder, &rcCfg)) != VCENC_OK) { @@ -326,7 +413,28 @@ int OpenEncoder(VCEncInst *encoder, EncoderParams *enc_params) { if (InitEncoderConfig(enc_params, encoder) != 0) return -1; - if (InitEncoderCodingConfig(encoder) != 0) + + return UpdateEncoderConfig(encoder, enc_params); +} + +/*------------------------------------------------------------------------------ + + UpdateEncoderConfig + Configure an encoder instance. + + Params: + encoder - place where to save the new encoder instance + enc_params - Arguments + Return: + 0 - for success + -1 - error + +------------------------------------------------------------------------------*/ +int UpdateEncoderConfig(VCEncInst *encoder, EncoderParams *enc_params) +{ + enc_params->idr_interval = enc_params->intraPicRate; + + if (InitEncoderCodingConfig(enc_params, encoder) != 0) return -1; if (InitEncoderRateConfig(enc_params, encoder) != 0) return -1; @@ -357,33 +465,6 @@ void CloseEncoder(VCEncInst encoder) -u32 SetupInputBuffer(EncoderParams *enc_params, VCEncIn *pEncIn) -{ - u32 src_img_size; - u32 bitsPerPixel = VCEncGetBitsPerPixel(enc_params->inputFormat); - ASSERT(bitsPerPixel != 0); - - u32 size_lum = 0; - u32 size_ch = 0; - - GetAlignedPicSizebyFormat(enc_params->inputFormat, enc_params->width, enc_params->height, enc_params->alignment, &size_lum, &size_ch, NULL); - - pEncIn->busLuma = enc_params->pictureMem.busAddress; - enc_params->lum = (u8 *)enc_params->pictureMem.virtualAddress; - pEncIn->busChromaU = pEncIn->busLuma + size_lum; - enc_params->cb = enc_params->lum + (u32)size_lum; - pEncIn->busChromaV = pEncIn->busChromaU + (u32)size_ch/2; - enc_params->cr = enc_params->cb + (u32)size_ch/2; - src_img_size = enc_params->width * enc_params->height * bitsPerPixel / 8; - - return src_img_size; -} - -static u32 get_aligned_width(int width, i32 input_format) -{ - return (width + 15) & (~15); -} - /*------------------------------------------------------------------------------ AllocRes @@ -398,53 +479,22 @@ static u32 get_aligned_width(int width, i32 input_format) as inside EWLMallocLinear(). ------------------------------------------------------------------------------*/ -int AllocRes(VCEncInst enc, EncoderParams *enc_params) +int AllocRes(EncoderParams *enc_params) { i32 ret; - u32 pictureSize; u32 outbufSize; - u32 lumaSize,chromaSize; - - if (enc_params->alignment == 0) - pictureSize = get_aligned_width(enc_params->width, enc_params->inputFormat) * enc_params->height * - VCEncGetBitsPerPixel(enc_params->inputFormat) / 8; - else + EWLInitParam_t ewl_params; + ewl_params.clientType = EWL_CLIENT_TYPE_HEVC_ENC; + enc_params->ewl = (void *)EWLInit(&ewl_params); + if (NULL == enc_params->ewl) { - if (enc_params->inputFormat==VCENC_YUV420_SEMIPLANAR||enc_params->inputFormat==VCENC_YUV420_SEMIPLANAR_VU) - { - lumaSize = STRIDE(enc_params->width,enc_params->alignment)* enc_params->height; - chromaSize = STRIDE(enc_params->width,enc_params->alignment)* enc_params->height/2; - pictureSize = lumaSize + chromaSize; - } - else if (enc_params->inputFormat==VCENC_YUV422_INTERLEAVED_YUYV||enc_params->inputFormat==VCENC_YUV422_INTERLEAVED_UYVY) - { - pictureSize = STRIDE(enc_params->width*2,enc_params->alignment)* enc_params->height; - }else{ - pictureSize = 0; //This situation won't happen - } - } - - GetAlignedPicSizebyFormat(enc_params->inputFormat,enc_params->width,enc_params->height, - enc_params->alignment,&lumaSize,&chromaSize,&pictureSize); - - enc_params->pictureMem.virtualAddress = NULL; - enc_params->outbufMem.virtualAddress = NULL; - - /* Here we use the EWL instance directly from the encoder - * because it is the easiest way to allocate the linear memories */ - ret = EWLMallocLinear(((struct vcenc_instance *)enc)->asic.ewl, pictureSize,enc_params->alignment, - &enc_params->pictureMem); - if (ret != EWL_OK) - { - enc_params->pictureMem.virtualAddress = NULL; return 1; } - + /* Limited amount of memory on some test environment */ - outbufSize = 4 * enc_params->pictureMem.size < ((u32)enc_params->outBufSizeMax * 1024 * 1024) ? - 4 * enc_params->pictureMem.size : ((u32)enc_params->outBufSizeMax * 1024 * 1024); + outbufSize = ((u32)enc_params->outBufSizeMax * 1024 * 1024); - ret = EWLMallocLinear(((struct vcenc_instance *)enc)->asic.ewl, outbufSize,enc_params->alignment, + ret = EWLMallocLinear((const void *)enc_params->ewl, outbufSize,enc_params->alignment, &enc_params->outbufMem); if (ret != EWL_OK) { @@ -462,12 +512,12 @@ int AllocRes(VCEncInst enc, EncoderParams *enc_params) Release all resources allcoated byt AllocRes() ------------------------------------------------------------------------------*/ -void FreeRes(VCEncInst enc, EncoderParams *enc_params) +void FreeRes(EncoderParams *enc_params) { - if (enc_params->pictureMem.virtualAddress != NULL) - EWLFreeLinear(((struct vcenc_instance *)enc)->asic.ewl, &enc_params->pictureMem); if (enc_params->outbufMem.virtualAddress != NULL) - EWLFreeLinear(((struct vcenc_instance *)enc)->asic.ewl, &enc_params->outbufMem); + EWLFreeLinear((const void *)enc_params->ewl, &enc_params->outbufMem); + if (NULL != enc_params->ewl) + (void)EWLRelease((const void *)enc_params->ewl); } @@ -479,188 +529,27 @@ VCEncRet EncodeFrame(EncoderParams *enc_params, VCEncInst encoder, VCEncOut *pEncOut = &(enc_params->encOut); pEncIn->codingType = (pEncIn->poc == 0) ? VCENC_INTRA_FRAME : enc_params->nextCodingType; - enc_params->last_idr_picture_cnt = enc_params->picture_cnt; + if (pEncIn->codingType == VCENC_INTRA_FRAME) + { + pEncIn->poc = 0; + enc_params->last_idr_picture_cnt = enc_params->picture_cnt; + } return VCEncStrmEncode(encoder, pEncIn, pEncOut, sliceReadyCbFunc, pAppData); } -void UpdateEncoder(EncoderParams *enc_params, - VCEncInst encoder) +void ForceKeyframe(EncoderParams *enc_params, VCEncInst encoder) { VCEncIn *pEncIn = &(enc_params->encIn); - VCEncOut *pEncOut = &(enc_params->encOut); - pEncIn->timeIncrement = enc_params->frameRateDenom; - enc_params->total_bits += pEncOut->streamSize * 8; - enc_params->streamSize += pEncOut->streamSize; - enc_params->validencodedframenumber++; - //Adaptive GOP size decision - if ((enc_params->gopSize == 0) && pEncIn->codingType != VCENC_INTRA_FRAME) - { - AdaptiveGOPSizeDecision(enc_params, encoder, pEncIn); - } - enc_params->nextCodingType = find_next_pic(enc_params); -} - -/*------------------------------------------------------------------------------ - - Static Functions - -------------------------------------------------------------------------------*/ - -static void AdaptiveGOPSizeDecision(EncoderParams *enc_params, - VCEncInst encoder, - VCEncIn *pEncIn) -{ - struct vcenc_instance *vcenc_instance = (struct vcenc_instance *)encoder; - unsigned int uiIntraCu8Num = vcenc_instance->asic.regs.intraCu8Num; - unsigned int uiSkipCu8Num = vcenc_instance->asic.regs.skipCu8Num; - unsigned int uiPBFrameCost = vcenc_instance->asic.regs.PBFrame4NRdCost; - double dIntraVsInterskip = (double)uiIntraCu8Num/(double)((enc_params->width/8) * (enc_params->height/8)); - double dSkipVsInterskip = (double)uiSkipCu8Num/(double)((enc_params->width/8) * (enc_params->height/8)); - - enc_params->gop_frm_num++; - enc_params->sum_intra_vs_interskip += dIntraVsInterskip; - enc_params->sum_skip_vs_interskip += dSkipVsInterskip; - enc_params->sum_costP += (pEncIn->codingType == VCENC_PREDICTED_FRAME)? uiPBFrameCost:0; - enc_params->sum_costB += (pEncIn->codingType == VCENC_BIDIR_PREDICTED_FRAME)? uiPBFrameCost:0; - enc_params->sum_intra_vs_interskipP += (pEncIn->codingType == VCENC_PREDICTED_FRAME)? dIntraVsInterskip:0; - enc_params->sum_intra_vs_interskipB += (pEncIn->codingType == VCENC_BIDIR_PREDICTED_FRAME)? dIntraVsInterskip:0; - if(pEncIn->gopPicIdx == pEncIn->gopSize-1)//last frame of the current gop. decide the gopsize of next gop. - { - dIntraVsInterskip = enc_params->sum_intra_vs_interskip/enc_params->gop_frm_num; - dSkipVsInterskip = enc_params->sum_skip_vs_interskip/enc_params->gop_frm_num; - enc_params->sum_costB = (enc_params->gop_frm_num>1)?(enc_params->sum_costB/(enc_params->gop_frm_num-1)):0xFFFFFFF; - enc_params->sum_intra_vs_interskipB = (enc_params->gop_frm_num>1)?(enc_params->sum_intra_vs_interskipB/(enc_params->gop_frm_num-1)):0xFFFFFFF; - //Enabled adaptive GOP size for large resolution - if (((enc_params->width * enc_params->height) >= (1280 * 720)) || ((MAX_ADAPTIVE_GOP_SIZE >3)&&((enc_params->width * enc_params->height) >= (416 * 240)))) - { - if ((((double)enc_params->sum_costP/(double)enc_params->sum_costB)<1.1)&&(dSkipVsInterskip >= 0.95)) - { - enc_params->last_gopsize = enc_params->nextGopSize = 1; - } - else if (((double)enc_params->sum_costP/(double)enc_params->sum_costB)>5) - { - enc_params->nextGopSize = enc_params->last_gopsize; - } - else - { - if( ((enc_params->sum_intra_vs_interskipP > 0.40) && (enc_params->sum_intra_vs_interskipP < 0.70)&& (enc_params->sum_intra_vs_interskipB < 0.10)) ) - { - enc_params->last_gopsize++; - if(enc_params->last_gopsize==5 || enc_params->last_gopsize==7) - { - enc_params->last_gopsize++; - } - enc_params->last_gopsize = MIN(enc_params->last_gopsize, MAX_ADAPTIVE_GOP_SIZE); - enc_params->nextGopSize = enc_params->last_gopsize; // - } - else if (dIntraVsInterskip >= 0.30) - { - enc_params->last_gopsize = enc_params->nextGopSize = 1; //No B - } - else if (dIntraVsInterskip >= 0.20) - { - enc_params->last_gopsize = enc_params->nextGopSize = 2; //One B - } - else if (dIntraVsInterskip >= 0.10) - { - enc_params->last_gopsize--; - if(enc_params->last_gopsize == 5 || enc_params->last_gopsize==7) - { - enc_params->last_gopsize--; - } - enc_params->last_gopsize = MAX(enc_params->last_gopsize, 3); - enc_params->nextGopSize = enc_params->last_gopsize; // - } - else - { - enc_params->last_gopsize++; - if(enc_params->last_gopsize==5 || enc_params->last_gopsize==7) - { - enc_params->last_gopsize++; - } - enc_params->last_gopsize = MIN(enc_params->last_gopsize, MAX_ADAPTIVE_GOP_SIZE); - enc_params->nextGopSize = enc_params->last_gopsize; // - } - } - } - else - { - enc_params->nextGopSize = 3; - } - enc_params->gop_frm_num = 0; - enc_params->sum_intra_vs_interskip = 0; - enc_params->sum_skip_vs_interskip = 0; - enc_params->sum_costP = 0; - enc_params->sum_costB = 0; - enc_params->sum_intra_vs_interskipP = 0; - enc_params->sum_intra_vs_interskipB = 0; - - enc_params->nextGopSize = MIN(enc_params->nextGopSize, MAX_ADAPTIVE_GOP_SIZE); - } + pEncIn->codingType = VCENC_INTRA_FRAME; + pEncIn->poc = 0; + enc_params->last_idr_picture_cnt = enc_params->picture_cnt; } -static void GetAlignedPicSizebyFormat(VCEncPictureType type, u32 width, u32 height, u32 alignment, - u32 *luma_Size, u32 *chroma_Size, u32 *picture_Size) +void UpdateEncoderGOP(EncoderParams *enc_params, VCEncInst encoder) { - u32 luma_stride = 0, chroma_stride = 0; - u32 lumaSize = 0, chromaSize = 0, pictureSize = 0; - - VCEncGetAlignedStride(width, type, &luma_stride, &chroma_stride, alignment); - switch(type) - { - case VCENC_YUV420_PLANAR: - lumaSize = luma_stride * height; - chromaSize = chroma_stride * height/2*2; - break; - case VCENC_YUV420_SEMIPLANAR: - case VCENC_YUV420_SEMIPLANAR_VU: - lumaSize = luma_stride * height; - chromaSize = chroma_stride * height/2; - break; - case VCENC_YUV422_INTERLEAVED_YUYV: - case VCENC_YUV422_INTERLEAVED_UYVY: - case VCENC_RGB565: - case VCENC_BGR565: - case VCENC_RGB555: - case VCENC_BGR555: - case VCENC_RGB444: - case VCENC_BGR444: - case VCENC_RGB888: - case VCENC_BGR888: - case VCENC_RGB101010: - case VCENC_BGR101010: - lumaSize = luma_stride * height; - chromaSize = 0; - break; - case VCENC_YUV420_PLANAR_10BIT_I010: - lumaSize = luma_stride * height; - chromaSize = chroma_stride * height/2*2; - break; - case VCENC_YUV420_PLANAR_10BIT_P010: - lumaSize = luma_stride * height; - chromaSize = chroma_stride * height/2; - break; - case VCENC_YUV420_PLANAR_10BIT_PACKED_PLANAR: - lumaSize = luma_stride *10/8 * height; - chromaSize = chroma_stride *10/8* height/2*2; - break; - case VCENC_YUV420_10BIT_PACKED_Y0L2: - lumaSize = luma_stride *2*2* height/2; - chromaSize = 0; - break; - default: - printf("not support this format\n"); - chromaSize = lumaSize = 0; - break; - } - - pictureSize = lumaSize + chromaSize; - if (luma_Size != NULL) - *luma_Size = lumaSize; - if (chroma_Size != NULL) - *chroma_Size = chromaSize; - if (picture_Size != NULL) - *picture_Size = pictureSize; + VCEncIn *pEncIn = &(enc_params->encIn); + pEncIn->timeIncrement = enc_params->frameRateDenom; + enc_params->validencodedframenumber++; + enc_params->nextCodingType = find_next_pic(enc_params); } - diff --git a/core/hailo/plugins/encoder/hailoencoder.h b/core/hailo/plugins/encoder/hailoencoder.h index 1096a1f..3884156 100644 --- a/core/hailo/plugins/encoder/hailoencoder.h +++ b/core/hailo/plugins/encoder/hailoencoder.h @@ -2,16 +2,18 @@ #include "enc_common.h" #include "gopconfig.h" -#define DEFAULT_BIT_DEPTH 10 // 10 bit depth - -void SetDefaultParameters(EncoderParams *enc_params); +void SetDefaultParameters(EncoderParams *enc_params, bool codecH264); +int InitEncoderRateConfig(EncoderParams *enc_params, VCEncInst *pEnc); +int UpdateEncoderROIArea(EncoderParams *enc_params, VCEncInst *pEnc); int OpenEncoder(VCEncInst *encoder, EncoderParams *enc_params); +int UpdateEncoderConfig(VCEncInst *encoder, EncoderParams *enc_params); void CloseEncoder(VCEncInst encoder); -int AllocRes(VCEncInst enc, EncoderParams *enc_params); -void FreeRes(VCEncInst enc, EncoderParams *enc_params); +int AllocRes(EncoderParams *enc_params); +void FreeRes(EncoderParams *enc_params); u32 SetupInputBuffer(EncoderParams *enc_params, VCEncIn *pEncIn); -void UpdateEncoder(EncoderParams *enc_params, +void UpdateEncoderGOP(EncoderParams *enc_params, VCEncInst encoder); VCEncRet EncodeFrame(EncoderParams *enc_params, VCEncInst encoder, VCEncSliceReadyCallBackFunc sliceReadyCbFunc, - void *pAppData); \ No newline at end of file + void *pAppData); +void ForceKeyframe(EncoderParams *enc_params, VCEncInst encoder); \ No newline at end of file diff --git a/core/hailo/plugins/encoder/meson.build b/core/hailo/plugins/encoder/meson.build deleted file mode 100644 index b89eb49..0000000 --- a/core/hailo/plugins/encoder/meson.build +++ /dev/null @@ -1,22 +0,0 @@ - -encoder_sources = [ - 'hailoencoder.c', - 'gopconfig.c', - 'gsthailoenc.c' -] - -################################################ -# Hailo Gstreamer Shared Library -################################################ -shared_library('gsthailoencoder', - encoder_sources, - cpp_args : hailo_lib_args + common_args, - c_args : hailo_lib_args + common_args + sysroot_arg, - link_args : sysroot_arg + ['-lhantro_vc8000e', '-lm'], - include_directories: [include_directories('../')], - dependencies : plugin_deps, - gnu_symbol_visibility : 'default', - version: meson.project_version(), - install: true, - install_dir: get_option('libdir') + '/gstreamer-1.0/', -) diff --git a/core/hailo/plugins/export/export_file/gsthailoexportfile.cpp b/core/hailo/plugins/export/export_file/gsthailoexportfile.cpp index 15740c3..d8ac0d4 100644 --- a/core/hailo/plugins/export/export_file/gsthailoexportfile.cpp +++ b/core/hailo/plugins/export/export_file/gsthailoexportfile.cpp @@ -187,6 +187,18 @@ gst_hailoexportfile_transform_ip(GstBaseTransform *trans, encoded_roi.AddMember("timestamp (ms)", rapidjson::Value(timenow), encoded_roi.GetAllocator()); encoded_roi.AddMember("buffer_offset", rapidjson::Value(hailoexportfile->buffer_offset), encoded_roi.GetAllocator()); + // Add the stream-id + std::string stream_id = hailo_roi->get_stream_id(); + + if (stream_id.length() == 0) + { + gchar * id = gst_pad_get_stream_id(trans->srcpad); + stream_id = std::string(reinterpret_cast(id)); + g_free(id); + } + + encoded_roi.AddMember("stream_id", stream_id, encoded_roi.GetAllocator() ); + // Open the file hailoexportfile->json_file = fopen(hailoexportfile->file_path, "rb+"); diff --git a/core/hailo/plugins/filter/gsthailofilter.cpp b/core/hailo/plugins/filter/gsthailofilter.cpp index 714a69a..9d4605a 100644 --- a/core/hailo/plugins/filter/gsthailofilter.cpp +++ b/core/hailo/plugins/filter/gsthailofilter.cpp @@ -292,6 +292,12 @@ void get_tensors_from_meta(GstBuffer *buffer, HailoROIPtr roi) { pmeta = reinterpret_cast(meta); (void)gst_buffer_map(pmeta->buffer, &info, GST_MAP_READWRITE); + // check if the buffer has tensor metadata + if (!gst_buffer_get_meta(pmeta->buffer, g_type_from_name(TENSOR_META_API_NAME))) + { + gst_buffer_unmap(pmeta->buffer, &info); + continue; + } const hailo_vstream_info_t vstream_info = reinterpret_cast(gst_buffer_get_meta(pmeta->buffer, g_type_from_name(TENSOR_META_API_NAME)))->info; roi->add_tensor(std::make_shared(reinterpret_cast(info.data), vstream_info)); gst_buffer_unmap(pmeta->buffer, &info); @@ -333,15 +339,15 @@ static GstFlowReturn gst_hailofilter_transform_ip(GstBaseTransform *trans, HailoROIPtr hailo_roi = get_hailo_main_roi(buffer, true); get_tensors_from_meta(buffer, hailo_roi); GstPad *srcpad = trans->srcpad; - + if (hailo_roi->get_stream_id().length() == 0) { - gchar * id = gst_pad_get_stream_id(srcpad); + gchar *id = gst_pad_get_stream_id(srcpad); std::string stream_id = std::string(reinterpret_cast(id)); g_free(id); hailo_roi->set_stream_id(stream_id); } - + // Call all functions. if (hailofilter->use_gst_buffer) { @@ -384,7 +390,7 @@ static GstFlowReturn gst_hailofilter_transform_ip(GstBaseTransform *trans, { remove_tensors(buffer, hailo_roi); } - + GST_DEBUG_OBJECT(hailofilter, "transform_ip"); return GST_FLOW_OK; } diff --git a/core/hailo/plugins/gray_scale/gsthailograytonv12.cpp b/core/hailo/plugins/gray_scale/gsthailograytonv12.cpp new file mode 100644 index 0000000..2b23c9c --- /dev/null +++ b/core/hailo/plugins/gray_scale/gsthailograytonv12.cpp @@ -0,0 +1,103 @@ +/** + * Copyright (c) 2021-2022 Hailo Technologies Ltd. All rights reserved. + * Distributed under the LGPL license (https://www.gnu.org/licenses/old-licenses/lgpl-2.1.txt) + **/ +#include "gsthailograytonv12.hpp" +#include "tensor_meta.hpp" +#include "gst_hailo_meta.hpp" +#include "hailo/hailort.h" +#include +#include +#include +#include +#include + +GST_DEBUG_CATEGORY_STATIC(gst_hailograytonv12_debug_category); +#define GST_CAT_DEFAULT gst_hailograytonv12_debug_category + +static GstFlowReturn gst_hailograytonv12_transform(GstBaseTransform *trans, GstBuffer *inbuf, GstBuffer *outbuf); +static GstFlowReturn gst_hailograytonv12_prepare_output_buffer(GstBaseTransform *trans, GstBuffer *inbuf, GstBuffer **outbuf); +GstCaps *gst_hailograytonv12_transform_caps(GstBaseTransform *trans, GstPadDirection direction, GstCaps *caps, GstCaps *filter); + +G_DEFINE_TYPE_WITH_CODE(GstHailograytonv12, gst_hailograytonv12, GST_TYPE_BASE_TRANSFORM, + GST_DEBUG_CATEGORY_INIT(gst_hailograytonv12_debug_category, "hailograytonv12", 0, + "debug category for hailograytonv12 element")); + +static void gst_hailograytonv12_class_init(GstHailograytonv12Class *klass) +{ + GstBaseTransformClass *base_transform_class = + GST_BASE_TRANSFORM_CLASS(klass); + + gst_element_class_add_pad_template(GST_ELEMENT_CLASS(klass), + gst_pad_template_new("src", GST_PAD_SRC, GST_PAD_ALWAYS, + gst_caps_from_string(GST_VIDEO_CAPS_MAKE("{ NV12 }")))); + gst_element_class_add_pad_template(GST_ELEMENT_CLASS(klass), + gst_pad_template_new("sink", GST_PAD_SINK, GST_PAD_ALWAYS, + gst_caps_from_string(GST_VIDEO_CAPS_MAKE("{ GRAY8 }")))); + gst_element_class_set_static_metadata(GST_ELEMENT_CLASS(klass), + "hailograytonv12 - postprocessing element", "Hailo/Tools", "Converts GRAY8 to NV12 by using the metadata that contains a pointer to the original NV12 buffer", + "hailo.ai "); + + base_transform_class->prepare_output_buffer = GST_DEBUG_FUNCPTR(gst_hailograytonv12_prepare_output_buffer); + base_transform_class->transform = GST_DEBUG_FUNCPTR(gst_hailograytonv12_transform); + base_transform_class->transform_caps = GST_DEBUG_FUNCPTR(gst_hailograytonv12_transform_caps); +} + +static void gst_hailograytonv12_init(GstHailograytonv12 *hailograytonv12) {} + +static GstFlowReturn gst_hailograytonv12_prepare_output_buffer(GstBaseTransform *trans, GstBuffer *inbuf, GstBuffer **outbuf) +{ + // here we want to get the meta data that has the pointer to the original NV12 buffer. we are looking for the GstParentBufferMeta that doesn't have the tensor meta + GstBuffer *original_buf = nullptr; + GstMeta *meta; + gpointer state = NULL; + while ((meta = gst_buffer_iterate_meta_filtered(inbuf, &state, GST_PARENT_BUFFER_META_API_TYPE))) + { + GstParentBufferMeta *parent_meta = reinterpret_cast(meta); + if (!gst_buffer_get_meta(parent_meta->buffer, g_type_from_name(TENSOR_META_API_NAME))) + { + original_buf = parent_meta->buffer; + gst_buffer_replace(outbuf, original_buf); + gst_buffer_remove_meta(inbuf, (GstMeta *)parent_meta); + break; + } + } + + return GST_FLOW_OK; +} + +static GstFlowReturn gst_hailograytonv12_transform(GstBaseTransform *trans, GstBuffer *inbuf, GstBuffer *outbuf) +{ + GstHailograytonv12 *hailograytonv12 = GST_HAILO_GRAY_TO_NV12(trans); + GST_DEBUG_OBJECT(hailograytonv12, "transform"); + gst_buffer_copy_into(outbuf, inbuf, GstBufferCopyFlags(GST_BUFFER_COPY_FLAGS | GST_BUFFER_COPY_TIMESTAMPS | GST_BUFFER_COPY_META), 0, -1); + + return GST_FLOW_OK; +} + +GstCaps *gst_hailograytonv12_transform_caps(GstBaseTransform *trans, GstPadDirection direction, GstCaps *caps, GstCaps *filter) +{ + /* + Removing the format field so that the caps can be intersected with the filter caps. + The same logic was implemented in gstbayer2rgb element and is useful for the case the input and output caps of an element don't share a common format. + */ + GstCaps *res_caps, *tmp_caps; + GstStructure *structure; + guint i, caps_size; + + res_caps = gst_caps_copy(caps); + caps_size = gst_caps_get_size(res_caps); + for (i = 0; i < caps_size; i++) + { + structure = gst_caps_get_structure(res_caps, i); + gst_structure_remove_field(structure, "format"); + } + + if (filter) + { + tmp_caps = res_caps; + res_caps = gst_caps_intersect_full(filter, tmp_caps, GST_CAPS_INTERSECT_FIRST); + gst_caps_unref(tmp_caps); + } + return res_caps; +} \ No newline at end of file diff --git a/core/hailo/plugins/gray_scale/gsthailograytonv12.hpp b/core/hailo/plugins/gray_scale/gsthailograytonv12.hpp new file mode 100644 index 0000000..7f07113 --- /dev/null +++ b/core/hailo/plugins/gray_scale/gsthailograytonv12.hpp @@ -0,0 +1,35 @@ +/** + * Copyright (c) 2021-2022 Hailo Technologies Ltd. All rights reserved. + * Distributed under the LGPL license (https://www.gnu.org/licenses/old-licenses/lgpl-2.1.txt) + **/ +#pragma once + +#include +#include +#include +#include + +G_BEGIN_DECLS + +#define GST_TYPE_HAILO_GRAY_TO_NV12 (gst_hailograytonv12_get_type()) +#define GST_HAILO_GRAY_TO_NV12(obj) (G_TYPE_CHECK_INSTANCE_CAST((obj), GST_TYPE_HAILO_GRAY_TO_NV12, GstHailograytonv12)) +#define GST_HAILO_GRAY_TO_NV12_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST((klass), GST_TYPE_HAILO_GRAY_TO_NV12, GstHailograytonv12Class)) +#define GST_IS_HAILO_GRAY_TO_NV12(obj) (G_TYPE_CHECK_INSTANCE_TYPE((obj), GST_TYPE_HAILO_GRAY_TO_NV12)) +#define GST_IS_HAILO_GRAY_TO_NV12_CLASS(obj) (G_TYPE_CHECK_CLASS_TYPE((klass), GST_TYPE_HAILO_GRAY_TO_NV12)) + +typedef struct _GstHailograytonv12 GstHailograytonv12; +typedef struct _GstHailograytonv12Class GstHailograytonv12Class; + +struct _GstHailograytonv12 +{ + GstBaseTransform base_hailograytonv12; +}; + +struct _GstHailograytonv12Class +{ + GstBaseTransformClass base_hailograytonv12_class; +}; + +GType gst_hailograytonv12_get_type(void); + +G_END_DECLS \ No newline at end of file diff --git a/core/hailo/plugins/gray_scale/gsthailonv12togray.cpp b/core/hailo/plugins/gray_scale/gsthailonv12togray.cpp new file mode 100644 index 0000000..27db7fa --- /dev/null +++ b/core/hailo/plugins/gray_scale/gsthailonv12togray.cpp @@ -0,0 +1,111 @@ +/** + * Copyright (c) 2021-2022 Hailo Technologies Ltd. All rights reserved. + * Distributed under the LGPL license (https://www.gnu.org/licenses/old-licenses/lgpl-2.1.txt) + **/ +#include "gsthailonv12togray.hpp" +#include "tensor_meta.hpp" +#include "gst_hailo_meta.hpp" +#include "hailo/hailort.h" +#include +#include +#include +#include +#include + +GST_DEBUG_CATEGORY_STATIC(gst_hailonv12togray_debug_category); +#define GST_CAT_DEFAULT gst_hailonv12togray_debug_category + +static GstFlowReturn gst_hailonv12togray_transform(GstBaseTransform *trans, GstBuffer *inbuf, GstBuffer *outbuf); +static GstFlowReturn gst_hailonv12togray_prepare_output_buffer(GstBaseTransform *trans, GstBuffer *inbuf, GstBuffer **outbuf); +GstCaps *gst_hailonv12togray_transform_caps(GstBaseTransform *trans, GstPadDirection direction, GstCaps *caps, GstCaps *filter); + +G_DEFINE_TYPE_WITH_CODE(GstHailonv12togray, gst_hailonv12togray, GST_TYPE_BASE_TRANSFORM, + GST_DEBUG_CATEGORY_INIT(gst_hailonv12togray_debug_category, "hailonv12togray", 0, + "debug category for hailonv12togray element")); + +static void gst_hailonv12togray_class_init(GstHailonv12tograyClass *klass) +{ + GstBaseTransformClass *base_transform_class = GST_BASE_TRANSFORM_CLASS(klass); + + gst_element_class_add_pad_template(GST_ELEMENT_CLASS(klass), + gst_pad_template_new("src", GST_PAD_SRC, GST_PAD_ALWAYS, + gst_caps_from_string(GST_VIDEO_CAPS_MAKE("{ GRAY8 }")))); + gst_element_class_add_pad_template(GST_ELEMENT_CLASS(klass), + gst_pad_template_new("sink", GST_PAD_SINK, GST_PAD_ALWAYS, + gst_caps_from_string(GST_VIDEO_CAPS_MAKE("{ NV12 }")))); + + gst_element_class_set_static_metadata(GST_ELEMENT_CLASS(klass), + "hailonv12togray - postprocessing element", "Hailo/Tools", "Converts NV12 to GRAY8 and keeps the original NV12 buffer as metadata", + "hailo.ai "); + + base_transform_class->prepare_output_buffer = GST_DEBUG_FUNCPTR(gst_hailonv12togray_prepare_output_buffer); + base_transform_class->transform = GST_DEBUG_FUNCPTR(gst_hailonv12togray_transform); + base_transform_class->transform_caps = GST_DEBUG_FUNCPTR(gst_hailonv12togray_transform_caps); +} + +static void gst_hailonv12togray_init(GstHailonv12togray *hailonv12togray){} + +static GstFlowReturn gst_hailonv12togray_prepare_output_buffer(GstBaseTransform *trans, GstBuffer *inbuf, GstBuffer **outbuf) +{ + GstVideoFrame frame; + GstPad *sinkpad = trans->sinkpad; + GstCaps *caps = gst_pad_get_current_caps(sinkpad); + GstVideoInfo info; + gst_video_info_from_caps(&info, caps); + gst_caps_unref(caps); + if (!(gst_video_frame_map(&frame, &info, inbuf, GstMapFlags(GST_MAP_READ)))) + { + throw std::runtime_error("Cannot map buffer to frame"); + } + + void *y_pointer = GST_VIDEO_FRAME_PLANE_DATA(&frame, 0); + + GstMapInfo map; + gst_buffer_map(inbuf, &map, GST_MAP_READ); + + *outbuf = gst_buffer_new_wrapped(y_pointer, map.size * 2 / 3); + gst_memory_ref(gst_buffer_get_all_memory(*outbuf)); + + gst_buffer_unmap(inbuf, &map); + gst_video_frame_unmap(&frame); + return GST_FLOW_OK; +} + +static GstFlowReturn gst_hailonv12togray_transform(GstBaseTransform *trans, GstBuffer *inbuf, GstBuffer *outbuf) +{ + /* + Just add the inbuf as a meta to the outbuf so the original buffer can be retrieved later. + */ + GstHailonv12togray *hailonv12togray = GST_HAILO_NV12_TO_GRAY(trans); + gst_buffer_add_parent_buffer_meta(outbuf, inbuf); // this refs the inbuf + + GST_DEBUG_OBJECT(hailonv12togray, "transform"); + return GST_FLOW_OK; +} + +GstCaps *gst_hailonv12togray_transform_caps(GstBaseTransform *trans, GstPadDirection direction, GstCaps *caps, GstCaps *filter) +{ + /* + Removing the format field so that the caps can be intersected with the filter caps. + The same logic was implemented in gstbayer2rgb element and is useful for the case the input and output caps of an element don't share a common format. + */ + GstCaps *res_caps, *tmp_caps; + GstStructure *structure; + guint i, caps_size; + + res_caps = gst_caps_copy(caps); + caps_size = gst_caps_get_size(res_caps); + for (i = 0; i < caps_size; i++) + { + structure = gst_caps_get_structure(res_caps, i); + gst_structure_remove_field(structure, "format"); + } + + if (filter) + { + tmp_caps = res_caps; + res_caps = gst_caps_intersect_full(filter, tmp_caps, GST_CAPS_INTERSECT_FIRST); + gst_caps_unref(tmp_caps); + } + return res_caps; +} \ No newline at end of file diff --git a/core/hailo/plugins/gray_scale/gsthailonv12togray.hpp b/core/hailo/plugins/gray_scale/gsthailonv12togray.hpp new file mode 100644 index 0000000..6f4df9e --- /dev/null +++ b/core/hailo/plugins/gray_scale/gsthailonv12togray.hpp @@ -0,0 +1,35 @@ +/** + * Copyright (c) 2021-2022 Hailo Technologies Ltd. All rights reserved. + * Distributed under the LGPL license (https://www.gnu.org/licenses/old-licenses/lgpl-2.1.txt) + **/ +#pragma once + +#include +#include +#include +#include + +G_BEGIN_DECLS + +#define GST_TYPE_HAILO_NV12_TO_GRAY (gst_hailonv12togray_get_type()) +#define GST_HAILO_NV12_TO_GRAY(obj) (G_TYPE_CHECK_INSTANCE_CAST((obj), GST_TYPE_HAILO_NV12_TO_GRAY, GstHailonv12togray)) +#define GST_HAILO_NV12_TO_GRAY_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST((klass), GST_TYPE_HAILO_NV12_TO_GRAY, GstHailonv12tograyClass)) +#define GST_IS_HAILO_NV12_TO_GRAY(obj) (G_TYPE_CHECK_INSTANCE_TYPE((obj), GST_TYPE_HAILO_NV12_TO_GRAY)) +#define GST_IS_HAILO_NV12_TO_GRAY_CLASS(obj) (G_TYPE_CHECK_CLASS_TYPE((klass), GST_TYPE_HAILO_NV12_TO_GRAY)) + +typedef struct _GstHailonv12togray GstHailonv12togray; +typedef struct _GstHailonv12tograyClass GstHailonv12tograyClass; + +struct _GstHailonv12togray +{ + GstBaseTransform base_hailonv12togray; +}; + +struct _GstHailonv12tograyClass +{ + GstBaseTransformClass base_hailonv12togray_class; +}; + +GType gst_hailonv12togray_get_type(void); + +G_END_DECLS \ No newline at end of file diff --git a/core/hailo/plugins/gsthailotools.cpp b/core/hailo/plugins/gsthailotools.cpp index 3982d6d..3364cde 100644 --- a/core/hailo/plugins/gsthailotools.cpp +++ b/core/hailo/plugins/gsthailotools.cpp @@ -25,7 +25,16 @@ #include "export/export_file/gsthailoexportfile.hpp" #include "export/export_zmq/gsthailoexportzmq.hpp" #include "import/import_zmq/gsthailoimportzmq.hpp" - +#include "gray_scale/gsthailonv12togray.hpp" +#include "gray_scale/gsthailograytonv12.hpp" +#ifdef HAILO15_TARGET +#include "dsp/upload/gsthailoupload.hpp" +#include "dsp/upload2/gsthailoupload2.hpp" +#include "dsp/gsthailovideoscale.hpp" +#include "dsp/osd/gsthailoosd.hpp" +#include "encoder/gsthailoh265enc.h" +#include "encoder/gsthailoh264enc.h" +#endif static gboolean plugin_init(GstPlugin *plugin) @@ -45,6 +54,16 @@ plugin_init(GstPlugin *plugin) gst_element_register(plugin, "hailoexportfile", GST_RANK_PRIMARY, GST_TYPE_HAILO_EXPORT_FILE); gst_element_register(plugin, "hailoexportzmq", GST_RANK_PRIMARY, GST_TYPE_HAILO_EXPORT_ZMQ); gst_element_register(plugin, "hailoimportzmq", GST_RANK_PRIMARY, GST_TYPE_HAILO_IMPORT_ZMQ); + gst_element_register(plugin, "hailonv12togray", GST_RANK_PRIMARY, GST_TYPE_HAILO_NV12_TO_GRAY); + gst_element_register(plugin, "hailograytonv12", GST_RANK_PRIMARY, GST_TYPE_HAILO_GRAY_TO_NV12); + #ifdef HAILO15_TARGET + gst_element_register(plugin, "hailoosd", GST_RANK_PRIMARY, GST_TYPE_HAILO_OSD); + gst_element_register(plugin, "hailoupload", GST_RANK_PRIMARY, GST_TYPE_HAILO_UPLOAD); + gst_element_register(plugin, "hailoupload2", GST_RANK_PRIMARY, GST_TYPE_HAILO_UPLOAD2); + gst_element_register(plugin, "hailovideoscale", GST_RANK_PRIMARY, GST_TYPE_HAILO_VIDEOSCALE); + gst_element_register(plugin, "hailoh265enc", GST_RANK_PRIMARY, GST_TYPE_HAILO_H265_ENC); + gst_element_register(plugin, "hailoh264enc", GST_RANK_PRIMARY, GST_TYPE_HAILO_H264_ENC); + #endif gst_hailo_meta_get_info(); gst_hailo_meta_api_get_type(); gst_hailo_cropping_meta_get_info(); diff --git a/core/hailo/plugins/meson.build b/core/hailo/plugins/meson.build index eb9a507..d954d7c 100644 --- a/core/hailo/plugins/meson.build +++ b/core/hailo/plugins/meson.build @@ -16,10 +16,13 @@ if meson.is_cross_build() sysroot_arg += '--sysroot=' + sysroot endif endif +hailo_link_args = [] plugin_sources = [ 'gsthailotools.cpp', 'filter/gsthailofilter.cpp', + 'gray_scale/gsthailonv12togray.cpp', + 'gray_scale/gsthailograytonv12.cpp', 'filter/gsthailocounter.cpp', 'muxer/gsthailomuxer.cpp', 'muxer/gsthailoroundrobin.cpp', @@ -44,28 +47,48 @@ dl_dep = meson.get_compiler('c').find_library('dl', required : false) # ZMQ dep zmq_dep = dependency('libzmq', method : 'pkg-config') - gsthailotools_deps = plugin_deps + [meta_dep, dl_dep, opencv_dep, tracker_dep, zmq_dep] +if get_option('target_platform') == 'hailo15' + dsp_dep = meson.get_compiler('c').find_library('libhailodsp', required: true, dirs: sysroot + '/usr/lib/') + plugin_sources += [ + 'dsp/gsthailodsp.c', + 'dsp/gsthailodspbufferpool.cpp', + 'dsp/gsthailodspbufferpoolutils.cpp', + 'dsp/osd/osd.cpp', + 'dsp/osd/gsthailoosd.cpp', + 'dsp/gsthailodspbasetransform.cpp', + 'dsp/upload/gsthailoupload.cpp', + 'dsp/upload2/gsthailoupload2.cpp', + 'dsp/gsthailovideoscale.cpp', + 'encoder/hailoencoder.c', + 'encoder/gopconfig.c', + 'encoder/gsthailoh265enc.c', + 'encoder/gsthailoh264enc.c' + ] + gsthailotools_deps += [dsp_dep] + common_args += ['-DHAILO15_TARGET'] + hailo_link_args += sysroot_arg + hailo_link_args += ['-lhantro_vc8000e', '-lm'] +endif + + ################################################ # Hailo Gstreamer Shared Library ################################################ shared_library('gsthailotools', plugin_sources, - cpp_args : hailo_lib_args + common_args + sysroot_arg, + cpp_args : hailo_lib_args + common_args + sysroot_arg+['-pthread'], + c_args : hailo_lib_args + common_args + sysroot_arg, + link_args: hailo_link_args, include_directories: [hailo_general_inc, xtensor_inc, rapidjson_inc, hailo_mat_inc], - dependencies : gsthailotools_deps, + dependencies : gsthailotools_deps + [dependency('threads')], gnu_symbol_visibility : 'default', version: meson.project_version(), install: true, install_dir: get_option('libdir') + '/gstreamer-1.0/', ) -if get_option('target_platform') == 'hailo15' - subdir('encoder') -endif - - if not get_option('include_python') subdir_done() endif diff --git a/core/hailo/plugins/muxer/gsthailomuxer.cpp b/core/hailo/plugins/muxer/gsthailomuxer.cpp index 174fa4b..0c22da2 100644 --- a/core/hailo/plugins/muxer/gsthailomuxer.cpp +++ b/core/hailo/plugins/muxer/gsthailomuxer.cpp @@ -7,7 +7,6 @@ * @title: hailomuxer * * Takes packets from various input sinks into one output source. - * TODO: Add description * */ #include "gsthailomuxer.hpp" diff --git a/core/hailo/plugins/muxer/gsthailoroundrobin.cpp b/core/hailo/plugins/muxer/gsthailoroundrobin.cpp index 8077908..1c58d91 100644 --- a/core/hailo/plugins/muxer/gsthailoroundrobin.cpp +++ b/core/hailo/plugins/muxer/gsthailoroundrobin.cpp @@ -29,9 +29,44 @@ GType gst_hailo_round_robin_pad_get_type(void); #define GST_HAILO_ROUND_ROBIN_PAD_CAST(obj) \ ((GstHailoRoundRobinPad *)(obj)) +#define DEFAULT_RETRIES_NUM 1 +#define MAX_RETRIES_NUM 20 +#define MIN_RETRIES_NUM 1 + +#define DEFAULT_QUEUE_SIZE 3 +#define MAX_QUEUE_SIZE 10 +#define MIN_QUEUE_SIZE 1 + +#define DEFAULT_WAIT_TIME 5 +#define MAX_WAIT_TIME 500 +#define MIN_WAIT_TIME 0 + +#define DEFAULT_PREROLL_FRAMES 3 +#define MAX_PREROLL_FRAMES 30 +#define MIN_PREROLL_FRAMES 1 + typedef struct _GstHailoRoundRobinPad GstHailoRoundRobinPad; typedef struct _GstHailoRoundRobinPadClass GstHailoRoundRobinPadClass; +#define GST_TYPE_HAILOROUNDROBIN_MODE (gst_hailoroundrobin_mode_get_type()) +static GType +gst_hailoroundrobin_mode_get_type(void) +{ + static GType hailoroundrobin_mode_type = 0; + static const GEnumValue hailoroundrobin_modes[] = { + {GST_HAILO_ROUND_ROBIN_MODE_FUNNEL_MODE, "Funnel Mode (push every buffer when it is ready)", "funnel-mode"}, + {GST_HAILO_ROUND_ROBIN_MODE_BLOCKING, "Blocking Mode (push every buffer when it is its pad's turn, and if the buffer is not ready, block until ready)", "blocking-mode"}, + {GST_HAILO_ROUND_ROBIN_MODE_NON_BLOCKING, "Non Blocking Mode (push every buffer when it is its pad's turn, and if the buffer is not ready, skip it)", "non-blocking-mode"}, + {0, NULL, NULL}, + }; + if (!hailoroundrobin_mode_type) + { + hailoroundrobin_mode_type = + g_enum_register_static("GstHailoRoundRobinMode", hailoroundrobin_modes); + } + return hailoroundrobin_mode_type; +} + struct _GstHailoRoundRobinPad { GstPad parent; @@ -47,7 +82,7 @@ static size_t get_pad_num(GstPad *pad) { gchar *name = gst_pad_get_name(pad); std::string pad_name_str(name); - std::string pad_num_str = pad_name_str.substr(pad_name_str.find("_") + 1); + std::string pad_num_str = pad_name_str.substr(pad_name_str.find_first_of("0123456789")); size_t pad_num = stoi(pad_num_str); g_free(name); return pad_num; @@ -60,7 +95,11 @@ G_DEFINE_TYPE(GstHailoRoundRobinPad, gst_hailo_round_robin_pad, GST_TYPE_PAD); enum { PROP_0, - PROP_FUNNEL_MODE, + PROP_MODE, + PROP_RETRIES_NUM, + PROP_QUEUE_SIZE, + PROP_WAIT_TIME, + PROP_PREROLL_FRAMES, }; static void @@ -92,9 +131,18 @@ G_DEFINE_TYPE_WITH_CODE(GstHailoRoundRobin, gst_hailo_round_robin, GST_TYPE_ELEM static GstStateChangeReturn gst_hailo_round_robin_change_state(GstElement *element, GstStateChange transition); -static GstFlowReturn gst_hailo_round_robin_sink_chain(GstPad *pad, - GstObject *parent, - GstBuffer *buf); +static GstFlowReturn gst_hailo_round_robin_sink_chain_preroll(GstPad *pad, + GstObject *parent, + GstBuffer *buf); +static GstFlowReturn gst_hailo_round_robin_sink_chain_funnel_mode(GstPad *pad, + GstObject *parent, + GstBuffer *buf); +static GstFlowReturn gst_hailo_round_robin_sink_chain_blocking_mode(GstPad *pad, + GstObject *parent, + GstBuffer *buf); +static GstFlowReturn gst_hailo_round_robin_sink_chain_non_blocking_mode(GstPad *pad, + GstObject *parent, + GstBuffer *buf); static gboolean gst_hailo_round_robin_sink_event(GstPad *pad, GstObject *parent, @@ -108,16 +156,37 @@ static GstPad *gst_hailo_round_robin_request_new_pad(GstElement *element, static void gst_hailo_round_robin_release_pad(GstElement *element, GstPad *pad); static void gst_hailo_round_robin_dispose(GObject *object); - static void gst_hailo_round_robin_set_property(GObject *object, guint prop_id, const GValue *value, GParamSpec *pspec) { switch (prop_id) { - case PROP_FUNNEL_MODE: - GST_HAILO_ROUND_ROBIN(object)->funnel_mode = g_value_get_boolean(value); + case PROP_MODE: + { + GST_HAILO_ROUND_ROBIN(object)->mode = (GstHailoRoundRobinMode)g_value_get_enum(value); + break; + } + case PROP_RETRIES_NUM: + { + GST_HAILO_ROUND_ROBIN(object)->retries_num = g_value_get_uint(value); + break; + } + case PROP_QUEUE_SIZE: + { + GST_HAILO_ROUND_ROBIN(object)->queue_size = g_value_get_uint(value); + break; + } + case PROP_WAIT_TIME: + { + GST_HAILO_ROUND_ROBIN(object)->wait_time = g_value_get_uint(value); break; + } + case PROP_PREROLL_FRAMES: + { + GST_HAILO_ROUND_ROBIN(object)->preroll_frames = g_value_get_uint(value); + break; + } default: G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec); break; @@ -130,8 +199,20 @@ gst_hailo_round_robin_get_property(GObject *object, guint prop_id, GValue *value { switch (prop_id) { - case PROP_FUNNEL_MODE: - g_value_set_boolean(value, GST_HAILO_ROUND_ROBIN(object)->funnel_mode); + case PROP_MODE: + g_value_set_enum(value, GST_HAILO_ROUND_ROBIN(object)->mode); + break; + case PROP_RETRIES_NUM: + g_value_set_uint(value, GST_HAILO_ROUND_ROBIN(object)->retries_num); + break; + case PROP_QUEUE_SIZE: + g_value_set_uint(value, GST_HAILO_ROUND_ROBIN(object)->queue_size); + break; + case PROP_WAIT_TIME: + g_value_set_uint(value, GST_HAILO_ROUND_ROBIN(object)->wait_time); + break; + case PROP_PREROLL_FRAMES: + g_value_set_uint(value, GST_HAILO_ROUND_ROBIN(object)->preroll_frames); break; default: G_OBJECT_WARN_INVALID_PROPERTY_ID(object, prop_id, pspec); @@ -139,7 +220,117 @@ gst_hailo_round_robin_get_property(GObject *object, guint prop_id, GValue *value } } +size_t get_current_pad_num(GstHailoRoundRobin *hailo_round_robin) +{ + std::shared_lock lock(*(hailo_round_robin->current_pad_mutex.get())); // reader lock + return hailo_round_robin->current_pad_num; +} +void set_current_pad_num(GstHailoRoundRobin *hailo_round_robin, size_t num) +{ + std::unique_lock lock(*(hailo_round_robin->current_pad_mutex.get())); // writer lock + hailo_round_robin->current_pad_num = num; +} + +int get_buffer_counter_value(GstHailoRoundRobin *hailo_round_robin) +{ + std::shared_lock lock(*(hailo_round_robin->counter_mutex.get())); // reader lock + return hailo_round_robin->preroll_buffer_counter; +} + +void set_buffer_counter_value(GstHailoRoundRobin *hailo_round_robin, size_t val) +{ + std::unique_lock lock(*(hailo_round_robin->counter_mutex.get())); // writer lock + hailo_round_robin->preroll_buffer_counter = val; +} + +void increment_buffer_counter_value(GstHailoRoundRobin *hailo_round_robin) +{ + std::unique_lock lock(*(hailo_round_robin->counter_mutex.get())); // writer lock + if (hailo_round_robin->preroll_buffer_counter != -1) + hailo_round_robin->preroll_buffer_counter++; +} + +static gboolean +forward_events(GstPad *pad, GstEvent **event, gpointer user_data) +{ + gboolean res = TRUE; + // This function pushes forward all events that are not EOS. + GstPad *srcpad = GST_PAD_CAST(user_data); + + if (GST_EVENT_TYPE(*event) != GST_EVENT_EOS) + { + res = gst_pad_push_event(srcpad, gst_event_ref(*event)); + } + return res; +} + +void schedule(GstHailoRoundRobin *hailo_round_robin) +{ + GstFlowReturn res; + while (!hailo_round_robin->stop_thread) + { + // iterate the pad queues and push the first buffer in each queueq + for (uint i = 0; i < hailo_round_robin->pad_queues.size(); i++) + { + if (hailo_round_robin->pad_queues[i] != NULL) + { + for (uint j = 0; j <= hailo_round_robin->retries_num; j++) + { + if (hailo_round_robin->pad_queues[i]->empty()) + { + if (j < hailo_round_robin->retries_num) // don't sleep on the last iteration + { + sleep((float)hailo_round_robin->wait_time / 1000.0); + } + } + else + { + set_current_pad_num(hailo_round_robin, i); + GstBuffer *buf = hailo_round_robin->pad_queues[i]->front(); + hailo_round_robin->pad_queues[i]->pop(); + if (hailo_round_robin->condition_vars_non_blocking[i] == NULL) + { + continue; + } + hailo_round_robin->condition_vars_non_blocking[i]->notify_one(); + + GstPad *pad = gst_element_get_static_pad(GST_ELEMENT_CAST(hailo_round_robin), ("sink_" + std::to_string(i)).c_str()); + + if (pad == NULL) // not found, will try different method - maybe the pad names are hailoroundrobinpad0, hailoroundrobinpad1, etc. + { + pad = gst_element_get_static_pad(GST_ELEMENT_CAST(hailo_round_robin), ("hailoroundrobinpad" + std::to_string(i)).c_str()); + if (pad == NULL) + { + GST_ERROR_OBJECT(hailo_round_robin, "Failed to get pad %d", i); + continue; + } + } + gchar *pad_name = gst_pad_get_name(pad); + gchar *stream_id = gst_pad_get_stream_id(pad); + + // Add stream meta to the buffer including the pad name and stream id. + gst_buffer_add_hailo_stream_meta(buf, pad_name, stream_id); + + // Forward sticky events. + gst_pad_sticky_events_foreach(pad, forward_events, hailo_round_robin->srcpad); + + // Push out_buffer forward. + res = gst_pad_push(hailo_round_robin->srcpad, buf); + if (res != GST_FLOW_OK) + { + GST_ERROR_OBJECT(hailo_round_robin, "Failed to push buffer to srcpad"); + } + gst_object_unref(pad); + g_free(pad_name); + g_free(stream_id); + break; // make sure it only pushes 1 buffer per pad + } + } + } + } + } +} static void gst_hailo_round_robin_class_init(GstHailoRoundRobinClass *klass) @@ -163,26 +354,74 @@ gst_hailo_round_robin_class_init(GstHailoRoundRobinClass *klass) gstelement_class->release_pad = GST_DEBUG_FUNCPTR(gst_hailo_round_robin_release_pad); gstelement_class->change_state = GST_DEBUG_FUNCPTR(gst_hailo_round_robin_change_state); - // install new property funnel_mode + // install new property mode + g_object_class_install_property(gobject_class, + PROP_MODE, + g_param_spec_enum("mode", + "mode", + "Select the mode of the element (0 - funnel mode (push every buffer when it is ready), 1 - blocking mode (push every buffer when it is its pad's turn, and if the buffer is not ready, block until ready), 2 - non blocking mode(push every buffer when it is its pad's turn, and if the buffer is not ready, skip it))", + GST_TYPE_HAILOROUNDROBIN_MODE, + (gint)GST_HAILO_ROUND_ROBIN_MODE_BLOCKING, + (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); + + g_object_class_install_property(gobject_class, + PROP_RETRIES_NUM, + g_param_spec_uint("retries-num", + "Retries num", + "Number of retries to get a buffer from a pad queue (only relevant when using non-blocking mode)", + MIN_RETRIES_NUM, + MAX_RETRIES_NUM, + DEFAULT_RETRIES_NUM, + (GParamFlags)(GST_PARAM_CONTROLLABLE | G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); g_object_class_install_property(gobject_class, - PROP_FUNNEL_MODE, - g_param_spec_boolean("funnel-mode", - "Funnel mode", - "Disables the round robin logic and pushes all buffers when available", - false, - (GParamFlags)(GST_PARAM_CONTROLLABLE | G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); + PROP_QUEUE_SIZE, + g_param_spec_uint("queue-size", + "Queue size", + "Size of the queue for each pad (only relevant when using non-blocking mode)", + MIN_QUEUE_SIZE, + MAX_QUEUE_SIZE, + DEFAULT_QUEUE_SIZE, + (GParamFlags)(GST_PARAM_CONTROLLABLE | G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); + g_object_class_install_property(gobject_class, + PROP_WAIT_TIME, + g_param_spec_uint("wait-time", + "Wait Time", + "Time in ms to wait between tries to get a buffer from a pad queue (only relevant when using non-blocking mode)", + MIN_WAIT_TIME, + MAX_WAIT_TIME, + DEFAULT_WAIT_TIME, + (GParamFlags)(GST_PARAM_CONTROLLABLE | G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); + g_object_class_install_property(gobject_class, + PROP_PREROLL_FRAMES, + g_param_spec_uint("preroll-frames", + "Preroll frames", + "Number of frames to operate in blocking mode before moving to non-blocking-mode (only relevant when using non-blocking mode)", + MIN_PREROLL_FRAMES, + MAX_PREROLL_FRAMES, + DEFAULT_PREROLL_FRAMES, + (GParamFlags)(G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS))); } static void gst_hailo_round_robin_init(GstHailoRoundRobin *hailo_round_robin) { hailo_round_robin->current_pad_num = 0; - hailo_round_robin->mutexes.clear(); - hailo_round_robin->condition_vars.clear(); + hailo_round_robin->mutexes_blocking.clear(); + hailo_round_robin->mutexes_non_blocking.clear(); + hailo_round_robin->pad_queues.clear(); + hailo_round_robin->condition_vars_blocking.clear(); + hailo_round_robin->condition_vars_non_blocking.clear(); + hailo_round_robin->preroll_buffer_counter = 0; + hailo_round_robin->retries_num = DEFAULT_RETRIES_NUM; + hailo_round_robin->queue_size = DEFAULT_QUEUE_SIZE; + hailo_round_robin->wait_time = DEFAULT_WAIT_TIME; + hailo_round_robin->preroll_frames = DEFAULT_PREROLL_FRAMES; + hailo_round_robin->stop_thread = false; hailo_round_robin->srcpad = gst_pad_new_from_static_template(&src_template, "src"); - hailo_round_robin->funnel_mode = false; + hailo_round_robin->mode = GST_HAILO_ROUND_ROBIN_MODE_BLOCKING; + hailo_round_robin->current_pad_mutex = std::make_unique(); + hailo_round_robin->counter_mutex = std::make_unique(); gst_pad_use_fixed_caps(hailo_round_robin->srcpad); - gst_element_add_pad(GST_ELEMENT(hailo_round_robin), hailo_round_robin->srcpad); } @@ -192,24 +431,32 @@ gst_hailo_round_robin_dispose(GObject *object) GstHailoRoundRobin *hailo_round_robin = GST_HAILO_ROUND_ROBIN_CAST(object); hailo_round_robin->srcpad = NULL; hailo_round_robin->current_pad_num = 0; - hailo_round_robin->mutexes.clear(); - hailo_round_robin->condition_vars.clear(); + hailo_round_robin->pad_queues.clear(); + hailo_round_robin->preroll_buffer_counter = 0; + hailo_round_robin->mutexes_blocking.clear(); + hailo_round_robin->mutexes_non_blocking.clear(); + hailo_round_robin->condition_vars_blocking.clear(); + hailo_round_robin->condition_vars_non_blocking.clear(); G_OBJECT_CLASS(parent_class)->dispose(object); } +void set_chain_to_all_pads(GstHailoRoundRobin *hailo_round_robin, GstPadChainFunction chain_function) +{ + GstElement *element = GST_ELEMENT_CAST(hailo_round_robin); + GList *item; + for (item = element->sinkpads; item != NULL; item = g_list_next(item)) + { + GstPad *sinkpad = GST_PAD_CAST(item->data); + gst_pad_set_chain_function(sinkpad, GST_DEBUG_FUNCPTR(chain_function)); + } +} + static GstPad * gst_hailo_round_robin_request_new_pad(GstElement *element, GstPadTemplate *templ, const gchar *name, const GstCaps *caps) { - /* - * This function is called when a new sink pad is requested. - * for each request we create a new pad and return it. - * we add a new mutex and a new conditon variable. - */ - GstPad *sinkpad; GstHailoRoundRobin *hailo_round_robin = GST_HAILO_ROUND_ROBIN_CAST(element); - GST_DEBUG_OBJECT(element, "requesting pad"); sinkpad = GST_PAD_CAST(g_object_new(GST_TYPE_HAILO_ROUND_ROBIN_PAD, @@ -217,16 +464,39 @@ gst_hailo_round_robin_request_new_pad(GstElement *element, GstPadTemplate *templ NULL)); // Set sink_chain and sink_event funtions for the new pad - gst_pad_set_chain_function(sinkpad, - GST_DEBUG_FUNCPTR(gst_hailo_round_robin_sink_chain)); + switch (hailo_round_robin->mode) + { + case GST_HAILO_ROUND_ROBIN_MODE_FUNNEL_MODE: + { + gst_pad_set_chain_function(sinkpad, GST_DEBUG_FUNCPTR(gst_hailo_round_robin_sink_chain_funnel_mode)); + break; + } + case GST_HAILO_ROUND_ROBIN_MODE_BLOCKING: + { + gst_pad_set_chain_function(sinkpad, GST_DEBUG_FUNCPTR(gst_hailo_round_robin_sink_chain_blocking_mode)); + break; + } + case GST_HAILO_ROUND_ROBIN_MODE_NON_BLOCKING: + { + // this will be changed to non-blocking mode after specific number of buffers + gst_pad_set_chain_function(sinkpad, GST_DEBUG_FUNCPTR(gst_hailo_round_robin_sink_chain_preroll)); + break; + } + } + gst_pad_set_event_function(sinkpad, GST_DEBUG_FUNCPTR(gst_hailo_round_robin_sink_event)); GST_OBJECT_FLAG_SET(sinkpad, GST_PAD_FLAG_PROXY_CAPS); GST_OBJECT_FLAG_SET(sinkpad, GST_PAD_FLAG_PROXY_ALLOCATION); - hailo_round_robin->mutexes.emplace_back(std::make_unique()); - hailo_round_robin->condition_vars.emplace_back(std::make_unique()); + hailo_round_robin->mutexes_blocking.emplace_back(std::make_unique()); + hailo_round_robin->mutexes_non_blocking.emplace_back(std::make_unique()); + + // create a new queue for the new pad + hailo_round_robin->pad_queues.emplace_back(std::make_unique>()); + hailo_round_robin->condition_vars_blocking.emplace_back(std::make_unique()); + hailo_round_robin->condition_vars_non_blocking.emplace_back(std::make_unique()); gst_pad_set_active(sinkpad, TRUE); @@ -247,8 +517,9 @@ gst_hailo_round_robin_all_sinkpads_eos_unlocked(GstHailoRoundRobin *hailo_round_ // When there are no sinkpads we return FALSE because there is no need to send EOS down the pipeline, pipeline is still active. if (element->numsinkpads == 0) + { goto done; - + } // Iterate over all the sinkpads of the element, and check if they are in eos state for (item = element->sinkpads; item != NULL; item = g_list_next(item)) { @@ -273,40 +544,116 @@ gst_hailo_round_robin_release_pad(GstElement *element, GstPad *pad) GST_DEBUG_OBJECT(hailo_round_robin, "releasing pad %s:%s", GST_DEBUG_PAD_NAME(pad)); gst_pad_set_active(pad, FALSE); - if (hailo_round_robin->condition_vars[get_pad_num(pad)] != NULL) - hailo_round_robin->condition_vars[get_pad_num(pad)]->notify_all(); + if (hailo_round_robin->condition_vars_blocking[get_pad_num(pad)] != NULL) + hailo_round_robin->condition_vars_blocking[get_pad_num(pad)]->notify_all(); + + if (hailo_round_robin->condition_vars_non_blocking[get_pad_num(pad)] != NULL) + hailo_round_robin->condition_vars_non_blocking[get_pad_num(pad)]->notify_all(); gst_element_remove_pad(GST_ELEMENT_CAST(hailo_round_robin), pad); } -static gboolean -forward_events(GstPad *pad, GstEvent **event, gpointer user_data) +static GstFlowReturn +gst_hailo_round_robin_sink_chain_preroll(GstPad *pad, GstObject *parent, GstBuffer *buf) { - // This function pushes forward all events that are not EOS. - GstPad *srcpad = GST_PAD_CAST(user_data); + GstFlowReturn ret = GST_FLOW_ERROR; + GstHailoRoundRobin *hailo_round_robin = GST_HAILO_ROUND_ROBIN_CAST(parent); - if (GST_EVENT_TYPE(*event) != GST_EVENT_EOS) - gst_pad_push_event(srcpad, gst_event_ref(*event)); + size_t pad_num = get_pad_num(pad); + if (hailo_round_robin->current_pad_num != pad_num) + { + // Wait for the turn of this pad. + if (hailo_round_robin->condition_vars_blocking[pad_num] != NULL) + { + std::unique_lock lock(*hailo_round_robin->mutexes_blocking[pad_num].get()); + hailo_round_robin->condition_vars_blocking[pad_num]->wait(lock); + } + else + { + gst_buffer_unref(buf); + return ret; + } - return TRUE; -} + // If condition variable got notified but it is not this pad's turn, raise an error. + if ((hailo_round_robin->current_pad_num != pad_num) && (get_buffer_counter_value(hailo_round_robin) != -1)) + { + GST_ERROR_OBJECT(hailo_round_robin, + "Tried to send buf on pad %zu while current pad should be %zu, dropping buffer!", + pad_num, hailo_round_robin->current_pad_num); + gst_buffer_unref(buf); + return ret; + } + } + + buf = gst_buffer_make_writable(buf); + + gchar *pad_name = gst_pad_get_name(pad); + gchar *stream_id = gst_pad_get_stream_id(pad); + // Add stream meta to the buffer including the pad name and stream id. + gst_buffer_add_hailo_stream_meta(buf, pad_name, stream_id); + + // Forward sticky events. + gst_pad_sticky_events_foreach(pad, forward_events, hailo_round_robin->srcpad); + + // Push out_buffer forward. + ret = gst_pad_push(hailo_round_robin->srcpad, buf); + + increment_buffer_counter_value(hailo_round_robin); // increment only if not equal to -1 + + if (get_buffer_counter_value(hailo_round_robin) == (int)(hailo_round_robin->mutexes_blocking.size() * hailo_round_robin->preroll_frames)) + { + set_buffer_counter_value(hailo_round_robin, -1); // don't use it anymore + std::thread schedule_thread(schedule, hailo_round_robin); + hailo_round_robin->thread = std::move(&schedule_thread); + hailo_round_robin->thread->detach(); + set_chain_to_all_pads(hailo_round_robin, gst_hailo_round_robin_sink_chain_non_blocking_mode); + hailo_round_robin->current_pad_num = 0; + + // notify all pads that are waiting in the blocking mode + for (size_t i = 0; i < hailo_round_robin->mutexes_blocking.size(); i++) + { + if (hailo_round_robin->condition_vars_blocking[i] != NULL) + hailo_round_robin->condition_vars_blocking[i]->notify_all(); + } + } + + g_free(pad_name); + g_free(stream_id); + + if (get_buffer_counter_value(hailo_round_robin) != -1) + { + hailo_round_robin->current_pad_num++; + if (hailo_round_robin->current_pad_num == hailo_round_robin->mutexes_blocking.size()) + hailo_round_robin->current_pad_num = 0; + + while (hailo_round_robin->condition_vars_blocking[hailo_round_robin->current_pad_num] == NULL) + { + hailo_round_robin->current_pad_num++; + if (hailo_round_robin->current_pad_num == hailo_round_robin->mutexes_blocking.size()) + hailo_round_robin->current_pad_num = 0; + } + hailo_round_robin->condition_vars_blocking[hailo_round_robin->current_pad_num]->notify_one(); + } + return ret; +} static GstFlowReturn -gst_hailo_round_robin_sink_chain(GstPad *pad, GstObject *parent, GstBuffer *buf) +gst_hailo_round_robin_sink_chain_blocking_mode(GstPad *pad, GstObject *parent, GstBuffer *buf) { GstFlowReturn ret = GST_FLOW_ERROR; GstHailoRoundRobin *hailo_round_robin = GST_HAILO_ROUND_ROBIN_CAST(parent); size_t pad_num = get_pad_num(pad); - if (!hailo_round_robin->funnel_mode && hailo_round_robin->current_pad_num != pad_num) + if (hailo_round_robin->current_pad_num != pad_num) { // Wait for the turn of this pad. - if (hailo_round_robin->condition_vars[pad_num] != NULL) + if (hailo_round_robin->condition_vars_blocking[pad_num] != NULL) { - std::unique_lock lock(*hailo_round_robin->mutexes[pad_num].get()); - hailo_round_robin->condition_vars[pad_num]->wait(lock); + std::unique_lock lock(*hailo_round_robin->mutexes_blocking[pad_num].get()); + hailo_round_robin->condition_vars_blocking[pad_num]->wait(lock); } - else { + else + { gst_buffer_unref(buf); return ret; } @@ -339,21 +686,62 @@ gst_hailo_round_robin_sink_chain(GstPad *pad, GstObject *parent, GstBuffer *buf) g_free(pad_name); g_free(stream_id); - if (!hailo_round_robin->funnel_mode) + hailo_round_robin->current_pad_num++; + if (hailo_round_robin->current_pad_num == hailo_round_robin->mutexes_blocking.size()) + hailo_round_robin->current_pad_num = 0; + + while (hailo_round_robin->condition_vars_blocking[hailo_round_robin->current_pad_num] == NULL) { - // Update current pad num. hailo_round_robin->current_pad_num++; - if (hailo_round_robin->current_pad_num == hailo_round_robin->mutexes.size()) + if (hailo_round_robin->current_pad_num == hailo_round_robin->mutexes_blocking.size()) hailo_round_robin->current_pad_num = 0; - - while(hailo_round_robin->condition_vars[hailo_round_robin->current_pad_num] == NULL){ - hailo_round_robin->current_pad_num++; - if (hailo_round_robin->current_pad_num == hailo_round_robin->mutexes.size()) - hailo_round_robin->current_pad_num = 0; - } - hailo_round_robin->condition_vars[hailo_round_robin->current_pad_num]->notify_one(); } + hailo_round_robin->condition_vars_blocking[hailo_round_robin->current_pad_num]->notify_one(); + return ret; +} + +static GstFlowReturn +gst_hailo_round_robin_sink_chain_funnel_mode(GstPad *pad, GstObject *parent, GstBuffer *buf) +{ + GstFlowReturn ret = GST_FLOW_ERROR; + GstHailoRoundRobin *hailo_round_robin = GST_HAILO_ROUND_ROBIN_CAST(parent); + buf = gst_buffer_make_writable(buf); + gchar *pad_name = gst_pad_get_name(pad); + gchar *stream_id = gst_pad_get_stream_id(pad); + + // Add stream meta to the buffer including the pad name and stream id. + gst_buffer_add_hailo_stream_meta(buf, pad_name, stream_id); + + // Forward sticky events. + gst_pad_sticky_events_foreach(pad, forward_events, hailo_round_robin->srcpad); + ret = gst_pad_push(hailo_round_robin->srcpad, buf); + g_free(pad_name); + g_free(stream_id); + return ret; +} + +static GstFlowReturn +gst_hailo_round_robin_sink_chain_non_blocking_mode(GstPad *pad, GstObject *parent, GstBuffer *buf) +{ + GstFlowReturn ret = GST_FLOW_ERROR; + GstHailoRoundRobin *hailo_round_robin = GST_HAILO_ROUND_ROBIN_CAST(parent); + size_t pad_num = get_pad_num(pad); + + if (hailo_round_robin->condition_vars_non_blocking[pad_num] != NULL) + { + std::unique_lock lock(*hailo_round_robin->mutexes_non_blocking[pad_num].get()); + hailo_round_robin->condition_vars_non_blocking[pad_num]->wait(lock, [hailo_round_robin, pad_num] + { return hailo_round_robin->pad_queues[pad_num]->size() < hailo_round_robin->queue_size; }); + + hailo_round_robin->pad_queues[pad_num]->push(buf); + } + else + { + gst_buffer_unref(buf); + return ret; + } + ret = GST_FLOW_OK; return ret; } @@ -377,16 +765,16 @@ gst_hailo_round_robin_sink_event(GstPad *pad, GstObject *parent, GstEvent *event if (GST_EVENT_TYPE(event) == GST_EVENT_EOS) { - // EOS event received, we need to forward it to all sink pads and set got_eos to TRUE GST_OBJECT_LOCK(hailo_round_robin); fpad->got_eos = TRUE; - - hailo_round_robin->condition_vars[pad_num]->notify_all(); - hailo_round_robin->condition_vars[pad_num] = NULL; + hailo_round_robin->condition_vars_blocking[pad_num]->notify_all(); + hailo_round_robin->condition_vars_blocking[pad_num] = NULL; + hailo_round_robin->condition_vars_non_blocking[pad_num]->notify_all(); + hailo_round_robin->condition_vars_non_blocking[pad_num] = NULL; forward = gst_hailo_round_robin_all_sinkpads_eos_unlocked(hailo_round_robin); GST_OBJECT_UNLOCK(hailo_round_robin); } - else if (pad_num != hailo_round_robin->current_pad_num) + else if (pad_num != get_current_pad_num(hailo_round_robin)) { forward = FALSE; } @@ -411,7 +799,7 @@ gst_hailo_round_robin_sink_event(GstPad *pad, GstObject *parent, GstEvent *event GST_PAD_STREAM_LOCK(hailo_round_robin->srcpad); } - if (pad_num != hailo_round_robin->current_pad_num) + if (pad_num != get_current_pad_num(hailo_round_robin)) { gst_pad_sticky_events_foreach(pad, forward_events, hailo_round_robin->srcpad); } @@ -436,14 +824,26 @@ gst_hailo_round_robin_change_state(GstElement *element, GstStateChange transitio switch (transition) { - case GST_STATE_CHANGE_PAUSED_TO_READY: + case GST_STATE_CHANGE_READY_TO_NULL: { - for (uint i=0; i < hailo_round_robin->condition_vars.size(); i++) - if (hailo_round_robin->condition_vars[i] != NULL) - hailo_round_robin->condition_vars[i]->notify_all(); - break; + if (hailo_round_robin->mode != GST_HAILO_ROUND_ROBIN_MODE_FUNNEL_MODE) + { + hailo_round_robin->stop_thread = true; + for (uint i = 0; i < hailo_round_robin->condition_vars_blocking.size(); i++) + { + if (hailo_round_robin->condition_vars_blocking[i] != NULL) + hailo_round_robin->condition_vars_blocking[i]->notify_all(); + } + for (uint i = 0; i < hailo_round_robin->condition_vars_non_blocking.size(); i++) + { + if (hailo_round_robin->condition_vars_non_blocking[i] != NULL) + hailo_round_robin->condition_vars_non_blocking[i]->notify_all(); + } + break; + } } - default: + + default: break; } ret = GST_ELEMENT_CLASS(parent_class)->change_state(element, transition); diff --git a/core/hailo/plugins/muxer/gsthailoroundrobin.hpp b/core/hailo/plugins/muxer/gsthailoroundrobin.hpp index 1d68086..b366f77 100644 --- a/core/hailo/plugins/muxer/gsthailoroundrobin.hpp +++ b/core/hailo/plugins/muxer/gsthailoroundrobin.hpp @@ -1,7 +1,7 @@ /** -* Copyright (c) 2021-2022 Hailo Technologies Ltd. All rights reserved. -* Distributed under the LGPL license (https://www.gnu.org/licenses/old-licenses/lgpl-2.1.txt) -**/ + * Copyright (c) 2021-2022 Hailo Technologies Ltd. All rights reserved. + * Distributed under the LGPL license (https://www.gnu.org/licenses/old-licenses/lgpl-2.1.txt) + **/ /* * GStreamer ROUND_ROBIN element * @@ -13,8 +13,12 @@ #include #include +#include #include +#include #include +#include +#include G_BEGIN_DECLS @@ -33,6 +37,13 @@ G_BEGIN_DECLS typedef struct _GstHailoRoundRobin GstHailoRoundRobin; typedef struct _GstHailoRoundRobinClass GstHailoRoundRobinClass; +typedef enum +{ + GST_HAILO_ROUND_ROBIN_MODE_FUNNEL_MODE = 0, + GST_HAILO_ROUND_ROBIN_MODE_BLOCKING = 1, + GST_HAILO_ROUND_ROBIN_MODE_NON_BLOCKING = 2, +} GstHailoRoundRobinMode; + /** * GstHailoRoundRobin: * @@ -41,12 +52,23 @@ typedef struct _GstHailoRoundRobinClass GstHailoRoundRobinClass; struct _GstHailoRoundRobin { GstElement element; - /*< private >*/ GstPad *srcpad; size_t current_pad_num; - gboolean funnel_mode; - std::vector> mutexes; - std::vector> condition_vars; + GstHailoRoundRobinMode mode; + uint retries_num; + uint queue_size; + uint wait_time; + uint preroll_frames; + std::vector> mutexes_blocking; + std::vector> mutexes_non_blocking; + std::unique_ptr counter_mutex; + int preroll_buffer_counter; + std::vector> condition_vars_blocking; + std::vector> condition_vars_non_blocking; + std::vector>> pad_queues; + std::thread *thread; + gboolean stop_thread; + std::unique_ptr current_pad_mutex; }; struct _GstHailoRoundRobinClass diff --git a/core/hailo/plugins/overlay/gsthailooverlay.cpp b/core/hailo/plugins/overlay/gsthailooverlay.cpp index f58c4c6..b6b083c 100644 --- a/core/hailo/plugins/overlay/gsthailooverlay.cpp +++ b/core/hailo/plugins/overlay/gsthailooverlay.cpp @@ -240,10 +240,12 @@ gst_hailooverlay_transform_ip(GstBaseTransform *trans, caps = gst_pad_get_current_caps(trans->sinkpad); GstVideoInfo *info = gst_video_info_new(); + gst_video_info_from_caps(info, caps); + GstMapInfo map; gst_buffer_map(buffer, &map, GST_MAP_READWRITE); - gst_video_info_from_caps(info, caps); - std::shared_ptr hmat = get_mat_by_format(buffer, info, &map, hailooverlay->line_thickness, hailooverlay->font_thickness); + + std::shared_ptr hmat = get_mat_by_format(buffer, info, hailooverlay->line_thickness, hailooverlay->font_thickness); gst_video_info_free(info); hailo_roi = get_hailo_main_roi(buffer, true); @@ -265,7 +267,7 @@ gst_hailooverlay_transform_ip(GstBaseTransform *trans, } status = GST_FLOW_OK; cleanup: - gst_buffer_unmap(buffer, &map); gst_caps_unref(caps); + gst_buffer_unmap(buffer, &map); return status; } diff --git a/core/hailo/python/gsthailo/video_frame.py b/core/hailo/python/gsthailo/video_frame.py index eb4749c..0a37940 100644 --- a/core/hailo/python/gsthailo/video_frame.py +++ b/core/hailo/python/gsthailo/video_frame.py @@ -29,7 +29,7 @@ def __init__(self, buffer: Gst.Buffer, caps: Gst.Caps, roi: hailo.HailoROI): python_version = platform.sys.version_info - if python_version.major is not 3: + if python_version.major != 3: raise RuntimeError(f"Python {python_version.major}.{python_version.minor} is not supported") if python_version.minor < 10: diff --git a/docs/TAPPAS_architecture.rst b/docs/TAPPAS_architecture.rst index a96e60a..8f97d79 100644 --- a/docs/TAPPAS_architecture.rst +++ b/docs/TAPPAS_architecture.rst @@ -53,3 +53,7 @@ Hailo GStreamer Elements * `HailoTracker `_ - HailoTracker is an element that applies Joint Detection and Embedding (JDE) model with Kalman filtering to track object instances. * `HailoRoundRobin `_ - HailoRoundRobin is an element that provides muxing functionality in roundrobin method. * `HailoStreamRouter `_ - HailoStreamRouter is an element that provides de-muxing functionality. +* `HailoOSD `_ - HailoOSD is an element specifically designed for Hailo-15 system, which enables the user to draw static text, images, and timestamps on GstBuffers. +* `HailoUpload `_ - HailoUpload is an element specifically designed for Hailo-15 system. It is responsible for transformation between memory spaces. +* `HailoH265Enc `_ - HailoH265Enc is an element which enables the user to encode a video in h265 coding format using the using Hailo-15 encoding hardware accelerator. +* `HailoH264Enc `_ - HailoH264Enc is an element which enables the user to encode a video in h264 coding format using the using Hailo-15 encoding hardware accelerator. \ No newline at end of file diff --git a/docs/elements/hailo_cropper.rst b/docs/elements/hailo_cropper.rst index 9b0dc70..c5899b0 100644 --- a/docs/elements/hailo_cropper.rst +++ b/docs/elements/hailo_cropper.rst @@ -71,3 +71,19 @@ Hierarchy internal-offset : Whether to use Gstreamer offset of internal offset. flags: readable, writable, controllable Boolean. Default: false + +Hailo-15 +-------- +HailoCropper can utilize the on-chip DSP (Digital Signal Processor), to perform resize and crop operations. + +The DSP is used by default on the Hailo-15 machine, and can be disabled by setting the ``use-dsp`` property. +When disabled OpenCV will be used (on the CPU) to perform the resize and crop operations. + +HailoCropper holds a buffer pool (GstBufferPool) that manages the buffers, used by the DSP. +The bufferpool is responsible for allocating and freeing the buffers when the reference count of a buffer reaches 0. +Buffer pool size (maximum buffers that can be allocated simultaneously in the pool) can be controlled using the ``pool-size`` property. + +It is recommended to make sure that a buffer is contiguous in memory, before being sent to the DSP. +Non-contiguous buffers will be copied to a new buffer, before being sent to the DSP. + +.. image:: ../resources/cropper_hailo15.png \ No newline at end of file diff --git a/docs/elements/hailo_gray_to_nv12.rst b/docs/elements/hailo_gray_to_nv12.rst new file mode 100644 index 0000000..a466c32 --- /dev/null +++ b/docs/elements/hailo_gray_to_nv12.rst @@ -0,0 +1,62 @@ + +Hailo Gray to NV12 +================== + +Overview +-------- + +Hailograytonv12 is an element that replaces the GRAY8 buffer with an NV12 buffer that is stored in the metadata of the GRAY8 buffer. It is a mirror to the hailonv12togray element.\ +It is worth noting that this process does not involve any buffer copies, enabling high performance.\ +This element is particularly useful for NV12 pipelines that want to use a GRAY8 formatted HEF.\ +In such cases, the hailonv12togray element should be placed before the hailonet element, and the hailograytonv12 element should be placed after the hailonet element, to retrieve the original NV12 buffer. + +Hierarchy +--------- + +.. code-block:: + + GObject + +----GInitiallyUnowned + +----GstObject + +----GstElement + +----GstBaseTransform + +----GstHailograytonv12 + + Pad Templates: + SRC template: 'src' + Availability: Always + Capabilities: + video/x-raw + format: { (string)NV12 } + width: [ 1, 2147483647 ] + height: [ 1, 2147483647 ] + framerate: [ 0/1, 2147483647/1 ] + + SINK template: 'sink' + Availability: Always + Capabilities: + video/x-raw + format: { (string)GRAY8 } + width: [ 1, 2147483647 ] + height: [ 1, 2147483647 ] + framerate: [ 0/1, 2147483647/1 ] + + Element has no clocking capabilities. + Element has no URI handling capabilities. + + Pads: + SINK: 'sink' + Pad Template: 'sink' + SRC: 'src' + Pad Template: 'src' + + Element Properties: + name : The name of the object + flags: readable, writable + String. Default: "hailograytonv12-0" + parent : The parent of the object + flags: readable, writable + Object of type "GstObject" + qos : Handle Quality-of-Service events + flags: readable, writable + Boolean. Default: false \ No newline at end of file diff --git a/docs/elements/hailo_h264enc.rst b/docs/elements/hailo_h264enc.rst new file mode 100644 index 0000000..522acc5 --- /dev/null +++ b/docs/elements/hailo_h264enc.rst @@ -0,0 +1,184 @@ +Hailo H264 Encoder +================== + +Overview +-------- + +| HailoH264Enc is an element which enables the user to encode a video in h265 coding format, using the **Hailo-15 encoding hardware accelerator**. +| **Currently only NV12 pipelines are supported by HailoH264Enc.** + + +Parameters +^^^^^^^^^^ + +| The hailoh265enc element provides a veriaty of properties that control the encoding performance and quality. +| Among those properties are the encoding profile, bitrate, encoding level, etc... +| When a property has the "changeable in NULL, READY, PAUSED or PLAYING state" flag, it means that it can be changed during the pipeline. + +.. image:: ../resources/dynamic_param.png + +| Changes to such properties during the pipeline will be updated at the end of the group of images (GOP). +| A change in bitrate during the pipeline will be applied at the end of the GOP, for instance. +| It is important to note that forcing a keyframe will reset the GOP and the new property will take effect at the end of the next GOP. + +Hierarchy +--------- + +.. code-block:: + + GObject + +----GInitiallyUnowned + +----GstObject + +----GstElement + +----GstVideoEncoder + +----GstHailoH264Enc + + Implemented Interfaces: + GstPreset + + Pad Templates: + SINK template: 'sink' + Availability: Always + Capabilities: + video/x-raw + format: NV12 + width: [ 16, 2147483647 ] + height: [ 16, 2147483647 ] + framerate: [ 0/1, 2147483647/1 ] + + SRC template: 'src' + Availability: Always + Capabilities: + video/x-h264 + stream-format: byte-stream + alignment: au + profile: { (string)base, (string)main, (string)high, (string)high-10 } + + Element has no clocking capabilities. + Element has no URI handling capabilities. + + Pads: + SINK: 'sink' + Pad Template: 'sink' + SRC: 'src' + Pad Template: 'src' + + Element Properties: + bframe-qp-delta : QP difference between BFrame QP and target QP, -1 = Disabled + flags: readable, writable, changeable in NULL, READY, PAUSED or PLAYING state + Integer. Range: -1 - 51 Default: 0 + bitrate : Target bitrate for rate control in bits/second + flags: readable, writable, changeable in NULL, READY, PAUSED or PLAYING state + Unsigned Integer. Range: 10000 - 40000000 Default: 40000000 + bitvar-range-b : Percent variations over average bits per frame for B frame + flags: readable, writable, changeable in NULL, READY, PAUSED or PLAYING state + Unsigned Integer. Range: 0 - 2000 Default: 2000 + bitvar-range-i : Percent variations over average bits per frame for I frame + flags: readable, writable, changeable in NULL, READY, PAUSED or PLAYING state + Unsigned Integer. Range: 0 - 2000 Default: 2000 + bitvar-range-p : Percent variations over average bits per frame for P frame + flags: readable, writable, changeable in NULL, READY, PAUSED or PLAYING state + Unsigned Integer. Range: 0 - 2000 Default: 2000 + block-rc-size : Size of block rate control + flags: readable, writable, changeable in NULL, READY, PAUSED or PLAYING state + Enum "GstHailoH264EncBlockRcSize" Default: 0, "64x64" + (0): 64x64 - 64X64 + (1): 32x32 - 32X32 + (2): 16x16 - 16X16 + compressor : Enable/Disable Embedded Compression + flags: readable, writable + Enum "GstHailoH264EncCompressor" Default: 3, "enable-both" + (0): disable - Disable Compression + (1): enable-luma - Only Enable Luma Compression + (2): enable-chroma - Only Enable Chroma Compression + (3): enable-both - Enable Both Luma and Chroma Compression + ctb-rc : Adaptive adjustment of QP inside frame + flags: readable, writable, changeable in NULL, READY, PAUSED or PLAYING state + Boolean. Default: false + fixed-intra-qp : Use fixed QP value for every intra frame in stream, 0 = disabled + flags: readable, writable, changeable in NULL, READY, PAUSED or PLAYING state + Unsigned Integer. Range: 0 - 51 Default: 0 + gop-length : GOP Length + flags: readable, writable, changeable in NULL, READY, PAUSED or PLAYING state + Unsigned Integer. Range: 1 - 300 Default: 30 + gop-size : GOP Size (1 - No B Frames) + flags: readable, writable, changeable in NULL, READY, PAUSED or PLAYING state + Unsigned Integer. Range: 1 - 8 Default: 1 + hrd : Restricts the instantaneous bitrate and total bit amount of every coded picture. + flags: readable, writable, changeable in NULL, READY, PAUSED or PLAYING state + Boolean. Default: false + hrd-cpb-size : Buffer size used by the HRD model in bits + flags: readable, writable, changeable in NULL, READY, PAUSED or PLAYING state + Unsigned Integer. Range: 10000 - 40000000 Default: 10000000 + intra-pic-rate : I frames interval (0 - Dynamic IDR Interval) + flags: readable, writable, changeable in NULL, READY, PAUSED or PLAYING state + Unsigned Integer. Range: 0 - 300 Default: 30 + intra-qp-delta : QP difference between target QP and intra frame QP + flags: readable, writable, changeable in NULL, READY, PAUSED or PLAYING state + Integer. Range: -51 - 51 Default: -5 + level : level to encoder + flags: readable, writable + Enum "GstHailoH264EncLevel" Default: 52, "level-5-2" + (10): level-1 - Level 1 + (99): level-1-b - Level 1b + (11): level-1-1 - Level 1.1 + (12): level-1-2 - Level 1.2 + (13): level-1-3 - Level 1.3 + (20): level-2 - Level 2 + (21): level-2-1 - Level 2.1 + (22): level-2-2 - Level 2.2 + (30): level-3 - Level 3 + (31): level-3-1 - Level 3.1 + (32): level-3-2 - Level 3.2 + (40): level-4 - Level 4 + (41): level-4-1 - Level 4.1 + (42): level-4-2 - Level 4.2 + (50): level-5 - Level 5 + (51): level-5-1 - Level 5.1 + (52): level-5-2 - Level 5.2 + min-force-key-unit-interval: Minimum interval between force-keyunit requests in nanoseconds + flags: readable, writable + Unsigned Integer64. Range: 0 - 18446744073709551615 Default: 0 + monitor-frames : How many frames will be monitored for moving bit rate. Default is using framerate + flags: readable, writable, changeable in NULL, READY, PAUSED or PLAYING state + Unsigned Integer. Range: 10 - 120 Default: 0 + name : The name of the object + flags: readable, writable, 0x2000 + String. Default: "hailoh264enc0" + parent : The parent of the object + flags: readable, writable, 0x2000 + Object of type "GstObject" + picture-rc : Adjust QP between pictures + flags: readable, writable, changeable in NULL, READY, PAUSED or PLAYING state + Boolean. Default: true + picture-skip : Allow rate control to skip pictures + flags: readable, writable, changeable in NULL, READY, PAUSED or PLAYING state + Boolean. Default: false + profile : profile to encoder + flags: readable, writable + Enum "GstHailoH264EncProfile" Default: 12, "high-10" + (9): base - Base Profile + (10): main - Main Profile + (11): high - High Profile + (12): high-10 - High 10 Profile + qos : Handle Quality-of-Service events from downstream + flags: readable, writable + Boolean. Default: false + qp-hdr : Initial target QP, -1 = Encoder calculates initial QP + flags: readable, writable, changeable in NULL, READY, PAUSED or PLAYING state + Integer. Range: -1 - 51 Default: 26 + qp-max : Maximum frame header QP + flags: readable, writable, changeable in NULL, READY, PAUSED or PLAYING state + Unsigned Integer. Range: 0 - 51 Default: 51 + qp-min : Minimum frame header QP + flags: readable, writable, changeable in NULL, READY, PAUSED or PLAYING state + Unsigned Integer. Range: 0 - 51 Default: 0 + roi-area1 : Specifying rectangular area of CTBs as Region Of Interest with lower QP, left:top:right:bottom:delta_qp format + flags: readable, writable, changeable in NULL, READY, PAUSED or PLAYING state + String. Default: null + roi-area2 : Specifying rectangular area of CTBs as Region Of Interest with lower QP, left:top:right:bottom:delta_qp format + flags: readable, writable, changeable in NULL, READY, PAUSED or PLAYING state + String. Default: null + tol-moving-bitrate : Percent tolerance over target bitrate of moving bit rate + flags: readable, writable, changeable in NULL, READY, PAUSED or PLAYING state + Unsigned Integer. Range: 0 - 2000 Default: 2000 \ No newline at end of file diff --git a/docs/elements/hailo_h265enc.rst b/docs/elements/hailo_h265enc.rst new file mode 100644 index 0000000..667b907 --- /dev/null +++ b/docs/elements/hailo_h265enc.rst @@ -0,0 +1,175 @@ +Hailo H265 Encoder +================== + +Overview +-------- + +| HailoH265Enc is an element which enables the user to encode a video in h265 coding format, using the **Hailo-15 encoding hardware accelerator**. +| **Currently only NV12 pipelines are supported by HailoH265Enc.** + + +Parameters +^^^^^^^^^^ + +| The hailoh265enc element provides a veriaty of properties that control the encoding performance and quality. +| Among those properties are the encoding profile, bitrate, encoding level, etc... +| When a property has the "changeable in NULL, READY, PAUSED or PLAYING state" flag, it means that it can be changed during the pipeline. + +.. image:: ../resources/dynamic_param.png + +| Changes to such properties during the pipeline will be updated at the end of the group of images (GOP). +| A change in bitrate during the pipeline will be applied at the end of the GOP, for instance. +| It is important to note that forcing a keyframe will reset the GOP and the new property will take effect at the end of the next GOP. + +Hierarchy +--------- + +.. code-block:: + + GObject + +----GInitiallyUnowned + +----GstObject + +----GstElement + +----GstVideoEncoder + +----GstHailoH265Enc + + Implemented Interfaces: + GstPreset + + Pad Templates: + SINK template: 'sink' + Availability: Always + Capabilities: + video/x-raw + format: NV12 + width: [ 16, 2147483647 ] + height: [ 16, 2147483647 ] + framerate: [ 0/1, 2147483647/1 ] + + SRC template: 'src' + Availability: Always + Capabilities: + video/x-h265 + stream-format: byte-stream + alignment: au + profile: { (string)main, (string)main-still-picture, (string)main-intra, (string)main-10, (string)main-10-intra } + + Element has no clocking capabilities. + Element has no URI handling capabilities. + + Pads: + SINK: 'sink' + Pad Template: 'sink' + SRC: 'src' + Pad Template: 'src' + + Element Properties: + bframe-qp-delta : QP difference between BFrame QP and target QP, -1 = Disabled + flags: readable, writable, changeable in NULL, READY, PAUSED or PLAYING state + Integer. Range: -1 - 51 Default: 0 + bitrate : Target bitrate for rate control in bits/second + flags: readable, writable, changeable in NULL, READY, PAUSED or PLAYING state + Unsigned Integer. Range: 10000 - 40000000 Default: 40000000 + bitvar-range-b : Percent variations over average bits per frame for B frame + flags: readable, writable, changeable in NULL, READY, PAUSED or PLAYING state + Unsigned Integer. Range: 0 - 2000 Default: 2000 + bitvar-range-i : Percent variations over average bits per frame for I frame + flags: readable, writable, changeable in NULL, READY, PAUSED or PLAYING state + Unsigned Integer. Range: 0 - 2000 Default: 2000 + bitvar-range-p : Percent variations over average bits per frame for P frame + flags: readable, writable, changeable in NULL, READY, PAUSED or PLAYING state + Unsigned Integer. Range: 0 - 2000 Default: 2000 + block-rc-size : Size of block rate control + flags: readable, writable, changeable in NULL, READY, PAUSED or PLAYING state + Enum "GstHailoH265EncBlockRcSize" Default: 0, "64x64" + (0): 64x64 - 64X64 + (1): 32x32 - 32X32 + (2): 16x16 - 16X16 + compressor : Enable/Disable Embedded Compression + flags: readable, writable + Enum "GstHailoH265EncCompressor" Default: 3, "enable-both" + (0): disable - Disable Compression + (1): enable-luma - Only Enable Luma Compression + (2): enable-chroma - Only Enable Chroma Compression + (3): enable-both - Enable Both Luma and Chroma Compression + ctb-rc : Adaptive adjustment of QP inside frame + flags: readable, writable, changeable in NULL, READY, PAUSED or PLAYING state + Boolean. Default: false + fixed-intra-qp : Use fixed QP value for every intra frame in stream, 0 = disabled + flags: readable, writable, changeable in NULL, READY, PAUSED or PLAYING state + Unsigned Integer. Range: 0 - 51 Default: 0 + gop-length : GOP Length + flags: readable, writable, changeable in NULL, READY, PAUSED or PLAYING state + Unsigned Integer. Range: 1 - 300 Default: 30 + gop-size : GOP Size (1 - No B Frames) + flags: readable, writable, changeable in NULL, READY, PAUSED or PLAYING state + Unsigned Integer. Range: 1 - 8 Default: 1 + hrd : Restricts the instantaneous bitrate and total bit amount of every coded picture. + flags: readable, writable, changeable in NULL, READY, PAUSED or PLAYING state + Boolean. Default: false + hrd-cpb-size : Buffer size used by the HRD model in bits + flags: readable, writable, changeable in NULL, READY, PAUSED or PLAYING state + Unsigned Integer. Range: 10000 - 40000000 Default: 10000000 + intra-pic-rate : I frames interval (0 - Dynamic IDR Interval) + flags: readable, writable, changeable in NULL, READY, PAUSED or PLAYING state + Unsigned Integer. Range: 0 - 300 Default: 30 + intra-qp-delta : QP difference between target QP and intra frame QP + flags: readable, writable, changeable in NULL, READY, PAUSED or PLAYING state + Integer. Range: -51 - 51 Default: -5 + level : level to encoder + flags: readable, writable + Enum "GstHailoH265EncLevel" Default: 153, "level-5-1" + (30): level-1 - Level 1 + (60): level-2 - Level 2 + (63): level-2-1 - Level 2.1 + (90): level-3 - Level 3 + (93): level-3-1 - Level 3.1 + (120): level-4 - Level 4 + (123): level-4-1 - Level 4.1 + (150): level-5 - Level 5 + (153): level-5-1 - Level 5.1 + min-force-key-unit-interval: Minimum interval between force-keyunit requests in nanoseconds + flags: readable, writable + Unsigned Integer64. Range: 0 - 18446744073709551615 Default: 0 + monitor-frames : How many frames will be monitored for moving bit rate. Default is using framerate + flags: readable, writable, changeable in NULL, READY, PAUSED or PLAYING state + Unsigned Integer. Range: 10 - 120 Default: 0 + name : The name of the object + flags: readable, writable, 0x2000 + String. Default: "hailoh265enc0" + parent : The parent of the object + flags: readable, writable, 0x2000 + Object of type "GstObject" + picture-rc : Adjust QP between pictures + flags: readable, writable, changeable in NULL, READY, PAUSED or PLAYING state + Boolean. Default: true + picture-skip : Allow rate control to skip pictures + flags: readable, writable, changeable in NULL, READY, PAUSED or PLAYING state + Boolean. Default: false + profile : profile to encoder + flags: readable, writable + Enum "GstHailoH265EncProfile" Default: 0, "main" + (0): main - Main Profile + (1): main-still-picture - Main Still Picture Profile + (2): main-10 - Main 10 Profile + qos : Handle Quality-of-Service events from downstream + flags: readable, writable + Boolean. Default: false + qp-hdr : Initial target QP, -1 = Encoder calculates initial QP + flags: readable, writable, changeable in NULL, READY, PAUSED or PLAYING state + Integer. Range: -1 - 51 Default: 26 + qp-max : Maximum frame header QP + flags: readable, writable, changeable in NULL, READY, PAUSED or PLAYING state + Unsigned Integer. Range: 0 - 51 Default: 51 + qp-min : Minimum frame header QP + flags: readable, writable, changeable in NULL, READY, PAUSED or PLAYING state + Unsigned Integer. Range: 0 - 51 Default: 0 + roi-area1 : Specifying rectangular area of CTBs as Region Of Interest with lower QP, left:top:right:bottom:delta_qp format + flags: readable, writable, changeable in NULL, READY, PAUSED or PLAYING state + String. Default: null + roi-area2 : Specifying rectangular area of CTBs as Region Of Interest with lower QP, left:top:right:bottom:delta_qp format + flags: readable, writable, changeable in NULL, READY, PAUSED or PLAYING state + String. Default: null + tol-moving-bitrate : Percent tolerance over target bitrate of moving bit rate + flags: readable, writable, changeable in NULL, READY, PAUSED or PLAYING state + Unsigned Integer. Range: 0 - 2000 Default: 2000 diff --git a/docs/elements/hailo_nv12_to_gray.rst b/docs/elements/hailo_nv12_to_gray.rst new file mode 100644 index 0000000..f38d465 --- /dev/null +++ b/docs/elements/hailo_nv12_to_gray.rst @@ -0,0 +1,62 @@ + +Hailo NV12 to Gray +================== + +Overview +-------- + +Hailonv12togray is an element that performs conversion of NV12 format to GRAY8 format, while preserving the initial NV12 buffer as the metadata of the resulting GRAY8 buffer. It is a mirror to the hailograytonv12 element.\ +It is worth noting that this process does not involve any buffer copies, enabling high performance.\ +This element is particularly useful for NV12 pipelines that want to use a GRAY8 formatted HEF.\ +In such cases, the hailonv12togray element should be placed before the hailonet element, and the hailograytonv12 element should be placed after the hailonet element, to retrieve the original NV12 buffer. + +Hierarchy +--------- + +.. code-block:: + + GObject + +----GInitiallyUnowned + +----GstObject + +----GstElement + +----GstBaseTransform + +----GstHailonv12togray + + Pad Templates: + SRC template: 'src' + Availability: Always + Capabilities: + video/x-raw + format: { (string)GRAY8 } + width: [ 1, 2147483647 ] + height: [ 1, 2147483647 ] + framerate: [ 0/1, 2147483647/1 ] + + SINK template: 'sink' + Availability: Always + Capabilities: + video/x-raw + format: { (string)NV12 } + width: [ 1, 2147483647 ] + height: [ 1, 2147483647 ] + framerate: [ 0/1, 2147483647/1 ] + + Element has no clocking capabilities. + Element has no URI handling capabilities. + + Pads: + SINK: 'sink' + Pad Template: 'sink' + SRC: 'src' + Pad Template: 'src' + + Element Properties: + name : The name of the object + flags: readable, writable + String. Default: "hailonv12togray0" + parent : The parent of the object + flags: readable, writable + Object of type "GstObject" + qos : Handle Quality-of-Service events + flags: readable, writable + Boolean. Default: false \ No newline at end of file diff --git a/docs/elements/hailo_osd.rst b/docs/elements/hailo_osd.rst new file mode 100644 index 0000000..274a92a --- /dev/null +++ b/docs/elements/hailo_osd.rst @@ -0,0 +1,114 @@ +Hailo On Screen Display +======================= + +Overview +-------- + +| HailoOSD is an element which enables the user to draw static text, images, and timestamps on GstBuffers **using the DSP provided in Hailo-15**. + By offloading the image blending to the DSP, high performance overlays can be acheived. + The DSP also supports transparent blending, allowing HailoOSD to draw image files with transparency. +| **Currently only NV12 pipelines are supported by HailoOSD.** + +.. image:: ../resources/osd_example.png + + +Parameters +^^^^^^^^^^ + +| The hailoosd element provides default behavior on what telemetry to draw. +| The user can customize the overlay contents via json through the **config-path** property. This property accepts the path to a json that follows the following schema: +| Any number of entries can be added to the **"staticImage"**, **"staticText"**, and **"dateTime"** arrays in the json. + +.. code-block:: + + { + "staticImage": [ + { + "image_path": "/path/to/image", + "width": 0.2, + "height": 0.13, + "x": 0.76, + "y": 0.05 + } + ], + "dateTime": [ + { + "font_size": 2, + "line_thickness": 3, + "rgb": [0, 0, 255], + "x": 0.1, + "y": 0.7 + } + ], + "staticText": [ + { + "label": "example text 1", + "font_size": 2, + "line_thickness": 3, + "rgb": [255, 0, 0], + "x": 0.7, + "y": 0.8 + }, + { + "label": "example text 2", + "font_size": 2, + "line_thickness": 3, + "x": 0.05, + "y": 0.1 + } + ] + } + +Hierarchy +--------- + +.. code-block:: + + GObject + +----GInitiallyUnowned + +----GstObject + +----GstElement + +----GstBaseTransform + +----GstHailoOsd + + Pad Templates: + SINK template: 'sink' + Availability: Always + Capabilities: + video/x-raw + format: { (string)NV12 } + width: [ 1, 2147483647 ] + height: [ 1, 2147483647 ] + framerate: [ 0/1, 2147483647/1 ] + + SRC template: 'src' + Availability: Always + Capabilities: + video/x-raw + format: { (string)NV12 } + width: [ 1, 2147483647 ] + height: [ 1, 2147483647 ] + framerate: [ 0/1, 2147483647/1 ] + + Element has no clocking capabilities. + Element has no URI handling capabilities. + + Pads: + SINK: 'sink' + Pad Template: 'sink' + SRC: 'src' + Pad Template: 'src' + + Element Properties: + config-path : json config file path + flags: readable, writable, changeable only in NULL or READY state + String. Default: "NULL" + name : The name of the object + flags: readable, writable, 0x2000 + String. Default: "hailoosd0" + parent : The parent of the object + flags: readable, writable, 0x2000 + Object of type "GstObject" + qos : Handle Quality-of-Service events + flags: readable, writable + Boolean. Default: false \ No newline at end of file diff --git a/docs/elements/hailo_roundrobin.rst b/docs/elements/hailo_roundrobin.rst index e4efb93..1790e77 100644 --- a/docs/elements/hailo_roundrobin.rst +++ b/docs/elements/hailo_roundrobin.rst @@ -10,6 +10,25 @@ It receives input from one or more sink pads and forwards them into a single src It also adds metadata to each buffer with the input pad name it was received on, The metadata's pupose is to be able to de-mux it easily later on by `hailostreamrouter `_ . +De-muxing by streamiddemux is not supported with this element. + +It can work in 3 modes: +* Funnel mode - push every buffer when it is ready no matter which pad it came from. +* Blocking mode - push every buffer when it is its pad's turn, and if the buffer is not ready, block until ready. This is the default mode. +* Non Blocking mode - push every buffer when it is its pad's turn, and if the buffer is not ready, skip it. This mode is useful when the video sources are not stable and may stop sending buffers for a while. In this case, the pipeline should not be blocked and should continue to process the other streams. + +When using non-blocking mode, the element maintains a queue for sink pad that holds pointers to buffers. +When a buffer is pushed to a sink pad, it is added to the queue. +When the src pad wants to push a buffer, the element tries to get a buffer from the queue of the pad that is next in line. +If the queue is empty, the element retries to get a buffer from the queue for a number of times (retries-num property). +If the queue is still empty, the element skips the pad and tries to get a buffer from the next pad in line. +The size of the queue and the number of retries can be configured by the properties: +* queue-size - Size of the queue for each pad. +* retries-num - Number of retries to get a buffer from a pad queue. + +When using non-blocking mode, Compositor element is not supported, since it requires all the streams to be synchronized. + + Example ------- @@ -23,7 +42,7 @@ and then de-muxing them into 2 separate pipelines - one for person attributes an for ((n = 0; n < 4; n++)); do filesrc location=video_$n ! decodebin ! roundrobin.sink_$n - hailoroundrobin name=roundrobin funnel-mode=false ! + hailoroundrobin name=roundrobin mode=1 ! Hierarchy @@ -56,12 +75,25 @@ Pads: Pad Template: 'src' Element Properties: + mode : Select the mode of the element (0 - funnel mode (push every buffer when it is ready), 1 - blocking mode (push every buf +fer when it is its pad's turn, and if the buffer is not ready, block until ready), 2 - non blocking mode(push every buffer when it is its pad's + turn, and if the buffer is not ready, skip it)) + flags: readable, writable + Enum "GstHailoRoundRobinMode" Default: 1, "blocking-mode" + (0): funnel-mode - Funnel Mode (push every buffer when it is ready) + (1): blocking-mode - Blocking Mode (push every buffer when it is its pad's turn, and if the buffer is not ready, +block until ready) + (2): non-blocking-mode - Non Blocking Mode (push every buffer when it is its pad's turn, and if the buffer is not re +ady, skip it) name : The name of the object flags: readable, writable String. Default: "hailoroundrobin0" parent : The parent of the object flags: readable, writable Object of type "GstObject" - funnel-mode : Disables the round robin logic and pushes all buffers when available + queue-size : Size of the queue for each pad (only relevant when using non-blocking mode) + flags: readable, writable, controllable + Unsigned Integer. Range: 1 - 10 Default: 3 + retries-num : Number of retries to get a buffer from a pad queue (only relevant when using non-blocking mode) flags: readable, writable, controllable - Boolean. Default: false + Unsigned Integer. Range: 1 - 20 Default: 3 diff --git a/docs/elements/hailoupload.rst b/docs/elements/hailoupload.rst new file mode 100644 index 0000000..aeb91a0 --- /dev/null +++ b/docs/elements/hailoupload.rst @@ -0,0 +1,86 @@ + +Hailo Upload +============ + +Overview +-------- + +``HailoUpload`` is an element specifically designed for Hailo-15 system. +It is responsible for transformation between memory spaces. + +The element can be described by the following steps: + +- Request an address (pointer) to a physically contiguous buffer from the kernel at a specifc size (determined by the format and resolution of the frame). +- Perform a memory copy to the target memory. +- Hold the address in a buffer pool - to free the buffer when its reference count is zero. +- Ensure that the output buffer is physically contiguous in memory and also virtually contiguous. + +DSP and Encoder require physically contiguous buffers. +It is recommended to make sure that a buffer is contiguous in memory, +before being sent to an element that uses DSP- it is essential to avoid memcopies. (Like in `HailoCropper `_). +In Encoder - contiguous memory is mandatory. (`HailoH265Enc `_ element). + +As an example, file source allocates non-contiguous buffers, meaning a hailoupload is required to use ``hailoh265enc``: + +.. code-block:: + + gst-launch-1.0 filesrc location=video.raw name=src_0 ! rawvideoparse format=rgb width=1920 height=1080 ! hailoupload ! hailoh265enc ! fakesink + +.. image:: ../resources/hailoupload.png + +Another example is to use hailoupload before hailonet, to make sure that the input buffer is virtually contiguous in memory: + +.. code-block:: + + gst-launch-1.0 v4l2src device=/dev/video0 io-mode=mmap ! video/x-raw,format=NV12,width=1920,height=1080 ! hailoupload ! hailonet hef-path=yolov5m_wo_spp_60p_nv12.hef ! fakesink + + +``HailoUpload`` inherits from ``HailoDspBaseTransform`` which is responsible for managing the allocation queries and buffer pool. + + +Hierarchy +--------- + + .. code-block:: + + GObject │ + +----GInitiallyUnowned │ + +----GstObject │ + +----GstElement │ + +----GstBaseTransform │ + +----GstHailoDspBaseTransform │ + +----GstHailoUpload │ + │ + Pad Templates: │ + SINK template: 'sink' │ + Availability: Always │ + Capabilities: │ + ANY │ + │ + SRC template: 'src' │ + Availability: Always │ + Capabilities: │ + ANY │ + │ + Element has no clocking capabilities. │ + Element has no URI handling capabilities. │ + │ + Pads: │ + SINK: 'sink' │ + Pad Template: 'sink' │ + SRC: 'src' │ + Pad Template: 'src' │ + │ + Element Properties: │ + name : The name of the object │ + flags: readable, writable, 0x2000 │ + String. Default: "hailoupload0" │ + parent : The parent of the object │ + flags: readable, writable, 0x2000 │ + Object of type "GstObject" │ + pool-size : Size of the pool of buffers to use for cropping. Default 10 │ + flags: readable, writable, changeable only in NULL or READY state │ + Unsigned Integer. Range: 1 - 2147483647 Default: 10 │ + qos : Handle Quality-of-Service events │ + flags: readable, writable │ + Boolean. Default: false diff --git a/docs/installation/manual-install.rst b/docs/installation/manual-install.rst index b8d0edb..ba725e3 100644 --- a/docs/installation/manual-install.rst +++ b/docs/installation/manual-install.rst @@ -2,8 +2,8 @@ Manual installation =================== -Manual installing TAPPAS requires preparations, our recommended method is to begin with ``Hailo SW Suite`` or ``Pre-built Docker image``. -In this guide we instruct how to install our required components manually. +The manual installation of TAPPAS requires preparations, our recommended method is to begin with ``Hailo SW Suite`` or ``Pre-built Docker image``. +In this guide we instruct how to install the required components manually. .. note:: Only Ubuntu 20.04 and 22.04 are supported @@ -77,7 +77,7 @@ To install the above packages, run the following command: sudo apt-get install -y rsync ffmpeg x11-utils python3-dev python3-pip python3-setuptools python3-virtualenv python-gi-dev libgirepository1.0-dev gcc-9 g++-9 cmake git libzmq3-dev -The following packages are required as well, and see their installation instructions below: +The following packages are required as well, and their installation instructions can be viewed from the links below: * `OpenCV installation`_. * `GStreamer installation`_. @@ -151,7 +151,7 @@ Please refer to: `PyGobject offical installation guide `_ -On Raspberry Pi, run: +Raspberry Pi 4 has its own set of example applications, so the installation command on this platform is slightly different: .. code-block:: sh ./install.sh --skip-hailort --target-platform rpi -And then, `Get back to Raspberry Pi section <./raspberry-pi-install.rst>`_ +And then, `Go back to Raspberry Pi section <./raspberry-pi-install.rst>`_ + +On Rockchip, run: + +.. code-block:: sh + + ./install.sh --skip-hailort --target-platform rockchip + +And then, `Go back to Rockchip section <./rockchip.rst>`_ On Rockchip, run: @@ -191,7 +199,7 @@ Remove old ``libgsthailotools.so`` rm /usr/lib/$(uname -m)-linux-gnu/gstreamer-1.0/libgsthailotools.so -And then, `TAPPAS installation section`_ +and then, `TAPPAS installation section`_ Troubleshooting --------------- @@ -199,13 +207,13 @@ Troubleshooting Cannot allocate memory in static TLS block ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -In some sceneraios (especially aarch64), you might face the following error: +In some sceneraios (especially aarch64), you might experience the following: .. code-block:: sh (gst-plugin-scanner:15): GStreamer-WARNING **: 13:58:20.557: Failed to load plugin '/usr/lib/aarch64-linux-gnu/gstreamer-1.0/libgstlibav.so': /lib/aarch64-linux-gnu/libgomp.so.1: cannot allocate memory in static TLS block -The solution is to export an enviroment variable: +The solution is to export an environment variable: .. code-block:: sh diff --git a/docs/installation/yocto.rst b/docs/installation/yocto.rst index 4795929..f2f9e35 100644 --- a/docs/installation/yocto.rst +++ b/docs/installation/yocto.rst @@ -19,6 +19,8 @@ The layers are stored in `Meta-Hailo Github + + + + + + + + + + + + + + + + + + + + + + + + Page-20 + + + + Rectangle.2 + + + + + + + L Shape.53 + + + + + + + + Rectangle.1 + + + + + + + Rectangle.4 + + + + + + + Rectangle + + + + + + + Rectangle.7 + Hailo Dataflow Compiler (SDK) + + + + + + + Hailo Dataflow Compiler (SDK) + + Rectangle.10 + Hailo Driver + + + + + + + Hailo Driver + + Rectangle.11 + + + + + + + Rectangle.13 + Hailo software components + + + + + + + Hailo software components + + Rectangle.14 + Other software components + + + + + + + Other software components + + Rectangle.16 + NN Core (part of Hailo Vision Processor or AI Accelerator) + + + + + + + NN Core(part of Hailo Vision Processor or AI Accelerator) + + Rectangle.18 + + + + + + + Rectangle.23 + Model Parser + + + + + + + Model Parser + + Rectangle.24 + Model Optimizer + + + + + + + Model Optimizer + + Rectangle.25 + Resource Allocator + + + + + + + Resource Allocator + + Rectangle.19 + Compiler + + + + + + + Compiler + + Rectangle.20 + Profiler + + + + + + + Profiler + + Rectangle.21 + Emulator + + + + + + + Emulator + + Rectangle.27 + CLI tools + + + + + + + CLI tools + + Rectangle.29 + Python API + + + + + + + Python API + + Rectangle.33 + C/C++ API and Library + + + + + + + C/C++ API and Library + + Rectangle.37 + OS IP Stack + + + + + + + OS IP Stack + + Sheet.38 + PCIe + + + + PCIe + + Sheet.39 + Ethernet + + + + Ethernet + + Rectangle.49 + Hailo Model Zoo + + + + + + + Hailo Model Zoo + + Rectangle.42 + User Models + + + + + + + UserModels + + L Shape + + + + + + + + Rectangle.44 + Pipeline Elements + + + + + + + Pipeline Elements + + Rectangle.46 + Application Examples + + + + + + + Application Examples + + Rectangle.47 + TAPPAS + + + + + + + TAPPAS + + L Shape.48 + + + + + + + + Rectangle.49 + User Applications + + + + + + + UserApplications + + Rectangle.54 + HailoRT + + + + + + + HailoRT + + Rectangle.56 + Integration Tool + + + + + + + Integration Tool + + Rectangle.64 + + + + + + + Rectangle.32 + Tools + + + + + + + TOOLS + + Rectangle.67 + Runtime Frameworks Plugins + + + + + + + RUNTIME FRAMEWORKS PLUGINS + + Sheet.70 + + + + Sheet.71 + + + + Rectangle.75 + Machine Learning Frameworks + + + + + + + Machine Learning Frameworks + + Rectangle.50 + Pre-trained Models + + + + + + + Pre-trained Models + + Rectangle.52 + Re-training Env + + + + + + + Re-training Env + + Rectangle.51 + Build & Eval Tools + + + + + + + Build & Eval Tools + + Rectangle.93 + + + + + + + Rectangle.94 + + + + + + + Rectangle.95 + In preview + + + + + + + In preview + + Sheet.105 + + Sheet.106 + + Sheet.107 + + Sheet.108 + + Sheet.109 + + + + Sheet.110 + + + + Sheet.111 + + + + Sheet.112 + + + + + Sheet.113 + + Sheet.114 + + + + Sheet.115 + + + + Sheet.116 + + + + Sheet.117 + + + + Sheet.118 + + + + Sheet.119 + + + + Sheet.120 + + + + + + Sheet.121 + + Sheet.122 + + Sheet.123 + + + + Sheet.124 + + + + + Sheet.125 + + + + + Sheet.126 + + Sheet.127 + + + + Sheet.128 + + + + + Sheet.129 + + + + + Sheet.130 + + Sheet.131 + + + + Sheet.132 + + + + + Sheet.133 + + + + + Sheet.134 + + Sheet.135 + + + + Sheet.136 + + + + + Sheet.137 + + + + + Sheet.138 + + Sheet.139 + + + + Sheet.140 + + + + + Sheet.141 + + + + + Sheet.142 + + + + + + + Rectangle.63 + pyHailoRT (Python API) + + + + + + + pyHailoRT (Python API) + + Rectangle.55 + CLI + + + + + + + CLI + + Rectangle.160 + + + + + + + Sheet.167 + + Sheet.104 + + + + + + Sheet.161 + + + + + + Sheet.162 + + + + + + Sheet.163 + + + + + + + + + Sheet.164 + + + + + + + Sheet.168 + + + + Sheet.169 + + + + Rectangle.41 + + + + + + + path3 + + + + path5 + + + + path7 + + + + path9 + + + + path11 + + + + path13 + + + + path15 + + + + path17 + + + + path19 + + + + path21 + + + + path23 + + + + path25 + + + + Rectangle.187 + Runtime Environment + + + + + + + Runtime Environment + + Rectangle.188 + Model Build Environment + + + + + + + Model Build Environment + + Sheet.1000 + + + + Sheet.1001 + Integrated + + + + Integrated + + diff --git a/resources/TAPPAS.png b/resources/TAPPAS.png deleted file mode 100644 index ff54c14..0000000 Binary files a/resources/TAPPAS.png and /dev/null differ diff --git a/resources/github_TAPPAS.jpg b/resources/github_TAPPAS.jpg new file mode 100644 index 0000000..04a9d1a Binary files /dev/null and b/resources/github_TAPPAS.jpg differ diff --git a/scripts/vaapi/install_vaapi.sh b/scripts/vaapi/install_vaapi.sh index 84c621c..a5219f2 100755 --- a/scripts/vaapi/install_vaapi.sh +++ b/scripts/vaapi/install_vaapi.sh @@ -16,6 +16,7 @@ no_cache=false num_cores_to_use=$(($(nproc) / 2)) log_filename="tappas_vaapi.log" skip_vainfo=false +skip_clean=false function log() { message=$1 @@ -63,6 +64,7 @@ function print_usage() { echo " --log-name Specify output log file name (default is $log_filename) - Pass $NO_LOG string to disable" echo " --compile-num-of-cores Number of cpu cores to compile with (more cores makes the compilation process faster, but may cause 'out of swap memory' issue on weak machines)" echo " --skip-vainfo Skip VA-Info check" + echo " --skip-clean Skip the cleaning of build artifacts" exit 1 } @@ -80,6 +82,8 @@ function parse_args() { shift elif [ "$1" == "--skip-vainfo" ]; then skip_vainfo=true + elif [ "$1" == "--skip-clean" ]; then + skip_clean=true else echo "Unknown parameters, exiting" print_usage @@ -159,7 +163,11 @@ function gmmlib_install() { cmake -DCMAKE_BUILD_TYPE=Release CMAKE_C_COMPILER=/usr/bin/gcc-9 -DCMAKE_CXX_COMPILER=/usr/bin/g++-9 ../ cmake --build . --config Release -j $num_cores_to_use sudo cmake --install . + popd + if [ "$skip_clean" = false ]; then + rm -rf build + fi popd } @@ -170,6 +178,11 @@ function libva_install() { meson libva_build --buildtype release -Dwith_x11=yes ninja -C libva_build sudo env "PATH=$PATH" ninja -C libva_build install + + if [ "$skip_clean" = false ]; then + rm -rf libva_build + fi + popd } @@ -180,6 +193,11 @@ function libva_utils_install() { meson libva_build --buildtype release ninja -C libva_build sudo env "PATH=$PATH" ninja -C libva_build install + + if [ "$skip_clean" = false ]; then + rm -rf libva_build + fi + popd } @@ -193,7 +211,11 @@ function media_driver_install() { cmake -DCMAKE_BUILD_TYPE=Release CMAKE_C_COMPILER=/usr/bin/gcc-9 -DCMAKE_CXX_COMPILER=/usr/bin/g++-9 ../ make -j $num_cores_to_use sudo make install + popd + if [ "$skip_clean" = false ]; then + rm -rf build + fi popd } @@ -240,12 +262,12 @@ function main() { trap 'err_report $LINENO' ERR pushd ${TAPPAS_WORKSPACE}/sources install_libva_essentials + install_gstreamer_vaapi if [ "$skip_vainfo" = false ]; then check_va_info fi - install_gstreamer_vaapi log_success "Installed VA-API successfully" } diff --git a/tools/cross_compiler/cross_compile_gsthailotools.py b/tools/cross_compiler/cross_compile_gsthailotools.py index 1c916b1..d6f191a 100755 --- a/tools/cross_compiler/cross_compile_gsthailotools.py +++ b/tools/cross_compiler/cross_compile_gsthailotools.py @@ -10,6 +10,7 @@ from common import working_directory, ShellRunner, install_compilers_apt_packages, extract_and_install_toolchain, Arch, Target POSSIBLE_BUILD_TYPES = ['debug', 'release'] +POSSIBLE_BUILD_LIBS = ['all', 'apps', 'plugins', 'libs', 'tracers'] TAPPAS_WORKSPACE = Path(os.environ['TAPPAS_WORKSPACE']).resolve() FOLDER_NAME = Path(__file__).resolve().parent @@ -21,13 +22,15 @@ class GstreamerInstall: ] LIBARGS_TEMPLATE = "{}-std=c++17" - def __init__(self, arch, target, build_type, toolchain_tar_path, yocto_distribution='poky', remove_cache=False): + def __init__(self, arch, target, build_type, toolchain_tar_path, yocto_distribution='poky', build_lib='all', remove_cache=False, install_to_rootfs=False): self._logger = logging.getLogger(__file__) self._arch = arch self._target_platform = target self._build_type = build_type self._build_dir = FOLDER_NAME / f'{self._arch.value}-gsthailotools-build-{self._build_type}' self._remove_cache = remove_cache + self.install_to_rootfs = install_to_rootfs + self._build_lib = build_lib self._runner = ShellRunner() self._toolchain_tar_path = Path(toolchain_tar_path).absolute().resolve() @@ -36,7 +39,6 @@ def __init__(self, arch, target, build_type, toolchain_tar_path, yocto_distribut self._open_source_root = f'{TAPPAS_WORKSPACE}/core/open_source' self._tappas_sources_dir = f'{TAPPAS_WORKSPACE}/core/hailo' - self._cross_file = FOLDER_NAME / "cross_files" / f"{arch}_cross_file.txt" self._hailort_cross_compiled_output_dir = FOLDER_NAME / f"build.linux.{self._arch.value}.{self._build_type}" self._initialize_toolchain() @@ -50,10 +52,6 @@ def _initialize_toolchain(self): dir_to_install_toolchain_in=self._unpacked_toolchain_dir) self._logger.info('Toolchain ready to use ({})'.format(self._unpacked_toolchain_dir)) - def get_image_user_path(self): - usr_path = os.path.join(self._toolchain_rootfs_base_path, 'usr') - return usr_path - def get_libargs_line(self, rootfs_base_path): def get_includes(includes): include_string = '' @@ -88,16 +86,24 @@ def run_meson_build_command(self, env=None): '-Dprefix={}'.format(usr_path), '-Dinclude_blas=false', '-Dtarget_platform={}'.format(self._target_platform), + '-Dtarget={}'.format(self._build_lib), '-Dlibxtensor={}'.format(xtensor_base_root), '-Dlibblas={}'.format(xtensor_blas_root), '-Dlibcxxopts={}'.format(cxxopts_root), '-Dlibrapidjson={}'.format(rapidjson_root), - '-Dinclude_unit_tests=false', - '--cross-file={}'.format(self._cross_file)] + '-Dinclude_unit_tests=false'] self._runner.run(build_cmd, env=env, print_output=True) self._logger.info('Done running Meson command') + def run_ninja_install_command(self, env=None): + self._logger.info("Running Ninja install command.") + + with working_directory(self._tappas_sources_dir): + ninja_cmd = ['ninja', 'install', '-C', self._build_dir] + self._runner.run(ninja_cmd, env, print_output=True) + self._logger.info('Done running Ninja install') + def run_ninja_build_command(self, env=None): self._logger.info("Running Ninja command.") @@ -130,6 +136,8 @@ def build(self): self.run_meson_build_command(env) self.run_ninja_build_command(env) + if self.install_to_rootfs: + self.run_ninja_install_command(env) self._logger.info(f"Build done. Outputs could be found in {self._build_dir}") @@ -141,7 +149,9 @@ def parse_args(): parser.add_argument('build_type', choices=POSSIBLE_BUILD_TYPES, help='Build and compilation type') parser.add_argument('toolchain_tar_path', help='Toolchain TAR path') parser.add_argument('--yocto-distribution', help='The name of the Yocto distribution to use (default poky)', default='poky') + parser.add_argument('--build-lib', help='Build a specific tappas lib target (default all)', choices=POSSIBLE_BUILD_LIBS, default='all') parser.add_argument('--remove-cache', action='store_true', help='Delete previous build cache (default false)', default=False) + parser.add_argument('--install-to-rootfs', action='store_true', help='Install to rootfs (default false)', default=False) return parser.parse_args() @@ -155,5 +165,7 @@ def parse_args(): gst_installer = GstreamerInstall(arch=args.arch, target=args.target, build_type=args.build_type, toolchain_tar_path=args.toolchain_tar_path, yocto_distribution=args.yocto_distribution, - remove_cache=args.remove_cache) + build_lib=args.build_lib, + remove_cache=args.remove_cache, + install_to_rootfs=args.install_to_rootfs) gst_installer.build() diff --git a/tools/cross_compiler/cross_files/aarch64_cross_file.txt b/tools/cross_compiler/cross_files/aarch64_cross_file.txt deleted file mode 100644 index f08a4e9..0000000 --- a/tools/cross_compiler/cross_files/aarch64_cross_file.txt +++ /dev/null @@ -1,15 +0,0 @@ -[host_machine] -system = 'linux' -cpu_family = 'aarch64' -cpu = 'arm64' -endian = 'little' - -[properties] -skip_sanity_check = true - -[binaries] -c = ['aarch64-linux-gnu-gcc'] -cpp = ['aarch64-linux-gnu-g++'] -ar = ['aarch64-linux-gnu-ar'] -pkgconfig = 'pkg-config' -strip = ['aarch64-linux-gnu-strip'] diff --git a/tools/cross_compiler/cross_files/armv7lhf_cross_file.txt b/tools/cross_compiler/cross_files/armv7lhf_cross_file.txt deleted file mode 100644 index 249ea46..0000000 --- a/tools/cross_compiler/cross_files/armv7lhf_cross_file.txt +++ /dev/null @@ -1,15 +0,0 @@ -[host_machine] -system = 'linux' -cpu_family = 'arm' -cpu = 'arm' -endian = 'little' - -[properties] -skip_sanity_check = true - -[binaries] -c = ['arm-linux-gnueabihf-gcc'] -cpp = ['arm-linux-gnueabihf-g++'] -ar = ['arm-linux-gnueabihf-ar'] -pkgconfig = 'pkg-config' -strip = ['arm-linux-gnueabihf-strip'] \ No newline at end of file diff --git a/tools/cross_compiler/cross_files/armv8a_cross_file.txt b/tools/cross_compiler/cross_files/armv8a_cross_file.txt deleted file mode 100644 index f08a4e9..0000000 --- a/tools/cross_compiler/cross_files/armv8a_cross_file.txt +++ /dev/null @@ -1,15 +0,0 @@ -[host_machine] -system = 'linux' -cpu_family = 'aarch64' -cpu = 'arm64' -endian = 'little' - -[properties] -skip_sanity_check = true - -[binaries] -c = ['aarch64-linux-gnu-gcc'] -cpp = ['aarch64-linux-gnu-g++'] -ar = ['aarch64-linux-gnu-ar'] -pkgconfig = 'pkg-config' -strip = ['aarch64-linux-gnu-strip'] diff --git a/tools/decoding_display_pipeline/README b/tools/decoding_display_pipeline/README index 5b7a705..d22c533 100644 --- a/tools/decoding_display_pipeline/README +++ b/tools/decoding_display_pipeline/README @@ -42,7 +42,9 @@ We would begin by creating soft-links for files in three common resolution: ## Running the benchmarks ```sh -$ ./decoding_display_pipeline.sh --use-vaapi --use-vaapi-scale --video-prefix-path 640x640_video --format RGBA --max-num-of-sources 8 + +$ ./decoding_display_pipeline.sh --use-vaapi --use-vaapi-scale --video-prefix-path 640x640_video --format RGBA --max-num-of-sources 8 + Num of sources: 1, Average FPS: -nan Num of sources: 2, Average FPS: -nan Num of sources: 3, Average FPS: 2268.17 @@ -53,6 +55,7 @@ Num of sources: 7, Average FPS: 1960.4 Num of sources: 8, Average FPS: 2020.77 $ ./decoding_display_pipeline.sh --use-vaapi --use-vaapi-scale --video-prefix-path 1280x720_video --format RGBA --max-num-of-sources 8 + Num of sources: 1, Average FPS: 211.077 Num of sources: 2, Average FPS: 332.512 Num of sources: 3, Average FPS: 364.933 @@ -63,6 +66,7 @@ Num of sources: 7, Average FPS: 364.404 Num of sources: 8, Average FPS: 355.913 $ ./decoding_display_pipeline.sh --use-vaapi --use-vaapi-scale --video-prefix-path 1920x1080_video --format RGBA --max-num-of-sources 8 + Num of sources: 1, Average FPS: 96.7667 Num of sources: 2, Average FPS: 161.755 Num of sources: 3, Average FPS: 173.417 diff --git a/tools/decoding_display_pipeline/create_soft_links.sh b/tools/decoding_display_pipeline/create_soft_links.sh index b269da9..0b81bd2 100755 --- a/tools/decoding_display_pipeline/create_soft_links.sh +++ b/tools/decoding_display_pipeline/create_soft_links.sh @@ -1,3 +1,6 @@ +#!/bin/bash +set -e + output_prefix='video' num_of_copies='' input='' diff --git a/tools/decoding_display_pipeline/decoding_display_pipeline.sh b/tools/decoding_display_pipeline/decoding_display_pipeline.sh index f521d40..8a33405 100755 --- a/tools/decoding_display_pipeline/decoding_display_pipeline.sh +++ b/tools/decoding_display_pipeline/decoding_display_pipeline.sh @@ -76,9 +76,17 @@ function parse_args() { done } +function check_invalid_combinations() { + if [ "$no_display" = '' ] && [ "$converted_format" = "--format RGBA" ]; then + echo "Received invalid configuration. Cannot use display with RGBA format." + exit 2 + fi +} + function main() { init_variables $@ parse_args $@ + check_invalid_combinations for ((n = $min_num_of_src; n <= $max_num_of_src; n++)); do avg_fps=$($TAPPAS_WORKSPACE/tools/decoding_display_pipeline/run_decoding_display_pipeline.sh --video-prefix-path \ diff --git a/tools/decoding_display_pipeline/run_decoding_display_pipeline.sh b/tools/decoding_display_pipeline/run_decoding_display_pipeline.sh index f4438e2..f587f54 100755 --- a/tools/decoding_display_pipeline/run_decoding_display_pipeline.sh +++ b/tools/decoding_display_pipeline/run_decoding_display_pipeline.sh @@ -2,6 +2,10 @@ set -e function init_variables() { + # Assure all check are ready before running the pipeline + script_dir=$(dirname $(realpath "$0")) + source $script_dir/../../scripts/misc/checks_before_run.sh --no-hailo + decode_element="decodebin" num_of_src=4 num_of_buffers=500 @@ -109,6 +113,8 @@ function post_parse_args() { if [ -n "${converted_format_caps}" ]; then decode_element+=$converted_format_caps fi + + source $script_dir/../../scripts/vaapi/set_env.sh else decode_element="decodebin ! video/x-raw, width=$width, height=$height" fi