[Tizen/Web] Add ml offloading example using Yolov8

This patch adds ml offloading example for Tizen client side. Yolov8 is used for image classification. Signed-off-by: Yelin Jeong <[email protected]>
nnstreamer · Jun 18, 2024 · c8cba38 · c8cba38
1 parent c2b7e9e
commit c8cba38
Show file tree

Hide file tree

Showing 16 changed files with 466 additions and 1 deletion.
diff --git a/.github/workflows/tizen-web.yml b/.github/workflows/tizen-web.yml
@@ -14,7 +14,7 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        app: ['ImageClassificationSingleShot', 'ImageClassificationPipeline', 'ImageClassificationOffloading']
+        app: ['ImageClassificationSingleShot', 'ImageClassificationPipeline', 'ImageClassificationOffloading', 'ImageClassificationOffloadingYolo']
     steps:
     - uses: actions/checkout@v4
       with:

diff --git a/Tizen.web/ImageClassificationOffloadingYolo/.project b/Tizen.web/ImageClassificationOffloadingYolo/.project
@@ -0,0 +1,24 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<projectDescription>
+	<name>ImageClassificationOffloadingYolo</name>
+	<comment></comment>
+	<projects>
+	</projects>
+	<buildSpec>
+		<buildCommand>
+			<name>json.validation.builder</name>
+			<arguments>
+			</arguments>
+		</buildCommand>
+		<buildCommand>
+			<name>org.tizen.web.project.builder.WebBuilder</name>
+			<arguments>
+			</arguments>
+		</buildCommand>
+	</buildSpec>
+	<natures>
+		<nature>json.validation.nature</nature>
+		<nature>org.eclipse.wst.jsdt.core.jsNature</nature>
+		<nature>org.tizen.web.project.builder.WebNature</nature>
+	</natures>
+</projectDescription>
diff --git a/Tizen.web/ImageClassificationOffloadingYolo/.tproject b/Tizen.web/ImageClassificationOffloadingYolo/.tproject
@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<tproject xmlns="http://www.tizen.org/tproject">
+    <platforms>
+        <platform>
+            <name>tizen-8.0</name>
+        </platform>
+    </platforms>
+    <package>
+        <blacklist/>
+    </package>
+</tproject>
diff --git a/Tizen.web/ImageClassificationOffloadingYolo/README.md b/Tizen.web/ImageClassificationOffloadingYolo/README.md
@@ -0,0 +1,18 @@
+# Image Classification Sample App (Offloading version)
+## Description
+* This is a sample application of Tizen ML Web APIs.
+* If you want to run it on your device, Tizen 8.0 or higher is required.
+* `appsrc` and `tensor_query_client` element are used.
+* Tflite model is created by this [guide](https://github.com/nnstreamer/nnstreamer-example/tree/main/bash_script/example_yolo#export-to-tflite-and-torchscript-model-1).
+```py
+from ultralytics import YOLO
+
+# Load a model
+model = YOLO("yolov8s.pt") # load a pretrained model
+
+# Export the model
+model.export(format="tflite", imgsz=224) # export the model to tflite format
+```
+
+## Demo
+![Alt demo](./image_classification_offloading_yolo.png)
diff --git a/Tizen.web/ImageClassificationOffloadingYolo/config.xml b/Tizen.web/ImageClassificationOffloadingYolo/config.xml
@@ -0,0 +1,13 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<widget xmlns:tizen="http://tizen.org/ns/widgets" xmlns="http://www.w3.org/ns/widgets" id="http://yourdomain/ImageClassificationOffloadingYolo" version="1.0.0" viewmodes="maximized">
+    <tizen:application id="kpElI8dRr7.ImageClassificationOffloadingYolo" package="kpElI8dRr7" required_version="8.0"/>
+    <content src="index.html"/>
+    <feature name="http://tizen.org/feature/screen.size.all"/>
+    <icon src="icon.png"/>
+    <name>ImageClassificationOffloadingYolo</name>
+    <tizen:privilege name="http://tizen.org/privilege/filesystem.read"/>
+    <tizen:privilege name="http://tizen.org/privilege/filesystem.write"/>
+    <tizen:privilege name="http://tizen.org/privilege/mediastorage"/>
+    <tizen:privilege name="http://tizen.org/privilege/internet"/>
+    <tizen:profile name="tizen"/>
+</widget>
diff --git a/Tizen.web/ImageClassificationOffloadingYolo/css/style.css b/Tizen.web/ImageClassificationOffloadingYolo/css/style.css
@@ -0,0 +1,26 @@
+html,
+body {
+    width: 100%;
+    height: 100%;
+    margin: 0 auto;
+    padding: 0;
+    background-color: #222222;
+    color: #ffffff;
+}
+#container {
+    display: flex;
+}
+.element {
+    flex: 1;
+    text-align: center;
+    font-size: 30px;
+}
+.button {
+    padding: 10px 10px;
+    font-size: 20px;
+    margin: 10px;
+    text-align: center;
+}
+.canvas {
+    margin: 50px
+}
diff --git a/Tizen.web/ImageClassificationOffloadingYolo/icon.png b/Tizen.web/ImageClassificationOffloadingYolo/icon.png
diff --git a/....web/ImageClassificationOffloadingYolo/image_classification_offloading_yolo.png b/....web/ImageClassificationOffloadingYolo/image_classification_offloading_yolo.png
diff --git a/Tizen.web/ImageClassificationOffloadingYolo/images/tizen_32.png b/Tizen.web/ImageClassificationOffloadingYolo/images/tizen_32.png
diff --git a/Tizen.web/ImageClassificationOffloadingYolo/index.html b/Tizen.web/ImageClassificationOffloadingYolo/index.html
@@ -0,0 +1,35 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <meta charset="utf-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0">
+    <meta name="description" content="Tizen Image Classification SingleShot Example" />
+    <title>Tizen Image Classification Offloading Yolo Example</title>
+    <link rel="stylesheet" type="text/css" href="css/style.css" />
+    <script src="js/util.js" type="module"></script>
+    <script src="js/main.js" type="module"></script>
+  </head>
+  <body>
+    <h1 align="center" margin-bottom="50px">Image Classification YoloV8</h1>
+    <div id="container">
+      <div class="element">
+        <button class="button" id="start_local">Create Local Pipeline</button>
+        <div id="local">
+          <canvas class="canvas" id="canvas_local" width="224" height="224"></canvas>
+          <div id="time_local">Tab here</div>
+        </div>
+      </div>
+      <div class="element">
+        <button class="button" id="start_offloading">Create Offloading Pipeline</button>
+        <br>
+        <input class="button" id="ip" type="text" name="ip" value="127.0.0.1"/>
+        <br>
+        <input class="button" id="port" type="text" name="port" value="3000"/>
+        <div id="offloading">
+          <canvas id="canvas_offloading" width="224" height="224"></canvas>
+          <div id="time_offloading">Tab here</div>
+        </div>
+      </div>
+    </div>
+  </body>
+</html>
diff --git a/Tizen.web/ImageClassificationOffloadingYolo/js/main.js b/Tizen.web/ImageClassificationOffloadingYolo/js/main.js
@@ -0,0 +1,200 @@
+/* SPDX-License-Identifier: Apache-2.0 */
+
+/**
+ * @file main.js
+ * @date 17 June 2024
+ * @brief Yolo Image classification Offloading example
+ * @author Yelin Jeong <[email protected]>
+ * @bug When drawing the results on the canvas,
+ * a blank image appears until the second attempts.
+ * (Only on Tizen RPI4 devices)
+ */
+
+import {
+    getAbsPath,
+    getNetworkType,
+    getIpAddress,
+    GetImgPath,
+} from "./utils.js";
+
+let fHandle = null;
+let tensorsData = null;
+let tensorsInfo = null;
+
+function disposeData() {
+    if (fHandle != null) {
+        fHandle.close();
+    }
+
+    if (tensorsData != null) {
+        tensorsData.dispose();
+    }
+
+    if (tensorsInfo != null) {
+        tensorsInfo.dispose();
+    }
+}
+
+let localHandle;
+let offloadingHandle;
+
+function createPipelineDescription(isLocal, filter) {
+    const absLabelPath = getAbsPath("coco.txt");
+
+    return (
+        "appsrc caps=image/jpeg name=srcx_" + (isLocal ? "local" : "offloading") + " ! jpegdec ! " +
+        "videoconvert ! video/x-raw,format=RGB,width=224,height=224,framerate=0/1 ! tee name=t " +
+        "t. ! tensor_converter ! tensor_transform mode=arithmetic option=typecast:float32,div:255.0 ! " +
+        "other/tensors,num_tensors=1,format=static,dimensions=(string)3:224:224:1,types=float32,framerate=0/1  ! " +
+        filter + " ! " + "other/tensors,num_tensors=1,types=float32,format=static,dimensions=1029:84:1 ! " +
+        "tensor_transform mode=transpose option=1:0:2:3 ! " +
+        "other/tensors,num_tensors=1,types=float32,format=static,dimensions=84:1029:1 !" +
+        "tensor_decoder mode=bounding_boxes option1=yolov8 option2=" + absLabelPath + " option3=0 option4=224:224 option5=224:224 ! " +
+        "video/x-raw,width=224,height=224,format=RGBA,framerate=0/1 ! mix.sink_0 t. ! mix.sink_1 " +
+        "compositor name=mix sink_0::zorder=2 sink_1::zorder=1 ! video/x-raw,width=224,height=224,format=RGBA,framerate=0/1  ! tensor_converter ! " +
+        "other/tensors,num_tensors=1,types=uint8,format=static,dimensions=4:224:224:1 ! " +
+        "appsink sync=false name=sinkx_" + (isLocal ? "local" : "offloading")
+    );
+}
+
+/**
+ * Callback function for pipeline sink listener
+ */
+function sinkListenerCallback(sinkName, data) {
+    const endTime = performance.now();
+
+    const tensorsRetData = data.getTensorRawData(0).data;
+    const pixelData = new Uint8ClampedArray(tensorsRetData);
+    const imageData = new ImageData(pixelData, 224);
+
+    let type;
+    if (sinkName.endsWith("local")) {
+        type = "local";
+    } else {
+        type = "offloading";
+    }
+
+    const canvas = document.getElementById("canvas_" + type);
+    canvas.width = 224;
+    canvas.height = 224;
+    const ctx = canvas.getContext("2d");
+    ctx.putImageData(imageData, 0, 0);
+
+    const time = document.getElementById("time_" + type);
+    time.innerText = type + " : " + (endTime - startTime) + " ms";
+}
+
+/**
+ * Run a pipeline that uses Tizen device's resources
+ */
+function runLocal() {
+    const absModelPath = getAbsPath("yolov8s_float32.tflite");
+    const filter =
+        "tensor_filter framework=tensorflow-lite model=" + absModelPath;
+
+    const pipelineDescription = createPipelineDescription(true, filter);
+
+    localHandle = tizen.ml.pipeline.createPipeline(pipelineDescription);
+    localHandle.start();
+    localHandle.registerSinkListener("sinkx_local", sinkListenerCallback);
+}
+
+/**
+ * Run a pipeline that uses other device's resources
+ */
+function runOffloading() {
+    const filter =
+        "tensor_query_client host=" + ip +
+        " port=" + document.getElementById("port").value +
+        " dest-host=" + document.getElementById("ip").value +
+        " dest-port=" + document.getElementById("port").value +
+        " timeout=1000";
+
+    const pipelineDescription = createPipelineDescription(false, filter);
+
+    offloadingHandle = tizen.ml.pipeline.createPipeline(pipelineDescription);
+    offloadingHandle.start();
+    offloadingHandle.registerSinkListener(
+        "sinkx_offloading",
+        sinkListenerCallback,
+    );
+}
+
+let startTime;
+
+/**
+ * Run a pipeline that uses other device's resources
+ */
+function inference(isLocal) {
+    const img_path = GetImgPath();
+    let img = new Image();
+    img.src = img_path;
+
+    img.onload = function () {
+        disposeData();
+        fHandle = tizen.filesystem.openFile("wgt-package" + img_path, "r");
+        const imgUInt8Array = fHandle.readData();
+
+        tensorsInfo = new tizen.ml.TensorsInfo();
+        tensorsInfo.addTensorInfo("tensor", "UINT8", [imgUInt8Array.length]);
+        tensorsData = tensorsInfo.getTensorsData();
+        tensorsData.setTensorRawData(0, imgUInt8Array);
+
+        startTime = performance.now();
+
+        if (isLocal) {
+        localHandle.getSource("srcx_local").inputData(tensorsData);
+        } else {
+        offloadingHandle.getSource("srcx_offloading").inputData(tensorsData);
+        }
+    };
+}
+
+let ip;
+
+window.onload = async function () {
+    const networkType = await getNetworkType();
+    ip = await getIpAddress(networkType);
+
+    document
+        .getElementById("start_local")
+        .addEventListener("click", function () {
+            runLocal();
+        });
+
+    document
+        .getElementById("start_offloading")
+        .addEventListener("click", function () {
+            runOffloading();
+        });
+
+    document
+        .getElementById("local")
+        .addEventListener("click", function () {
+            inference(true);
+        });
+
+    document
+        .getElementById("offloading")
+        .addEventListener("click", function () {
+            inference(false);
+        });
+
+    /* add eventListener for tizenhwkey */
+    document.addEventListener("tizenhwkey", function (e) {
+        if (e.keyName === "back") {
+        try {
+            console.log("Pipeline is disposed!!");
+            localHandle.stop();
+            localHandle.dispose();
+
+            offloadingHandle.stop();
+            offloadingHandle.dispose();
+
+            disposeData();
+
+            tizen.application.getCurrentApplication().exit();
+        } catch (ignore) {}
+        }
+    });
+};