[Tizen/Web] Add ml offloading example for Tizen client side

This patch adds ml offloading example for Tizen client side. tensor_query_client is used. Signed-off-by: Yelin Jeong <[email protected]>
nnstreamer · Apr 30, 2024 · de3bffd · de3bffd
1 parent d8ab9cc
commit de3bffd
Show file tree

Hide file tree

Showing 12 changed files with 1,289 additions and 0 deletions.
diff --git a/Tizen.web/ImageClassificationOffloading/.project b/Tizen.web/ImageClassificationOffloading/.project
@@ -0,0 +1,24 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<projectDescription>
+	<name>ImageClassificationOffloading</name>
+	<comment></comment>
+	<projects>
+	</projects>
+	<buildSpec>
+		<buildCommand>
+			<name>json.validation.builder</name>
+			<arguments>
+			</arguments>
+		</buildCommand>
+		<buildCommand>
+			<name>org.tizen.web.project.builder.WebBuilder</name>
+			<arguments>
+			</arguments>
+		</buildCommand>
+	</buildSpec>
+	<natures>
+		<nature>json.validation.nature</nature>
+		<nature>org.eclipse.wst.jsdt.core.jsNature</nature>
+		<nature>org.tizen.web.project.builder.WebNature</nature>
+	</natures>
+</projectDescription>
diff --git a/Tizen.web/ImageClassificationOffloading/.tproject b/Tizen.web/ImageClassificationOffloading/.tproject
@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<tproject xmlns="http://www.tizen.org/tproject">
+    <platforms>
+        <platform>
+            <name>tizen-8.0</name>
+        </platform>
+    </platforms>
+    <package>
+        <blacklist/>
+    </package>
+</tproject>
diff --git a/Tizen.web/ImageClassificationOffloading/README.md b/Tizen.web/ImageClassificationOffloading/README.md
@@ -0,0 +1,8 @@
+# Image Classification Sample App (Offloading version)
+## Description
+* This is a sample application of Tizen ML Web APIs.
+* If you want to run it on your device, Tizen 8.0 or higher is required.
+* `appsrc` and `tensor_query_client` element are used.
+
+## Demo
+TBD
diff --git a/Tizen.web/ImageClassificationOffloading/config.xml b/Tizen.web/ImageClassificationOffloading/config.xml
@@ -0,0 +1,13 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<widget xmlns:tizen="http://tizen.org/ns/widgets" xmlns="http://www.w3.org/ns/widgets" id="http://yourdomain/ImageClassificationOffloading" version="1.0.0" viewmodes="maximized">
+    <tizen:application id="EQmf4iSfpX.ImageClassificationOffloading" package="EQmf4iSfpX" required_version="8.0"/>
+    <content src="index.html"/>
+    <feature name="http://tizen.org/feature/screen.size.all"/>
+    <icon src="icon.png"/>
+    <name>ImageClassificationOffloading</name>
+    <tizen:privilege name="http://tizen.org/privilege/filesystem.read"/>
+    <tizen:privilege name="http://tizen.org/privilege/filesystem.write"/>
+    <tizen:privilege name="http://tizen.org/privilege/mediastorage"/>
+    <tizen:privilege name="http://tizen.org/privilege/internet"/>
+    <tizen:profile name="tizen"/>
+</widget>
diff --git a/Tizen.web/ImageClassificationOffloading/css/style.css b/Tizen.web/ImageClassificationOffloading/css/style.css
@@ -0,0 +1,24 @@
+html,
+body {
+    width: 100%;
+    height: 100%;
+    margin: 0 auto;
+    padding: 0;
+    background-color: #222222;
+    color: #ffffff;
+}
+.page {
+    width: 100%;
+    height: 100%;
+    display: table;
+}
+.contents {
+    display: table-cell;
+    vertical-align: middle;
+    text-align: center;
+    -webkit-tap-highlight-color: transparent;
+}
+#content-text {
+    font-weight: bold;
+    font-size: 5em;
+}
diff --git a/Tizen.web/ImageClassificationOffloading/icon.png b/Tizen.web/ImageClassificationOffloading/icon.png
diff --git a/Tizen.web/ImageClassificationOffloading/images/tizen_32.png b/Tizen.web/ImageClassificationOffloading/images/tizen_32.png
diff --git a/Tizen.web/ImageClassificationOffloading/index.html b/Tizen.web/ImageClassificationOffloading/index.html
@@ -0,0 +1,42 @@
+<!DOCTYPE html>
+<html>
+
+<head>
+    <meta charset="utf-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0">
+    <meta name="description" content="Tizen Image Classification SingleShot Example" />
+
+    <title>Tizen Image Classification Offloading Example</title>
+
+    <link rel="stylesheet" type="text/css" href="css/style.css" />
+    <script src="js/main.js"></script>
+</head>
+
+<body>
+    <h1 align="center">Image Classification</h1>
+    <div id="main" class="page" align="center">
+        <img src="res/0.jpg" />
+        <div>local </div>
+        <div id="local_time"></div>
+        <br>
+        <div>offloading </div>
+        <div id="offloading_time"></div>
+        <br>
+        <button type="button" id="local">Start Client (local)</button>
+        <br>
+        <button type="button" id="offloading">Start Client (offloading)</button>
+        <br>
+        <button type="button" id="localPush">Push data (local)</button>
+        <br>
+        <button type="button" id="offloadingPush">Push data (offloading)</button>
+        <br>
+        <br>
+        <label for="port">port:</label>
+        <input id="port" type="text" name="port" value="0">
+        <br>
+        <br>
+        <div id="label"></div>
+    </div>
+</body>
+
+</html>
diff --git a/Tizen.web/ImageClassificationOffloading/js/main.js b/Tizen.web/ImageClassificationOffloading/js/main.js
@@ -0,0 +1,166 @@
+/* SPDX-License-Identifier: Apache-2.0-only */
+
+/**
+ * @file main.js
+ * @date 30 April 2024
+ * @brief Image classification Offloading example
+ * @author Yelin Jeong <[email protected]>
+ * @bug When pushing data to appsrc for the first time
+ * after creating a pipeline, the sink listener is not called.
+ */
+
+var localSrc;
+var remoteSrc;
+var labels;
+var startTime;
+
+/**
+ * Find the index of maximum value in the given array
+ * @param array the score list of each class
+ * @returns the index of the maximum value
+ */
+function GetMaxIdx(array) {
+    if (array.length === 0) {
+        return -1;
+    }
+
+    var max = array[0];
+    var maxIdx = 0;
+
+    for (var i = 0; i < array.length; ++i) {
+        if (array[i] > max) {
+            maxIdx = i;
+            max = array[i];
+        }
+    }
+    return maxIdx;
+}
+
+/**
+ * Load the label from the text file and return the string array
+ * @returns string array
+ */
+function loadLabelInfo() {
+    var fHandle = tizen.filesystem.openFile("wgt-package/res/labels.txt", 'r');
+    var labelList = fHandle.readString();
+    return labelList.split('\n');
+}
+
+/**
+ * Run a pipeline that uses Tizen device's resources
+ */
+function runLocal() {
+    const modelPath = 'wgt-package/res/mobilenet_v1_1.0_224_quant.tflite';
+    var URI_PREFIX = 'file://';
+    var absModelPath = tizen.filesystem.toURI(modelPath).substr(URI_PREFIX.length);
+
+    var pipelineDescription = "appsrc caps=image/jpeg name=srcx ! jpegdec ! " +
+        "videoconvert ! video/x-raw,format=RGB,framerate=0/1,width=224,height=224 ! tensor_converter ! " +
+        "tensor_filter framework=tensorflow-lite model=" + absModelPath + " ! " +
+        "appsink name=sinkx_local";
+
+    var pHandle = tizen.ml.pipeline.createPipeline(pipelineDescription);
+    pHandle.start();
+
+    localSrc = pHandle.getSource('srcx');
+
+    pHandle.registerSinkListener('sinkx_local', function(sinkName, data) {
+        var endTime = performance.now();
+        var label = document.querySelector('#label');
+        var tensorsRetData = data.getTensorRawData(0);
+        var maxIdx = GetMaxIdx(tensorsRetData.data);
+        label.innerText = labels[maxIdx];
+
+        var time = document.querySelector('#local_time');
+        time.innerText = endTime - startTime + " ms"
+    });
+}
+
+/**
+ * Run a pipeline that uses other device's resources
+ */
+function runRemote() {
+    if (document.getElementById('port').value == 0) {
+        console.log("No port number is given")
+        return
+    }
+
+    /* TODO : Only use internal network now */
+    var pipelineDescription = "appsrc caps=image/jpeg name=srcx ! jpegdec ! " +
+        "videoconvert ! video/x-raw,format=RGB,framerate=0/1,width=224,height=224 ! tensor_converter  ! " +
+        "other/tensor,format=static,dimension=(string)3:224:224:1,type=uint8,framerate=0/1  ! " +
+        "tensor_query_client host=192.168.50.38 port=" + document.getElementById('port').value + " dest-host=" + "192.168.50.191" + " " +
+        "dest-port=" + document.getElementById('port').value + " ! " +
+        "other/tensor,format=static,dimension=(string)1001:1,type=uint8,framerate=0/1 ! tensor_sink name=sinkx_remote";
+
+    var pHandle = tizen.ml.pipeline.createPipeline(pipelineDescription);
+    pHandle.start();
+
+    remoteSrc = pHandle.getSource('srcx');
+
+    pHandle.registerSinkListener('sinkx_remote', function(sinkName, data) {
+        var endTime = performance.now();
+        var label = document.querySelector('#label');
+        var tensorsRetData = data.getTensorRawData(0);
+        var maxIdx = GetMaxIdx(tensorsRetData.data);
+        label.innerText = labels[maxIdx];
+
+        var time = document.querySelector('#offloading_time');
+        time.innerText = endTime - startTime + " ms"
+    });
+}
+
+window.onload = function() {
+    labels = loadLabelInfo();
+
+    /* TODO : Change the image for each inference */
+    var fHandle = tizen.filesystem.openFile("wgt-package/res/0.jpg", 'r');
+    var imgUInt8Array = fHandle.readData();
+    fHandle.close();
+
+    var tensorsInfo = new tizen.ml.TensorsInfo();
+    tensorsInfo.addTensorInfo("tensor", "UINT8", [imgUInt8Array.length]);
+    var tensorsData = tensorsInfo.getTensorsData();
+    tensorsData.setTensorRawData(0, imgUInt8Array);
+
+    const btnLocal = document.querySelector("#local");
+
+    btnLocal.addEventListener("click", function() {
+        runLocal();
+    });
+
+    const btnOffloading = document.querySelector("#offloading");
+
+    btnOffloading.addEventListener("click", function() {
+        runRemote();
+    });
+
+    const btnLocalPush = document.querySelector("#localPush");
+
+    btnLocalPush.addEventListener("click", function() {
+        startTime = performance.now()
+        localSrc.inputData(tensorsData);
+    });
+
+    const btnOffloadingPush = document.querySelector("#offloadingPush");
+
+    btnOffloadingPush.addEventListener("click", function() {
+        startTime = performance.now()
+        remoteSrc.inputData(tensorsData);
+    });
+
+    /* add eventListener for tizenhwkey */
+    document.addEventListener('tizenhwkey', function(e) {
+        if (e.keyName === "back") {
+            try {
+                console.log("Pipeline is disposed!!");
+                pHandle.stop();
+                pHandle.dispose();
+                tensorsData.dispose();
+                tensorsInfo.dispose();
+
+                tizen.application.getCurrentApplication().exit();
+            } catch (ignore) {}
+        }
+    });
+};
diff --git a/Tizen.web/ImageClassificationOffloading/res/0.jpg b/Tizen.web/ImageClassificationOffloading/res/0.jpg