Skip to content

Commit

Permalink
[Tizen/Web] Change input image for each inference
Browse files Browse the repository at this point in the history 
This patch changes input image for each inference.

Signed-off-by: Yelin Jeong <[email protected]>
  • Loading branch information
@niley7464 @jaeyun-jung
niley7464 authored and jaeyun-jung committed May 13, 2024
1 parent 59bff60 commit 9f186b5
Show file tree
Hide file tree
Showing 5 changed files with 126 additions and 101 deletions.
2 changes: 1 addition & 1 deletion Tizen.web/ImageClassificationOffloading/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,4 @@
* `appsrc` and `tensor_query_client` element are used.

## Demo
TBD
![Alt demo](./image_classification_offloading.png)
24 changes: 13 additions & 11 deletions Tizen.web/ImageClassificationOffloading/css/style.css
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,18 +7,20 @@ body {
background-color: #222222;
color: #ffffff;
}
.page {
width: 100%;
height: 100%;
display: table;
#container {
display: flex;
}
.element {
flex: 1;
text-align: center;
font-size: 30px;
}
.contents {
display: table-cell;
vertical-align: middle;
.button {
padding: 10px 10px;
font-size: 20px;
margin: 10px;
text-align: center;
-webkit-tap-highlight-color: transparent;
}
#content-text {
font-weight: bold;
font-size: 5em;
.canvas {
margin: 50px
}
Binary file added Tizen.web/ImageClassificationOffloading/image_classification_offloading.png
View file
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
56 changes: 24 additions & 32 deletions Tizen.web/ImageClassificationOffloading/index.html
View file
Original file line number Diff line number Diff line change
@@ -1,42 +1,34 @@
<!DOCTYPE html>
<html>

<head>
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0">
<meta name="description" content="Tizen Image Classification SingleShot Example" />

<title>Tizen Image Classification Offloading Example</title>

<link rel="stylesheet" type="text/css" href="css/style.css" />
<script src="js/main.js"></script>
</head>

<body>
<h1 align="center">Image Classification</h1>
<div id="main" class="page" align="center">
<img src="res/0.jpg" />
<div>local </div>
<div id="local_time"></div>
<br>
<div>offloading </div>
<div id="offloading_time"></div>
<br>
<button type="button" id="local">Start Client (local)</button>
<br>
<button type="button" id="offloading">Start Client (offloading)</button>
<br>
<button type="button" id="localPush">Push data (local)</button>
<br>
<button type="button" id="offloadingPush">Push data (offloading)</button>
<br>
<br>
<label for="port">port:</label>
<input id="port" type="text" name="port" value="0">
<br>
<br>
<div id="label"></div>
</head>
<body>
<h1 align="center" margin-bottom="50px">Image Classification</h1>
<div id="container">
<div class="element">
<button class="button" id="start_local">Create Local Pipeline</button>
<div id="local">
<canvas class="canvas" id="canvas_local" width="224" height="224"></canvas>
<div id="label_local">Tab here</div>
<div id="time_local"></div>
</div>
</div>
<div class="element">
<button class="button" id="start_offloading">Create Offloading Pipeline</button>
<br>
<input class="button" id="port" type="text" name="port" value="Port"/>
<div id="offloading">
<canvas id="canvas_offloading" width="224" height="224"></canvas>
<div id="label_offloading">Tab here</div>
<div id="time_offloading"></div>
</div>
</div>
</div>
</body>

</body>
</html>
145 changes: 88 additions & 57 deletions Tizen.web/ImageClassificationOffloading/js/main.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@ var localSrc;
var remoteSrc;
var labels;
var startTime;
var ctx;
var label;

/**
* Find the index of maximum value in the given array
Expand All @@ -21,6 +23,7 @@ var startTime;
*/
function GetMaxIdx(array) {
if (array.length === 0) {
console.log('array length zero')
return -1;
}

Expand All @@ -36,12 +39,24 @@ function GetMaxIdx(array) {
return maxIdx;
}

/**
* Get the jpeg image path
* @returns image path
*/
function GetImgPath() {
const MAX_IMG_CNT = 2;
var imgsrc = GetImgPath.count++ % MAX_IMG_CNT;
imgsrc = imgsrc.toString().concat('.jpg');
return '/res/'.concat(imgsrc);
}
GetImgPath.count = 0;

/**
* Load the label from the text file and return the string array
* @returns string array
*/
function loadLabelInfo() {
var fHandle = tizen.filesystem.openFile("wgt-package/res/labels.txt", 'r');
var fHandle = tizen.filesystem.openFile('wgt-package/res/labels.txt', 'r');
var labelList = fHandle.readString();
return labelList.split('\n');
}
Expand All @@ -51,28 +66,28 @@ function loadLabelInfo() {
*/
function runLocal() {
const modelPath = 'wgt-package/res/mobilenet_v1_1.0_224_quant.tflite';
var URI_PREFIX = 'file://';
var absModelPath = tizen.filesystem.toURI(modelPath).substr(URI_PREFIX.length);
const URI_PREFIX = 'file://';
const absModelPath = tizen.filesystem.toURI(modelPath).substr(URI_PREFIX.length);

var pipelineDescription = "appsrc caps=image/jpeg name=srcx ! jpegdec ! " +
"videoconvert ! video/x-raw,format=RGB,framerate=0/1,width=224,height=224 ! tensor_converter ! " +
"tensor_filter framework=tensorflow-lite model=" + absModelPath + " ! " +
"appsink name=sinkx_local";
const pipelineDescription = 'appsrc caps=image/jpeg name=srcx_local ! jpegdec ! ' +
'videoconvert ! video/x-raw,format=RGB,framerate=0/1,width=224,height=224 ! tensor_converter ! ' +
'tensor_filter framework=tensorflow-lite model=' + absModelPath + ' ! ' +
'appsink name=sinkx_local';

var pHandle = tizen.ml.pipeline.createPipeline(pipelineDescription);
const pHandle = tizen.ml.pipeline.createPipeline(pipelineDescription);
pHandle.start();

localSrc = pHandle.getSource('srcx');
localSrc = pHandle.getSource('srcx_local');

pHandle.registerSinkListener('sinkx_local', function(sinkName, data) {
var endTime = performance.now();
var label = document.querySelector('#label');
var tensorsRetData = data.getTensorRawData(0);
var maxIdx = GetMaxIdx(tensorsRetData.data);
const endTime = performance.now();
const label = document.getElementById('label_local')
const tensorsRetData = data.getTensorRawData(0);
const maxIdx = GetMaxIdx(tensorsRetData.data);
label.innerText = labels[maxIdx];

var time = document.querySelector('#local_time');
time.innerText = endTime - startTime + " ms"
const time = document.getElementById('time_local');
time.innerText = 'local : ' + (endTime - startTime) + ' ms'
});
}

Expand All @@ -81,83 +96,99 @@ function runLocal() {
*/
function runRemote() {
if (document.getElementById('port').value == 0) {
console.log("No port number is given")
console.log('No port number is given')
return
}

/* TODO : Only use internal network now */
var pipelineDescription = "appsrc caps=image/jpeg name=srcx ! jpegdec ! " +
"videoconvert ! video/x-raw,format=RGB,framerate=0/1,width=224,height=224 ! tensor_converter ! " +
"other/tensor,format=static,dimension=(string)3:224:224:1,type=uint8,framerate=0/1 ! " +
"tensor_query_client host=192.168.50.38 port=" + document.getElementById('port').value + " dest-host=" + "192.168.50.191" + " " +
"dest-port=" + document.getElementById('port').value + " ! " +
"other/tensor,format=static,dimension=(string)1001:1,type=uint8,framerate=0/1 ! tensor_sink name=sinkx_remote";

var pHandle = tizen.ml.pipeline.createPipeline(pipelineDescription);
const pipelineDescription = 'appsrc caps=image/jpeg name=srcx_remote ! jpegdec ! ' +
'videoconvert ! video/x-raw,format=RGB,framerate=0/1,width=224,height=224 ! tensor_converter ! ' +
'other/tensor,format=static,dimension=(string)3:224:224:1,type=uint8,framerate=0/1 ! ' +
'tensor_query_client host=192.168.50.38 port=' + document.getElementById('port').value + ' dest-host=' + '192.168.50.191' + ' ' +
'dest-port=' + document.getElementById('port').value + ' ! ' +
'other/tensor,format=static,dimension=(string)1001:1,type=uint8,framerate=0/1 ! tensor_sink name=sinkx_remote';

const pHandle = tizen.ml.pipeline.createPipeline(pipelineDescription);
pHandle.start();

remoteSrc = pHandle.getSource('srcx');
remoteSrc = pHandle.getSource('srcx_remote');

pHandle.registerSinkListener('sinkx_remote', function(sinkName, data) {
var endTime = performance.now();
var label = document.querySelector('#label');
var tensorsRetData = data.getTensorRawData(0);
var maxIdx = GetMaxIdx(tensorsRetData.data);
const endTime = performance.now();
const label = document.getElementById('label_offloading');
const tensorsRetData = data.getTensorRawData(0);
const maxIdx = GetMaxIdx(tensorsRetData.data);
label.innerText = labels[maxIdx];

var time = document.querySelector('#offloading_time');
time.innerText = endTime - startTime + " ms"
const time = document.getElementById('time_offloading');
time.innerText = 'offloading : ' + (endTime - startTime) + ' ms'
});
}

window.onload = function() {
labels = loadLabelInfo();
function inference(src, canvas) {
const img_path = GetImgPath();
let img = new Image();
img.src = img_path;

img.onload = function () {
const fHandle = tizen.filesystem.openFile('wgt-package' + img_path, 'r');
const imgUInt8Array = fHandle.readData();
fHandle.close();

/* TODO : Change the image for each inference */
var fHandle = tizen.filesystem.openFile("wgt-package/res/0.jpg", 'r');
var imgUInt8Array = fHandle.readData();
fHandle.close();
const tensorsInfo = new tizen.ml.TensorsInfo();
tensorsInfo.addTensorInfo('tensor', 'UINT8', [imgUInt8Array.length]);
const tensorsData = tensorsInfo.getTensorsData();
tensorsData.setTensorRawData(0, imgUInt8Array);

var tensorsInfo = new tizen.ml.TensorsInfo();
tensorsInfo.addTensorInfo("tensor", "UINT8", [imgUInt8Array.length]);
var tensorsData = tensorsInfo.getTensorsData();
tensorsData.setTensorRawData(0, imgUInt8Array);
startTime = performance.now()
src.inputData(tensorsData);

const btnLocal = document.querySelector("#local");
tensorsData.dispose();
tensorsInfo.dispose();

btnLocal.addEventListener("click", function() {
const ctx = canvas.getContext('2d');
ctx.drawImage(img, 0, 0);
}
}

window.onload = function() {
labels = loadLabelInfo();

const btnLocal = document.getElementById('start_local');

btnLocal.addEventListener('click', function() {
runLocal();
});

const btnOffloading = document.querySelector("#offloading");
const btnOffloading = document.getElementById('start_offloading');

btnOffloading.addEventListener("click", function() {
btnOffloading.addEventListener('click', function() {
runRemote();
});

const btnLocalPush = document.querySelector("#localPush");
const localPage = document.getElementById('local');

btnLocalPush.addEventListener("click", function() {
startTime = performance.now()
localSrc.inputData(tensorsData);
localPage.addEventListener('click', function() {
if (localSrc) {
inference(localSrc, document.getElementById('canvas_local'));
}
});

const btnOffloadingPush = document.querySelector("#offloadingPush");
const offloadingPage = document.getElementById('offloading');

btnOffloadingPush.addEventListener("click", function() {
startTime = performance.now()
remoteSrc.inputData(tensorsData);
offloadingPage.addEventListener('click', function() {
if (remoteSrc) {
inference(remoteSrc, document.getElementById('canvas_offloading'));
}
});

/* add eventListener for tizenhwkey */
document.addEventListener('tizenhwkey', function(e) {
if (e.keyName === "back") {
if (e.keyName === 'back') {
try {
console.log("Pipeline is disposed!!");
console.log('Pipeline is disposed!!');
pHandle.stop();
pHandle.dispose();
tensorsData.dispose();
tensorsInfo.dispose();

tizen.application.getCurrentApplication().exit();
} catch (ignore) {}
Expand Down

0 comments on commit 9f186b5

Please sign in to comment.