Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Tizen/Web] Change input image for each inference #329

Merged
merged 1 commit into from
May 13, 2024
Merged

[Tizen/Web] Change input image for each inference #329

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Tizen.web/ImageClassificationOffloading/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,4 @@
* `appsrc` and `tensor_query_client` element are used.

## Demo
TBD
![Alt demo](./image_classification_offloading.png)
24 changes: 13 additions & 11 deletions Tizen.web/ImageClassificationOffloading/css/style.css
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,18 +7,20 @@ body {
background-color: #222222;
color: #ffffff;
}
.page {
width: 100%;
height: 100%;
display: table;
#container {
display: flex;
}
.element {
flex: 1;
text-align: center;
font-size: 30px;
}
.contents {
display: table-cell;
vertical-align: middle;
.button {
padding: 10px 10px;
font-size: 20px;
margin: 10px;
text-align: center;
-webkit-tap-highlight-color: transparent;
}
#content-text {
font-weight: bold;
font-size: 5em;
.canvas {
margin: 50px
}
Binary file added Tizen.web/ImageClassificationOffloading/image_classification_offloading.png
View file
Open in desktop
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
56 changes: 24 additions & 32 deletions Tizen.web/ImageClassificationOffloading/index.html
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,42 +1,34 @@
<!DOCTYPE html>
<html>

<head>
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0">
<meta name="description" content="Tizen Image Classification SingleShot Example" />

<title>Tizen Image Classification Offloading Example</title>

<link rel="stylesheet" type="text/css" href="css/style.css" />
<script src="js/main.js"></script>
</head>

<body>
<h1 align="center">Image Classification</h1>
<div id="main" class="page" align="center">
<img src="res/0.jpg" />
<div>local </div>
<div id="local_time"></div>
<br>
<div>offloading </div>
<div id="offloading_time"></div>
<br>
<button type="button" id="local">Start Client (local)</button>
<br>
<button type="button" id="offloading">Start Client (offloading)</button>
<br>
<button type="button" id="localPush">Push data (local)</button>
<br>
<button type="button" id="offloadingPush">Push data (offloading)</button>
<br>
<br>
<label for="port">port:</label>
<input id="port" type="text" name="port" value="0">
<br>
<br>
<div id="label"></div>
</head>
<body>
<h1 align="center" margin-bottom="50px">Image Classification</h1>
<div id="container">
<div class="element">
<button class="button" id="start_local">Create Local Pipeline</button>
<div id="local">
<canvas class="canvas" id="canvas_local" width="224" height="224"></canvas>
<div id="label_local">Tab here</div>
<div id="time_local"></div>
</div>
</div>
<div class="element">
<button class="button" id="start_offloading">Create Offloading Pipeline</button>
<br>
<input class="button" id="port" type="text" name="port" value="Port"/>
<div id="offloading">
<canvas id="canvas_offloading" width="224" height="224"></canvas>
<div id="label_offloading">Tab here</div>
<div id="time_offloading"></div>
</div>
</div>
</div>
</body>

</body>
</html>
145 changes: 88 additions & 57 deletions Tizen.web/ImageClassificationOffloading/js/main.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@ var localSrc;
var remoteSrc;
var labels;
var startTime;
var ctx;
var label;

/**
* Find the index of maximum value in the given array
Expand All @@ -21,6 +23,7 @@ var startTime;
*/
function GetMaxIdx(array) {
if (array.length === 0) {
console.log('array length zero')
return -1;
}

Expand All @@ -36,12 +39,24 @@ function GetMaxIdx(array) {
return maxIdx;
}

/**
* Get the jpeg image path
* @returns image path
*/
function GetImgPath() {
const MAX_IMG_CNT = 2;
var imgsrc = GetImgPath.count++ % MAX_IMG_CNT;
imgsrc = imgsrc.toString().concat('.jpg');
return '/res/'.concat(imgsrc);
}
GetImgPath.count = 0;

/**
* Load the label from the text file and return the string array
* @returns string array
*/
function loadLabelInfo() {
var fHandle = tizen.filesystem.openFile("wgt-package/res/labels.txt", 'r');
var fHandle = tizen.filesystem.openFile('wgt-package/res/labels.txt', 'r');
var labelList = fHandle.readString();
return labelList.split('\n');
}
Expand All @@ -51,28 +66,28 @@ function loadLabelInfo() {
*/
function runLocal() {
const modelPath = 'wgt-package/res/mobilenet_v1_1.0_224_quant.tflite';
var URI_PREFIX = 'file://';
var absModelPath = tizen.filesystem.toURI(modelPath).substr(URI_PREFIX.length);
const URI_PREFIX = 'file://';
const absModelPath = tizen.filesystem.toURI(modelPath).substr(URI_PREFIX.length);

var pipelineDescription = "appsrc caps=image/jpeg name=srcx ! jpegdec ! " +
"videoconvert ! video/x-raw,format=RGB,framerate=0/1,width=224,height=224 ! tensor_converter ! " +
"tensor_filter framework=tensorflow-lite model=" + absModelPath + " ! " +
"appsink name=sinkx_local";
const pipelineDescription = 'appsrc caps=image/jpeg name=srcx_local ! jpegdec ! ' +
'videoconvert ! video/x-raw,format=RGB,framerate=0/1,width=224,height=224 ! tensor_converter ! ' +
'tensor_filter framework=tensorflow-lite model=' + absModelPath + ' ! ' +
'appsink name=sinkx_local';

var pHandle = tizen.ml.pipeline.createPipeline(pipelineDescription);
const pHandle = tizen.ml.pipeline.createPipeline(pipelineDescription);
pHandle.start();

localSrc = pHandle.getSource('srcx');
localSrc = pHandle.getSource('srcx_local');

pHandle.registerSinkListener('sinkx_local', function(sinkName, data) {
var endTime = performance.now();
var label = document.querySelector('#label');
var tensorsRetData = data.getTensorRawData(0);
var maxIdx = GetMaxIdx(tensorsRetData.data);
const endTime = performance.now();
const label = document.getElementById('label_local')
const tensorsRetData = data.getTensorRawData(0);
const maxIdx = GetMaxIdx(tensorsRetData.data);
label.innerText = labels[maxIdx];

var time = document.querySelector('#local_time');
time.innerText = endTime - startTime + " ms"
const time = document.getElementById('time_local');
time.innerText = 'local : ' + (endTime - startTime) + ' ms'
});
}

Expand All @@ -81,83 +96,99 @@ function runLocal() {
*/
function runRemote() {
if (document.getElementById('port').value == 0) {
console.log("No port number is given")
console.log('No port number is given')
return
}

/* TODO : Only use internal network now */
var pipelineDescription = "appsrc caps=image/jpeg name=srcx ! jpegdec ! " +
"videoconvert ! video/x-raw,format=RGB,framerate=0/1,width=224,height=224 ! tensor_converter ! " +
"other/tensor,format=static,dimension=(string)3:224:224:1,type=uint8,framerate=0/1 ! " +
"tensor_query_client host=192.168.50.38 port=" + document.getElementById('port').value + " dest-host=" + "192.168.50.191" + " " +
"dest-port=" + document.getElementById('port').value + " ! " +
"other/tensor,format=static,dimension=(string)1001:1,type=uint8,framerate=0/1 ! tensor_sink name=sinkx_remote";

var pHandle = tizen.ml.pipeline.createPipeline(pipelineDescription);
const pipelineDescription = 'appsrc caps=image/jpeg name=srcx_remote ! jpegdec ! ' +
'videoconvert ! video/x-raw,format=RGB,framerate=0/1,width=224,height=224 ! tensor_converter ! ' +
'other/tensor,format=static,dimension=(string)3:224:224:1,type=uint8,framerate=0/1 ! ' +
'tensor_query_client host=192.168.50.38 port=' + document.getElementById('port').value + ' dest-host=' + '192.168.50.191' + ' ' +
'dest-port=' + document.getElementById('port').value + ' ! ' +
'other/tensor,format=static,dimension=(string)1001:1,type=uint8,framerate=0/1 ! tensor_sink name=sinkx_remote';

const pHandle = tizen.ml.pipeline.createPipeline(pipelineDescription);
pHandle.start();

remoteSrc = pHandle.getSource('srcx');
remoteSrc = pHandle.getSource('srcx_remote');

pHandle.registerSinkListener('sinkx_remote', function(sinkName, data) {
var endTime = performance.now();
var label = document.querySelector('#label');
var tensorsRetData = data.getTensorRawData(0);
var maxIdx = GetMaxIdx(tensorsRetData.data);
const endTime = performance.now();
const label = document.getElementById('label_offloading');
const tensorsRetData = data.getTensorRawData(0);
const maxIdx = GetMaxIdx(tensorsRetData.data);
label.innerText = labels[maxIdx];

var time = document.querySelector('#offloading_time');
time.innerText = endTime - startTime + " ms"
const time = document.getElementById('time_offloading');
time.innerText = 'offloading : ' + (endTime - startTime) + ' ms'
});
}

window.onload = function() {
labels = loadLabelInfo();
function inference(src, canvas) {
const img_path = GetImgPath();
let img = new Image();
img.src = img_path;

img.onload = function () {
const fHandle = tizen.filesystem.openFile('wgt-package' + img_path, 'r');
const imgUInt8Array = fHandle.readData();
fHandle.close();

/* TODO : Change the image for each inference */
var fHandle = tizen.filesystem.openFile("wgt-package/res/0.jpg", 'r');
var imgUInt8Array = fHandle.readData();
fHandle.close();
const tensorsInfo = new tizen.ml.TensorsInfo();
tensorsInfo.addTensorInfo('tensor', 'UINT8', [imgUInt8Array.length]);
const tensorsData = tensorsInfo.getTensorsData();
tensorsData.setTensorRawData(0, imgUInt8Array);

var tensorsInfo = new tizen.ml.TensorsInfo();
tensorsInfo.addTensorInfo("tensor", "UINT8", [imgUInt8Array.length]);
var tensorsData = tensorsInfo.getTensorsData();
tensorsData.setTensorRawData(0, imgUInt8Array);
startTime = performance.now()
src.inputData(tensorsData);

const btnLocal = document.querySelector("#local");
tensorsData.dispose();
tensorsInfo.dispose();

btnLocal.addEventListener("click", function() {
const ctx = canvas.getContext('2d');
ctx.drawImage(img, 0, 0);
}
}

window.onload = function() {
labels = loadLabelInfo();

const btnLocal = document.getElementById('start_local');

btnLocal.addEventListener('click', function() {
runLocal();
});

const btnOffloading = document.querySelector("#offloading");
const btnOffloading = document.getElementById('start_offloading');

btnOffloading.addEventListener("click", function() {
btnOffloading.addEventListener('click', function() {
runRemote();
});

const btnLocalPush = document.querySelector("#localPush");
const localPage = document.getElementById('local');

btnLocalPush.addEventListener("click", function() {
startTime = performance.now()
localSrc.inputData(tensorsData);
localPage.addEventListener('click', function() {
if (localSrc) {
inference(localSrc, document.getElementById('canvas_local'));
}
});

const btnOffloadingPush = document.querySelector("#offloadingPush");
const offloadingPage = document.getElementById('offloading');

btnOffloadingPush.addEventListener("click", function() {
startTime = performance.now()
remoteSrc.inputData(tensorsData);
offloadingPage.addEventListener('click', function() {
if (remoteSrc) {
inference(remoteSrc, document.getElementById('canvas_offloading'));
}
});

/* add eventListener for tizenhwkey */
document.addEventListener('tizenhwkey', function(e) {
if (e.keyName === "back") {
if (e.keyName === 'back') {
try {
console.log("Pipeline is disposed!!");
console.log('Pipeline is disposed!!');
pHandle.stop();
pHandle.dispose();
tensorsData.dispose();
tensorsInfo.dispose();

tizen.application.getCurrentApplication().exit();
} catch (ignore) {}
Expand Down