Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add CLAP demo (semantic audio search app) #442

Merged
merged 1 commit into from
Dec 8, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 24 additions & 0 deletions examples/semantic-audio-search/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
# Logs
logs
*.log
npm-debug.log*
yarn-debug.log*
yarn-error.log*
pnpm-debug.log*
lerna-debug.log*

node_modules
dist
dist-ssr
*.local

# Editor directories and files
.vscode/*
!.vscode/extensions.json
.idea
.DS_Store
*.suo
*.ntvs*
*.njsproj
*.sln
*.sw?
26 changes: 26 additions & 0 deletions examples/semantic-audio-search/index.html
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
<!doctype html>
<html lang="en">

<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Semantic Audio Search | Transformers.js</title>
<link rel="stylesheet" href="./style.css" />
</head>

<body>
<div id="header">
<div id="title">In-browser Semantic Audio Search</div>
<p>Powered by <a href="https://hf.co/docs/transformers.js" target="_blank">🤗 Transformers.js</a></p>
</div>
<div id="overlay"></div>
<div id="deepscatter"></div>
<div id="search-bar">
<input id="query" placeholder="Search for music..." type="text" />
<button id="search"></button>
</div>
</body>

<script src="./index.js" type="module"></script>

</html>
129 changes: 129 additions & 0 deletions examples/semantic-audio-search/index.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,129 @@

import Scatterplot from 'deepscatter';
import { getCachedJSON } from './utils';

// Start loading metadata and positions asynchronously as soon as possible.
let metadata = {};
getCachedJSON('https://huggingface.co/datasets/Xenova/MusicBenchEmbedded/resolve/main/metadata.json')
.then((data) => {
metadata = data;
})
.catch((e) => console.error(e));

let positions = {};
getCachedJSON('https://huggingface.co/datasets/Xenova/MusicBenchEmbedded/resolve/main/positions.json')
.then((data) => {
positions = data;
})
.catch((e) => console.error(e));

const scatterplot = new Scatterplot('#deepscatter');
window.scatterplot = scatterplot; // For debugging

// Initial call
scatterplot.plotAPI({
source_url: 'https://huggingface.co/datasets/Xenova/MusicBenchEmbedded/resolve/main/atlas',
max_points: 52768, // a full cap.
alpha: 35, // Target saturation for the full page.
zoom_balance: 0.5, // Rate at which points increase size. https://observablehq.com/@bmschmidt/zoom-strategies-for-huge-scatterplots-with-three-js
point_size: 3, // Default point size before application of size scaling
background_color: 'transparent',
encoding: {
// TODO Add colours
x: {
field: 'x',
transform: 'literal',
},
y: {
field: 'y',
transform: 'literal',
},
jitter_radius: {
method: 'uniform',
constant: 5,
},
},
tooltip_opacity: 1,
duration: 4000, // For slow initial transition
}).then(_ => scatterplot.plotAPI({
encoding: {
jitter_radius: {
method: 'uniform',
constant: 0,
},
},
}));

// Custom hover function
scatterplot.tooltip_html = (datum) => {
const item = metadata[datum.ix];
if (!item) return 'Loading...';
const location = item.location;

setTimeout(() => {
// Slight hack to append the audio element after the text
const tooltip = document.querySelector('.tooltip');
if (tooltip) {
tooltip.innerHTML = `
${item.main_caption}
<br>
<audio id="tooltip-audio" controls src="https://huggingface.co/datasets/Xenova/MusicBenchEmbedded/resolve/main/audio/${location}"></audio>
`;
}
}, 0);
return item.main_caption;
};

// Make references to DOM elements
const OVERLAY = document.getElementById('overlay');
const INPUT_ELEMENT = document.getElementById('query');
const SEARCH_BUTTON = document.getElementById('search');

// Set up worker
const worker = new Worker(new URL('./worker.js', import.meta.url), {
type: 'module'
});
worker.addEventListener('message', (e) => {
switch (e.data.status) {
case 'initiate':
OVERLAY.innerText = 'Loading model and embeddings database...';
OVERLAY.style.display = 'flex';
OVERLAY.style.pointerEvents = 'all';
break;
case 'ready':
OVERLAY.style.display = 'none';
OVERLAY.style.pointerEvents = 'none';
break;
case 'complete':
// Output is an array of [score, index] pairs. Get top item
const index = e.data.output[0][1];
const position = positions[index];

if (position) { // Just in case the position hasn't loaded yet (this should never happen)
// Zoom to result
scatterplot.plotAPI({
zoom: {
bbox: {
x: [position[0] - 0.5, position[0] + 0.5],
y: [position[1] - 0.5, position[1] + 0.5]
}
},
duration: 2000,
})
}
break;
}
});

const search = () => {
worker.postMessage({
status: 'search',
query: INPUT_ELEMENT.value,
});
};

// Set up event listeners
INPUT_ELEMENT.addEventListener('keypress', (event) => {
if (event.keyCode === 13) search();
});
SEARCH_BUTTON.addEventListener('click', search);
Loading
Loading