Create CLAP demo (#442)

huggingface · Dec 8, 2023 · 8e49e5e · 8e49e5e
1 parent ff3019f
commit 8e49e5e
Show file tree

Hide file tree

Showing 8 changed files with 2,559 additions and 0 deletions.
diff --git a/examples/semantic-audio-search/.gitignore b/examples/semantic-audio-search/.gitignore
@@ -0,0 +1,24 @@
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+pnpm-debug.log*
+lerna-debug.log*
+
+node_modules
+dist
+dist-ssr
+*.local
+
+# Editor directories and files
+.vscode/*
+!.vscode/extensions.json
+.idea
+.DS_Store
+*.suo
+*.ntvs*
+*.njsproj
+*.sln
+*.sw?
diff --git a/examples/semantic-audio-search/index.html b/examples/semantic-audio-search/index.html
@@ -0,0 +1,26 @@
+<!doctype html>
+<html lang="en">
+
+<head>
+  <meta charset="UTF-8" />
+  <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+  <title>Semantic Audio Search | Transformers.js</title>
+  <link rel="stylesheet" href="./style.css" />
+</head>
+
+<body>
+  <div id="header">
+    <div id="title">In-browser Semantic Audio Search</div>
+    <p>Powered by <a href="https://hf.co/docs/transformers.js" target="_blank">🤗 Transformers.js</a></p>
+  </div>
+  <div id="overlay"></div>
+  <div id="deepscatter"></div>
+  <div id="search-bar">
+    <input id="query" placeholder="Search for music..." type="text" />
+    <button id="search"></button>
+  </div>
+</body>
+
+<script src="./index.js" type="module"></script>
+
+</html>
diff --git a/examples/semantic-audio-search/index.js b/examples/semantic-audio-search/index.js
@@ -0,0 +1,129 @@
+
+import Scatterplot from 'deepscatter';
+import { getCachedJSON } from './utils';
+
+// Start loading metadata and positions asynchronously as soon as possible.
+let metadata = {};
+getCachedJSON('https://huggingface.co/datasets/Xenova/MusicBenchEmbedded/resolve/main/metadata.json')
+    .then((data) => {
+        metadata = data;
+    })
+    .catch((e) => console.error(e));
+
+let positions = {};
+getCachedJSON('https://huggingface.co/datasets/Xenova/MusicBenchEmbedded/resolve/main/positions.json')
+    .then((data) => {
+        positions = data;
+    })
+    .catch((e) => console.error(e));
+
+const scatterplot = new Scatterplot('#deepscatter');
+window.scatterplot = scatterplot; // For debugging
+
+// Initial call
+scatterplot.plotAPI({
+    source_url: 'https://huggingface.co/datasets/Xenova/MusicBenchEmbedded/resolve/main/atlas',
+    max_points: 52768, // a full cap.
+    alpha: 35, // Target saturation for the full page.
+    zoom_balance: 0.5, // Rate at which points increase size. https://observablehq.com/@bmschmidt/zoom-strategies-for-huge-scatterplots-with-three-js
+    point_size: 3, // Default point size before application of size scaling
+    background_color: 'transparent',
+    encoding: {
+        // TODO Add colours
+        x: {
+            field: 'x',
+            transform: 'literal',
+        },
+        y: {
+            field: 'y',
+            transform: 'literal',
+        },
+        jitter_radius: {
+            method: 'uniform',
+            constant: 5,
+        },
+    },
+    tooltip_opacity: 1,
+    duration: 4000, // For slow initial transition
+}).then(_ => scatterplot.plotAPI({
+    encoding: {
+        jitter_radius: {
+            method: 'uniform',
+            constant: 0,
+        },
+    },
+}));
+
+// Custom hover function
+scatterplot.tooltip_html = (datum) => {
+    const item = metadata[datum.ix];
+    if (!item) return 'Loading...';
+    const location = item.location;
+
+    setTimeout(() => {
+        // Slight hack to append the audio element after the text
+        const tooltip = document.querySelector('.tooltip');
+        if (tooltip) {
+            tooltip.innerHTML = `
+            ${item.main_caption}
+            <br>
+            <audio id="tooltip-audio" controls src="https://huggingface.co/datasets/Xenova/MusicBenchEmbedded/resolve/main/audio/${location}"></audio>
+            `;
+        }
+    }, 0);
+    return item.main_caption;
+};
+
+// Make references to DOM elements
+const OVERLAY = document.getElementById('overlay');
+const INPUT_ELEMENT = document.getElementById('query');
+const SEARCH_BUTTON = document.getElementById('search');
+
+// Set up worker
+const worker = new Worker(new URL('./worker.js', import.meta.url), {
+    type: 'module'
+});
+worker.addEventListener('message', (e) => {
+    switch (e.data.status) {
+        case 'initiate':
+            OVERLAY.innerText = 'Loading model and embeddings database...';
+            OVERLAY.style.display = 'flex';
+            OVERLAY.style.pointerEvents = 'all';
+            break;
+        case 'ready':
+            OVERLAY.style.display = 'none';
+            OVERLAY.style.pointerEvents = 'none';
+            break;
+        case 'complete':
+            // Output is an array of [score, index] pairs. Get top item
+            const index = e.data.output[0][1];
+            const position = positions[index];
+
+            if (position) { // Just in case the position hasn't loaded yet (this should never happen)
+                // Zoom to result
+                scatterplot.plotAPI({
+                    zoom: {
+                        bbox: {
+                            x: [position[0] - 0.5, position[0] + 0.5],
+                            y: [position[1] - 0.5, position[1] + 0.5]
+                        }
+                    },
+                    duration: 2000,
+                })
+            }
+            break;
+    }
+});
+
+const search = () => {
+    worker.postMessage({
+        status: 'search',
+        query: INPUT_ELEMENT.value,
+    });
+};
+
+// Set up event listeners
+INPUT_ELEMENT.addEventListener('keypress', (event) => {
+    if (event.keyCode === 13) search();
+});
+SEARCH_BUTTON.addEventListener('click', search);