Create depth anything demo (#540)

huggingface · Jan 25, 2024 · 497628f · 497628f
1 parent a2fcd11
commit 497628f
Show file tree

Hide file tree

Showing 7 changed files with 1,791 additions and 0 deletions.
diff --git a/examples/depth-anything-client/.gitignore b/examples/depth-anything-client/.gitignore
@@ -0,0 +1,24 @@
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+pnpm-debug.log*
+lerna-debug.log*
+
+node_modules
+dist
+dist-ssr
+*.local
+
+# Editor directories and files
+.vscode/*
+!.vscode/extensions.json
+.idea
+.DS_Store
+*.suo
+*.ntvs*
+*.njsproj
+*.sln
+*.sw?
diff --git a/examples/depth-anything-client/index.html b/examples/depth-anything-client/index.html
@@ -0,0 +1,29 @@
+<!DOCTYPE html>
+<html lang="en">
+
+<head>
+  <meta charset="UTF-8" />
+  <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+  <title>Transformers.js - Depth Anything</title>
+</head>
+
+<body>
+  <h1>Depth Anything w/ 🤗 Transformers.js</h1>
+  <div id="container">
+    <label id="upload-button" for="upload">
+      <svg width="25" height="25" viewBox="0 0 25 25" fill="none" xmlns="http://www.w3.org/2000/svg">
+        <path fill="#000"
+          d="M3.5 24.3a3 3 0 0 1-1.9-.8c-.5-.5-.8-1.2-.8-1.9V2.9c0-.7.3-1.3.8-1.9.6-.5 1.2-.7 2-.7h18.6c.7 0 1.3.2 1.9.7.5.6.7 1.2.7 2v18.6c0 .7-.2 1.4-.7 1.9a3 3 0 0 1-2 .8H3.6Zm0-2.7h18.7V2.9H3.5v18.7Zm2.7-2.7h13.3c.3 0 .5 0 .6-.3v-.7l-3.7-5a.6.6 0 0 0-.6-.2c-.2 0-.4 0-.5.3l-3.5 4.6-2.4-3.3a.6.6 0 0 0-.6-.3c-.2 0-.4.1-.5.3l-2.7 3.6c-.1.2-.2.4 0 .7.1.2.3.3.6.3Z">
+        </path>
+      </svg>
+      Click to upload image
+      <label id="example">(or try example)</label>
+    </label>
+  </div>
+  <label id="status"></label>
+  <input id="upload" type="file" accept="image/*" />
+
+  <script type="module" src="/main.js"></script>
+</body>
+
+</html>
diff --git a/examples/depth-anything-client/main.js b/examples/depth-anything-client/main.js
@@ -0,0 +1,150 @@
+import './style.css';
+
+import * as THREE from 'three';
+import { OrbitControls } from 'three/addons/controls/OrbitControls.js';
+
+import { pipeline, env, RawImage } from '@xenova/transformers';
+
+// Since we will download the model from the Hugging Face Hub, we can skip the local model check
+env.allowLocalModels = false;
+
+// Proxy the WASM backend to prevent the UI from freezing
+env.backends.onnx.wasm.proxy = true;
+
+// Constants
+const EXAMPLE_URL = 'https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/bread_small.png';
+const DEFAULT_SCALE = 0.75;
+
+// Reference the elements that we will need
+const status = document.getElementById('status');
+const fileUpload = document.getElementById('upload');
+const imageContainer = document.getElementById('container');
+const example = document.getElementById('example');
+
+// Create a new depth-estimation pipeline
+status.textContent = 'Loading model...';
+const depth_estimator = await pipeline('depth-estimation', 'Xenova/depth-anything-small-hf');
+status.textContent = 'Ready';
+
+example.addEventListener('click', (e) => {
+    e.preventDefault();
+    predict(EXAMPLE_URL);
+});
+
+fileUpload.addEventListener('change', function (e) {
+    const file = e.target.files[0];
+    if (!file) {
+        return;
+    }
+
+    const reader = new FileReader();
+
+    // Set up a callback when the file is loaded
+    reader.onload = e2 => predict(e2.target.result);
+
+    reader.readAsDataURL(file);
+});
+
+let onSliderChange;
+
+// Predict depth map for the given image
+async function predict(url) {
+    imageContainer.innerHTML = '';
+    const image = await RawImage.fromURL(url);
+
+    // Set up scene and slider controls
+    const { canvas, setDisplacementMap } = setupScene(url, image.width, image.height);
+
+    imageContainer.append(canvas);
+
+    status.textContent = 'Analysing...';
+    const { depth } = await depth_estimator(image);
+
+    setDisplacementMap(depth.toCanvas());
+    status.textContent = '';
+
+    // Add slider control
+    const slider = document.createElement('input');
+    slider.type = 'range';
+    slider.min = 0;
+    slider.max = 1;
+    slider.step = 0.01;
+    slider.addEventListener('input', (e) => {
+        onSliderChange(parseFloat(e.target.value));
+    });
+    slider.defaultValue = DEFAULT_SCALE;
+    imageContainer.append(slider);
+}
+
+function setupScene(url, w, h) {
+
+    // Create new scene
+    const canvas = document.createElement('canvas');
+    const width = canvas.width = imageContainer.offsetWidth;
+    const height = canvas.height = imageContainer.offsetHeight;
+
+    const scene = new THREE.Scene();
+
+    // Create camera and add it to the scene
+    const camera = new THREE.PerspectiveCamera(30, width / height, 0.01, 10);
+    camera.position.z = 2;
+    scene.add(camera);
+
+    const renderer = new THREE.WebGLRenderer({ canvas, antialias: true });
+    renderer.setSize(width, height);
+    renderer.setPixelRatio(window.devicePixelRatio);
+
+    // Add ambient light
+    const light = new THREE.AmbientLight(0xffffff, 2);
+    scene.add(light);
+
+    // Load depth texture
+    const image = new THREE.TextureLoader().load(url);
+    image.colorSpace = THREE.SRGBColorSpace;
+    const material = new THREE.MeshStandardMaterial({
+        map: image,
+        side: THREE.DoubleSide,
+    });
+    material.displacementScale = DEFAULT_SCALE;
+
+    const setDisplacementMap = (canvas) => {
+        material.displacementMap = new THREE.CanvasTexture(canvas);
+        material.needsUpdate = true;
+    }
+
+    const setDisplacementScale = (scale) => {
+        material.displacementScale = scale;
+        material.needsUpdate = true;
+    }
+    onSliderChange = setDisplacementScale;
+
+    // Create plane and rescale it so that max(w, h) = 1
+    const [pw, ph] = w > h ? [1, h / w] : [w / h, 1];
+    const geometry = new THREE.PlaneGeometry(pw, ph, w, h);
+    const plane = new THREE.Mesh(geometry, material);
+    scene.add(plane);
+
+    // Add orbit controls
+    const controls = new OrbitControls(camera, renderer.domElement);
+    controls.enableDamping = true;
+
+    renderer.setAnimationLoop(() => {
+        renderer.render(scene, camera);
+        controls.update();
+    });
+
+    window.addEventListener('resize', () => {
+        const width = imageContainer.offsetWidth;
+        const height = imageContainer.offsetHeight;
+
+        camera.aspect = width / height;
+        camera.updateProjectionMatrix();
+
+        renderer.setSize(width, height);
+    }, false);
+
+    return {
+        canvas: renderer.domElement,
+        setDisplacementMap,
+    };
+}