feat: Add VisionCamera integration (imageFromFrame)

react-native-pytorch-core/android/src/main/cpp/torchlive/media/NativeJSRefBridge.cpp

@@ -18,6 +18,11 @@
 #include "./image/Image.h"
 #include "./image/JIImage.h"
 
+#if __has_include(<FrameHostObject.h>)
+#define HAS_VISION_CAMERA
+#include <FrameHostObject.h>
+#endif
+
 namespace torchlive {
 
 using namespace facebook::jni;
@@ -72,6 +77,23 @@ std::shared_ptr<IImage> imageFromFile(std::string filepath) {
   return std::make_shared<Image>(make_global(image));
 }
 
+std::shared_ptr<IImage> imageFromFrame(jsi::Runtime& runtime, jsi::Object frameHostObject) {
+#ifdef HAS_VISION_CAMERA
+  auto hostObject = frameHostObject.asHostObject<vision::FrameHostObject>(runtime);
+
+  auto mediaUtilsClass = getMediaUtilsClass();
+  auto imageFromImageProxyMethod =
+      mediaUtilsClass->getStaticMethod<local_ref<JIImage>(local_ref<JImageProxy>)>(
+          "imageFromImageProxy");
+  // TODO: Figure out how to get Context here
+  local_ref<JIImage> image =
+      imageFromImageProxyMethod(mediaUtilsClass, hostObject->frame, nullptr);
+  return std::make_shared<Image>(make_global(image));
+#else
+  throw jsi::JSError(runtime, "Error converting Frame to Image - VisionCamera is not properly installed!");
+#endif
+}
+
 std::shared_ptr<IImage>
 imageFromBlob(const Blob& blob, double width, double height) {
   auto mediaUtilsClass = getMediaUtilsClass();

react-native-pytorch-core/android/src/main/java/org/pytorch/rn/core/media/MediaUtils.java

@@ -59,6 +59,12 @@ public static IImage imageFromFile(final String filepath) {
     return new Image(bitmap);
   }
 
+  @DoNotStrip
+  @Keep
+  public static IImage imageFromImageProxy(final ImageProxy imageProxy, Context context) {
+    return new Image(imageProxy, context);
+  }
+
   @DoNotStrip
   @Keep
   public static IImage imageFromBlob(

react-native-pytorch-core/cxx/src/torchlive/media/BlobHostObject.cpp

@@ -28,80 +28,95 @@ jsi::Value BlobObjectWithNoData(jsi::Runtime& runtime) {
 
 } // namespace
 
-static jsi::Value arrayBufferImpl(
-    jsi::Runtime& runtime,
-    const jsi::Value& thisValue,
-    const jsi::Value* arguments,
-    size_t count) {
-  utils::ArgumentParser args(runtime, thisValue, arguments, count);
-  const auto& blob = args.thisAsHostObject<BlobHostObject>()->blob;
-  auto promiseValue = torchlive::createPromiseAsJSIValue(
-      runtime,
-      [&blob](jsi::Runtime& rt, std::shared_ptr<torchlive::Promise> promise) {
-        auto buffer = blob->getDirectBytes();
-        auto size = blob->getDirectSize();
-        jsi::ArrayBuffer arrayBuffer =
-            rt.global()
-                .getPropertyAsFunction(rt, "ArrayBuffer")
-                .callAsConstructor(rt, static_cast<int>(size))
-                .asObject(rt)
-                .getArrayBuffer(rt);
-        std::memcpy(arrayBuffer.data(rt), buffer, size);
-        auto typedArray = rt.global()
-                              .getPropertyAsFunction(rt, "Uint8Array")
-                              .callAsConstructor(rt, std::move(arrayBuffer))
-                              .asObject(rt);
-        promise->resolve(std::move(typedArray));
-      });
-  return promiseValue;
-}
-
-static jsi::Value sliceImpl(
-    jsi::Runtime& runtime,
-    const jsi::Value& thisValue,
-    const jsi::Value* arguments,
-    size_t count) {
-  utils::ArgumentParser args(runtime, thisValue, arguments, count);
-  const auto& blob = args.thisAsHostObject<BlobHostObject>()->blob;
-  auto blobSize = static_cast<int>(blob->getDirectSize());
-
-  // Default values
-  int start = 0;
-  int end = blobSize;
-
-  // Optinal inputs
-  if (args.count() > 0) {
-    start = args.asInteger(0);
-  }
-  if (args.count() > 1) {
-    end = args.asInteger(1);
-  }
-
-  // Invalid cases
-  if (std::abs(start) > blobSize || std::abs(end) > blobSize) {
-    return BlobObjectWithNoData(runtime);
-  }
+namespace {
 
-  if (start < 0) {
-    start = blobSize + start;
-  }
-  if (end < 0) {
-    end = blobSize + end;
-  }
+jsi::Value arrayBufferImpl(
+                                  jsi::Runtime& runtime,
+                                  const jsi::Value& thisValue,
+                                  const jsi::Value* arguments,
+                                  size_t count) {
+    utils::ArgumentParser args(runtime, thisValue, arguments, count);
+    const auto& blob = args.thisAsHostObject<BlobHostObject>()->blob;
+    auto promiseValue = torchlive::createPromiseAsJSIValue(
+                                                           runtime,
+                                                           [&blob](jsi::Runtime& rt, std::shared_ptr<torchlive::Promise> promise) {
+                                                               auto buffer = blob->getDirectBytes();
+                                                               auto size = blob->getDirectSize();
+                                                               jsi::ArrayBuffer arrayBuffer =
+                                                               rt.global()
+                                                                   .getPropertyAsFunction(rt, "ArrayBuffer")
+                                                                   .callAsConstructor(rt, static_cast<int>(size))
+                                                                   .asObject(rt)
+                                                                   .getArrayBuffer(rt);
+                                                               std::memcpy(arrayBuffer.data(rt), buffer, size);
+                                                               auto typedArray = rt.global()
+                                                                   .getPropertyAsFunction(rt, "Uint8Array")
+                                                                   .callAsConstructor(rt, std::move(arrayBuffer))
+                                                                   .asObject(rt);
+                                                               promise->resolve(std::move(typedArray));
+                                                           });
+    return promiseValue;
+}
 
-  // More invalid cases
-  if (start >= end) {
-    return BlobObjectWithNoData(runtime);
-  }
+jsi::Value sliceImpl(
+                            jsi::Runtime& runtime,
+                            const jsi::Value& thisValue,
+                            const jsi::Value* arguments,
+                            size_t count) {
+    utils::ArgumentParser args(runtime, thisValue, arguments, count);
+    const auto& blob = args.thisAsHostObject<BlobHostObject>()->blob;
+    auto blobSize = static_cast<int>(blob->getDirectSize());
+
+    // Default values
+    int start = 0;
+    int end = blobSize;
+
+    // Optinal inputs
+    if (args.count() > 0) {
+        start = args.asInteger(0);
+    }
+    if (args.count() > 1) {
+        end = args.asInteger(1);
+    }
+
+    // Invalid cases
+    if (std::abs(start) > blobSize || std::abs(end) > blobSize) {
+        return BlobObjectWithNoData(runtime);
+    }
+
+    if (start < 0) {
+        start = blobSize + start;
+    }
+    if (end < 0) {
+        end = blobSize + end;
+    }
+
+    // More invalid cases
+    if (start >= end) {
+        return BlobObjectWithNoData(runtime);
+    }
+
+    // Implement slice(start, end)
+    auto size = end - start;
+    auto buffer = std::unique_ptr<uint8_t[]>(new uint8_t[size]);
+    std::memcpy(buffer.get(), blob->getDirectBytes() + start, size);
+
+    auto blobHostObject = std::make_shared<BlobHostObject>(
+                                                           runtime, std::make_unique<Blob>(std::move(buffer), size));
+    return jsi::Object::createFromHostObject(runtime, std::move(blobHostObject));
+}
 
-  // Implement slice(start, end)
-  auto size = end - start;
-  auto buffer = std::unique_ptr<uint8_t[]>(new uint8_t[size]);
-  std::memcpy(buffer.get(), blob->getDirectBytes() + start, size);
+jsi::Value releaseImpl(
+                            jsi::Runtime& runtime,
+                            const jsi::Value& thisValue,
+                            const jsi::Value* arguments,
+                     size_t count) {
+utils::ArgumentParser args(runtime, thisValue, arguments, count);
+    args.requireNumArguments(0);
+    args.thisAsHostObject<BlobHostObject>()->blob = nullptr;
+    return jsi::Value::undefined();
+}
 
-  auto blobHostObject = std::make_shared<BlobHostObject>(
-      runtime, std::make_unique<Blob>(std::move(buffer), size));
-  return jsi::Object::createFromHostObject(runtime, std::move(blobHostObject));
 }
 
 BlobHostObject::BlobHostObject(
@@ -116,6 +131,7 @@ BlobHostObject::BlobHostObject(
   // Functions
   setPropertyHostFunction(runtime, "arrayBuffer", 0, arrayBufferImpl);
   setPropertyHostFunction(runtime, "slice", 0, sliceImpl);
+  setPropertyHostFunction(runtime, "release", 0, releaseImpl);
 }
 
 } // namespace media

react-native-pytorch-core/cxx/src/torchlive/media/MediaNamespace.cpp

@@ -170,6 +170,26 @@ jsi::Value imageFromTensorImpl(
       runtime, std::move(image));
 }
 
+jsi::Value imageFromFrameImpl(
+    jsi::Runtime& runtime,
+    const jsi::Value& thisValue,
+    const jsi::Value* arguments,
+    size_t count) {
+  auto args = utils::ArgumentParser(runtime, thisValue, arguments, count);
+  args.requireNumArguments(1);
+
+  std::shared_ptr<IImage> image;
+  try {
+    image = torchlive::media::imageFromFrame(runtime, args[0].asObject(runtime));
+  } catch (const std::exception& e) {
+    throw jsi::JSError(
+        runtime,
+        std::string("error on converting frame to image!\n") + e.what());
+  }
+  return utils::helpers::createFromHostObject<ImageHostObject>(
+      runtime, std::move(image));
+}
+
 jsi::Value toBlobImpl(
     jsi::Runtime& runtime,
     const jsi::Value& thisValue,
@@ -225,6 +245,7 @@ jsi::Object buildNamespace(jsi::Runtime& rt, RuntimeExecutor rte) {
   jsi::Object ns(rt);
   setPropertyHostFunction(rt, ns, "imageFromBlob", 3, imageFromBlobImpl);
   setPropertyHostFunction(rt, ns, "imageFromTensor", 1, imageFromTensorImpl);
+  setPropertyHostFunction(rt, ns, "imageFromFrame", 1, imageFromFrameImpl);
   setPropertyHostFunction(rt, ns, "imageFromFile", 1, imageFromFileImpl);
   setPropertyHostFunction(rt, ns, "toBlob", 1, toBlobImpl);
   setPropertyHostFunction(rt, ns, "imageToFile", 1, imageToFileImpl);

react-native-pytorch-core/cxx/src/torchlive/media/NativeJSRefBridge.h

@@ -18,6 +18,8 @@ namespace torchlive {
 
 namespace media {
 
+using namespace facebook;
+
 /**
  * The resolveNativeJSRefToImage_DO_NOT_USE function is needed to resolve
  * NativeJSRef objects to IImage. This function will be removed without
@@ -35,6 +37,9 @@ std::shared_ptr<IImage> imageFromFile(std::string filepath);
 std::shared_ptr<IImage>
 imageFromBlob(const Blob& blob, double width, double height);
 
+std::shared_ptr<IImage>
+imageFromFrame(jsi::Runtime& runtime, jsi::Object frameHostObject);
+
 std::unique_ptr<torchlive::media::Blob> toBlob(const std::string& refId);
 
 std::unique_ptr<torchlive::media::Blob> toBlob(std::shared_ptr<IImage> image);

react-native-pytorch-core/cxx/src/torchlive/media/NativeJSRefBridgeCxx.cpp

@@ -15,6 +15,8 @@ namespace torchlive {
 
 namespace media {
 
+using namespace facebook;
+
 std::shared_ptr<IImage> resolveNativeJSRefToImage_DO_NOT_USE(
     const std::string& refId) {
   return nullptr;
@@ -35,6 +37,10 @@ std::shared_ptr<IImage> imageFromFile(std::string filepath) {
   return nullptr;
 }
 
+std::shared_ptr<IImage> imageFromFrame(jsi::Runtime& runtime, jsi::Object frameHostObject) {
+  return nullptr;
+}
+
 std::unique_ptr<torchlive::media::Blob> toBlob(const std::string& refId) {
   size_t const size = 0;
   auto data = std::unique_ptr<uint8_t[]>(0);

react-native-pytorch-core/cxx/src/torchlive/media/image/ImageHostObject.cpp

@@ -81,42 +81,20 @@ jsi::Value scaleImpl(
   utils::ArgumentParser args(runtime, thisValue, arguments, count);
   args.requireNumArguments(2);
   const auto& image = args.thisAsHostObject<ImageHostObject>()->getImage();
-  auto promiseValue = torchlive::createPromiseAsJSIValue(
-      runtime,
-      [&image, sx = args[0].asNumber(), sy = args[1].asNumber()](
-          jsi::Runtime& rt, std::shared_ptr<torchlive::Promise> promise) {
-        auto scaledImage = image->scale(sx, sy);
-        auto imageObject =
-            utils::helpers::createFromHostObject<ImageHostObject>(
-                rt, std::move(scaledImage));
-        promise->resolve(std::move(imageObject));
-      });
-  return promiseValue;
+  double sx = args[0].asNumber();
+  double sy = args[1].asNumber();
+  auto scaledImage = image->scale(sx, sy);
+  return utils::helpers::createFromHostObject<ImageHostObject>(runtime, std::move(scaledImage));
 };
 
 jsi::Value releaseImpl(
     jsi::Runtime& runtime,
     const jsi::Value& thisValue,
     const jsi::Value* arguments,
     size_t count) {
-  auto image = thisValue.asObject(runtime)
-                   .asHostObject<ImageHostObject>(runtime)
-                   ->getImage();
-  auto promiseValue = torchlive::createPromiseAsJSIValue(
-      runtime,
-      [image](jsi::Runtime& rt, std::shared_ptr<torchlive::Promise> promise) {
-        try {
-          image->close();
-          promise->resolve(jsi::Value::undefined());
-        } catch (std::exception& e) {
-          promise->reject("error on release: " + std::string(e.what()));
-        } catch (const char* error) {
-          promise->reject("error on release: " + std::string(error));
-        } catch (...) {
-          promise->reject("error on release");
-        }
-      });
-  return promiseValue;
+    auto image = thisValue.asObject(runtime).asHostObject<ImageHostObject>(runtime);
+    image->release();
+    return jsi::Value::undefined();
 };
 
 } // namespace
@@ -143,5 +121,9 @@ std::shared_ptr<IImage> ImageHostObject::getImage() const noexcept {
   return image_;
 }
 
+void ImageHostObject::release() noexcept {
+  image_ = nullptr;
+}
+
 } // namespace media
 } // namespace torchlive

react-native-pytorch-core/cxx/src/torchlive/media/image/ImageHostObject.h

@@ -21,9 +21,10 @@ class JSI_EXPORT ImageHostObject : public common::BaseHostObject {
       std::shared_ptr<IImage> image);
 
   std::shared_ptr<IImage> getImage() const noexcept;
-
+  void release() noexcept;
+
  private:
-  std::shared_ptr<IImage> image_;
+    std::shared_ptr<IImage> image_;
 };
 
 } // namespace media

react-native-pytorch-core/cxx/src/torchlive/torch/TensorHostObject.cpp

@@ -433,6 +433,17 @@ jsi::Value permuteImpl(
       runtime, std::move(tensor));
 }
 
+jsi::Value releaseImpl(
+    jsi::Runtime& runtime,
+    const jsi::Value& thisValue,
+    const jsi::Value* arguments,
+    size_t count) {
+  utils::ArgumentParser args(runtime, thisValue, arguments, count);
+  args.requireNumArguments(0);
+  args.thisAsHostObject<TensorHostObject>()->tensor.reset();
+  return jsi::Value::undefined();
+}
+
 jsi::Value reshapeImpl(
     jsi::Runtime& runtime,
     const jsi::Value& thisValue,
@@ -672,6 +683,7 @@ TensorHostObject::TensorHostObject(jsi::Runtime& runtime, torch_::Tensor t)
   setPropertyHostFunction(runtime, "matmul", 1, matmulImpl);
   setPropertyHostFunction(runtime, "mul", 1, mulImpl);
   setPropertyHostFunction(runtime, "permute", 1, permuteImpl);
+  setPropertyHostFunction(runtime, "release", 0, releaseImpl);
   setPropertyHostFunction(runtime, "reshape", 1, reshapeImpl);
   setPropertyHostFunction(runtime, "softmax", 1, softmaxImpl);
   setPropertyHostFunction(runtime, "squeeze", 1, squeezeImpl);