diff --git a/.formatter.exs b/.formatter.exs
new file mode 100644
index 0000000..e945e12
--- /dev/null
+++ b/.formatter.exs
@@ -0,0 +1,5 @@
+[
+ import_deps: [:phoenix],
+ plugins: [Phoenix.LiveView.HTMLFormatter],
+ inputs: ["*.{heex,ex,exs}", "{config,lib,test}/**/*.{heex,ex,exs}"]
+]
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
new file mode 100644
index 0000000..00472fc
--- /dev/null
+++ b/.github/dependabot.yml
@@ -0,0 +1,8 @@
+version: 2
+updates:
+- package-ecosystem: mix
+ directory: "/"
+ schedule:
+ interval: daily
+ time: "07:00"
+ timezone: Europe/London
\ No newline at end of file
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
new file mode 100644
index 0000000..6b8979b
--- /dev/null
+++ b/.github/workflows/ci.yml
@@ -0,0 +1,35 @@
+name: Elixir CI
+
+on:
+ push:
+ branches: [ main ]
+ pull_request:
+ branches: [ main ]
+
+jobs:
+ build:
+ name: Build and test
+ runs-on: ubuntu-latest
+
+ steps:
+ - uses: actions/checkout@v2
+ - name: Set up Elixir
+ uses: erlef/setup-beam@v1
+ with:
+ elixir-version: '1.14.2' # Define the elixir version [required]
+ otp-version: '25.1.2' # Define the OTP version [required]
+ - name: Restore dependencies cache
+ uses: actions/cache@v2
+ with:
+ path: deps
+ key: ${{ runner.os }}-mix-${{ hashFiles('**/mix.lock') }}
+ restore-keys: ${{ runner.os }}-mix-
+ - name: Install dependencies
+ run: mix deps.get
+ - name: Run Tests
+ run: mix coveralls.json
+ env:
+ MIX_ENV: test
+
+ - name: Upload coverage to Codecov
+ uses: codecov/codecov-action@v1
\ No newline at end of file
diff --git a/.gitignore b/.gitignore
index b263cd1..8073c0c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,10 +1,37 @@
-/_build
-/cover
-/deps
-/doc
+# The directory Mix will write compiled artifacts to.
+/_build/
+
+# If you run "mix test --cover", coverage assets end up here.
+/cover/
+
+# The directory Mix downloads your dependencies sources to.
+/deps/
+
+# Where 3rd-party dependencies like ExDoc output generated docs.
+/doc/
+
+# Ignore .fetch files in case you like to edit your project deps locally.
/.fetch
+
+# If the VM crashes, it generates a dump, let's ignore it too.
erl_crash.dump
+
+# Also ignore archive artifacts (built via "mix archive.build").
*.ez
-*.beam
-/config/*.secret.exs
-.elixir_ls/
+
+# Temporary files, for example, from tests.
+/tmp/
+
+# Ignore package tarball (built via "mix hex.build").
+app-*.tar
+
+# Ignore assets that are produced by build tools.
+/priv/static/assets/
+
+# Ignore digested assets cache.
+/priv/static/cache_manifest.json
+
+# In case you use Node.js/npm, you want to ignore these.
+npm-debug.log
+/assets/node_modules/
+
diff --git a/README.md b/README.md
index f103a53..326ea5e 100644
--- a/README.md
+++ b/README.md
@@ -1,2 +1,1415 @@
-# image-classifier
-Classify images and attempt to extract data from or describe their contents
+
+
+# Image Captioning in `Elixir`
+
+
+[](https://codecov.io/github/dwyl/image-classifier?branch=main)
+[](https://github.com/dwyl/image-classifier/issues)
+[](https://hits.dwyl.com/dwyl/image-classifier)
+
+Caption your images using
+machine learning models
+within `Phoenix`!
+
+
+
+
+
+- [Image Captioning in `Elixir`](#image-captioning-in-elixir)
+- [Why? π€·](#why-)
+- [What? π](#what-)
+- [Who? π€](#who-)
+- [How? π»](#how-)
+ - [Prerequisites](#prerequisites)
+ - [0. Creating a fresh `Phoenix` project](#0-creating-a-fresh-phoenix-project)
+ - [1. Installing initial dependencies](#1-installing-initial-dependencies)
+ - [2. Adding `LiveView` capabilities to our project](#2-adding-liveview-capabilities-to-our-project)
+ - [3. Receiving image files](#3-receiving-image-files)
+- [4. Integrating `Bumblebee` π](#4-integrating-bumblebee-)
+ - [4.1 `Nx` configuration βοΈ](#41-nx-configuration-οΈ)
+ - [4.2 `Async` processing the image for classification](#42-async-processing-the-image-for-classification)
+ - [4.2.1 Considerations regarding `async` processes](#421-considerations-regarding-async-processes)
+ - [4.2.2 Alternative for better testing](#422-alternative-for-better-testing)
+ - [4.3 Image pre-processing](#43-image-pre-processing)
+ - [4.4 Updating the view](#44-updating-the-view)
+ - [4.5 Check it out!](#45-check-it-out)
+ - [4.6 Considerations on user images](#46-considerations-on-user-images)
+- [5. Final Touches](#5-final-touches)
+ - [5.1 Setting max file size](#51-setting-max-file-size)
+ - [5.2 Show errors](#52-show-errors)
+ - [5.3 Show image preview](#53-show-image-preview)
+- [6. What about other models?](#6-what-about-other-models)
+- [_Please_ Star the repo! βοΈ](#please-star-the-repo-οΈ)
+
+
+
+
+# Why? π€·
+
+Building our
+[app](https://github.com/dwyl/app),
+we consider `images` an _essential_
+medium of communication.
+
+By adding a way of captioning images,
+we make it *easy* for people
+to suggest meta tags to describe images
+so they become **searchable**.
+
+
+# What? π
+
+This run-through will create a simple
+`Phoenix` web application
+that will allow you to choose/drag an image
+and caption the image.
+
+
+# Who? π€
+
+This tutorial is aimed at `Phoenix` beginners
+that want to grasp how to do image captioning
+within a `Phoenix` application.
+
+If you are completely new to `Phoenix` and `LiveView`,
+we recommend you follow the **`LiveView` _Counter_ Tutorial**:
+[dwyl/phoenix-liveview-counter-tutorial](https://github.com/dwyl/phoenix-liveview-counter-tutorial)
+
+
+# How? π»
+
+In this chapter, we'll go over the development process
+of this small application.
+You'll learn how to do this *yourself*,
+so grab some coffee and let's get cracking!
+
+
+## Prerequisites
+
+This tutorial requires you have `Elixir` and `Phoenix` installed.
+If you you don't, please see
+[how to install Elixir](https://github.com/dwyl/learn-elixir#installation)
+and
+[Phoenix](https://hexdocs.pm/phoenix/installation.html#phoenix).
+
+This guide assumes you know the basics of `Phoenix`
+and have *some* knowledge of how it works.
+If you don't,
+we *highly suggest* you follow our other tutorials first.
+e.g:
+[github.com/dwyl/**phoenix-chat-example**](https://github.com/dwyl/phoenix-chat-example)
+
+In addition to this,
+**_some_ knowledge of `AWS`** -
+what it is, what an `S3` bucket is/does -
+**is assumed**.
+
+> **Note**: if you have questions or get stuck,
+> please open an issue!
+> [/dwyl/image-classifier/issues](https://github.com/dwyl/image-classifier/issues)
+
+
+## 0. Creating a fresh `Phoenix` project
+
+Let's create a fresh `Phoenix` project.
+Run the following command in a given folder:
+
+```sh
+mix phx.new . --app app --no-dashboard --no-ecto --no-gettext --no-mailer
+```
+
+We're running [`mix phx.new`](https://hexdocs.pm/phoenix/Mix.Tasks.Phx.New.html)
+to generate a new project without a dashboard
+and mailer (email) service,
+since we don't need those features in our project.
+
+After this,
+if you run `mix phx.server` to run your server,
+you should be able to see the following page.
+
+
+
+
+
+We're ready to start building.
+
+
+## 1. Installing initial dependencies
+
+Now that we're ready to go,
+let's start by adding some dependencies.
+
+Head over to `mix.exs`
+and add the following dependencies
+to the `deps` section.
+
+```elixir
+{:bumblebee, "~> 0.4.2"},
+{:exla, "~> 0.6.1"},
+{:nx, "~> 0.6.2"},
+{:vix, "~> 0.23.1"}
+```
+
+- [**`bumblebee`**](https://github.com/elixir-nx/bumblebee),
+a framework that will allows us to integrate
+[`Transformer Models`](https://huggingface.co/docs/transformers/index) in `Phoenix`.
+`Transformers` (from [Hugging Face](https://huggingface.co/))
+are APIs that allow us to easily download and train
+[pretrained models](https://blogs.nvidia.com/blog/2022/12/08/what-is-a-pretrained-ai-model).
+`Bumblebee` aims to support all Transformer Models,
+however some are lacking.
+You may check which ones are supported by visiting
+`Bumblebee`'s repository
+or visiting https://jonatanklosko-bumblebee-tools.hf.space/apps/repository-inspector
+and checking if the model is currently supported.
+
+- [**`EXLA`**](https://hexdocs.pm/exla/EXLA.html),
+Elixir implementation of [Google's XLA](https://www.tensorflow.org/xla/),
+a compiler that provides faster linear algebra calculations
+with `TensorFlow` models.
+This backend compiler is needed for [`Nx`](https://github.com/elixir-nx/nx),
+a framework that allows support for tensors and numerical definitions
+in Elixir.
+We are installing `EXLA` because allows us to compile models
+*just-in-time* and run them on CPU and/or GPU.
+
+- [**`Nx`**](https://hexdocs.pm/nx/Nx.html),
+a library that allows us to work with
+[`Numerical Elixir`](https://github.com/elixir-nx/),
+Elixir's way of doing [numerical computing](https://www.hilarispublisher.com/open-access/introduction-to-numerical-computing-2168-9679-1000423.pdf).
+
+- [**`Vix`**](https://hexdocs.pm/vix/readme.html),
+an image processing library.
+
+In `config/config.exs`,
+let's add our `:nx` configuration
+to use `EXLA`.
+
+```elixir
+config :nx, default_backend: EXLA.Backend
+```
+
+## 2. Adding `LiveView` capabilities to our project
+
+As it stands,
+our project is not using `LiveView`.
+Let's fix this.
+
+In `lib/app_web/router.ex`,
+change the `scope "/"` to the following.
+
+```elixir
+ scope "/", AppWeb do
+ pipe_through :browser
+
+ live "/", PageLive
+ end
+```
+
+Instead of using the `PageController`,
+we are going to be creating `PageLive`,
+a `LiveView` file.
+
+Let's create our `LiveView` files.
+Inside `lib/app_web`,
+create a folder called `live`
+and create the following file
+`page_live.ex`.
+
+```elixir
+defmodule AppWeb.PageLive do
+ use AppWeb, :live_view
+
+ @impl true
+ def mount(_params, _session, socket) do
+ {:ok, socket}
+ end
+end
+
+```
+
+This is a simple `LiveView` controller.
+
+In the same `live` folder,
+create a file called `page_live.html.heex`
+and use the following code.
+
+```html
+<.flash_group flash={@flash} />
+
+
+
+
+
+```
+
+This is a simple HTML form that uses
+[`Tailwind CSS`](https://github.com/dwyl/learn-tailwind)
+to enhance the presentation of the upload form.
+We'll also remove the unused header of the page layout,
+while we're at it.
+
+Locate the file `lib/app_web/components/layouts/app.html.heex`
+and remove the `` class.
+The file should only have the following code:
+
+```html
+
+
+
+```
+
+Now you can safely delete the `lib/app_web/controllers` folder,
+which is no longer used.
+
+If you run `mix phx.server`,
+you should see the following screen:
+
+
+
+
+
+This means we've successfully added `LiveView`
+and changed our view!
+
+
+## 3. Receiving image files
+
+Now, let's start by receiving some image files.
+In order to classify them, we need to have access to begin with,
+right?
+
+With `LiveView`,
+we can easily do this by using
+[`allow_upload/3`](https://hexdocs.pm/phoenix_live_view/Phoenix.LiveView.html#allow_upload/3)
+when mounting our `LiveView`.
+With this function, we can easily accept
+file uploads with progress.
+We can define file types, max number of entries,
+max file size,
+validate the uploaded file and much more!
+
+Firstly,
+let's make some changes to
+`lib/app_web/live/page_live.html.heex`.
+
+```html
+<.flash_group flash={@flash} />
+
+
+
+
+
Image Classification
+
+ Do simple captioning with this LiveView
+ demo, powered by Bumblebee.
+
+
+
+
+
+
+
+
+
+
+
+
or drag and drop
+
+
PNG, JPG, GIF up to 5MB
+
+
+
+
+
+
+
+
+```
+
+We've added a few features:
+
+- used
+[`<.live_file_input/>`](https://hexdocs.pm/phoenix_live_view/Phoenix.Component.html#live_file_input/1)
+for `LiveView` file upload.
+We've wrapped this component
+with an element that is annotated with the `phx-drop-target` attribute
+pointing to the DOM `id` of the file input.
+- because `<.live_file_input/>` is being used,
+we need to annotate its wrapping element
+with `phx-submit` and `phx-change`,
+as per
+[hexdocs.pm/phoenix_live_view/uploads.html#render-reactive-elements](https://hexdocs.pm/phoenix_live_view/uploads.html#render-reactive-elements)
+
+Because we've added these bindings,
+we need to add the event handlers in
+`lib/app_web/live/page_live.ex`.
+Open it and update it to:
+
+```elixir
+defmodule AppWeb.PageLive do
+ use AppWeb, :live_view
+
+ @impl true
+ def mount(_params, _session, socket) do
+ {:ok,
+ socket
+ |> assign(label: nil, running: false, task_ref: nil)
+ |> allow_upload(:image_list,
+ accept: ~w(image/*),
+ auto_upload: true,
+ progress: &handle_progress/3,
+ max_entries: 1,
+ chunk_size: 64_000
+ )}
+ end
+
+ @impl true
+ def handle_event("validate", _params, socket) do
+ {:noreply, socket}
+ end
+
+ @impl true
+ def handle_event("remove-selected", %{"ref" => ref}, socket) do
+ {:noreply, cancel_upload(socket, :image_list, ref)}
+ end
+
+ @impl true
+ def handle_event("save", _params, socket) do
+ {:noreply, socket}
+ end
+
+ defp handle_progress(:image_list, entry, socket) do
+ if entry.done? do
+ uploaded_file =
+ consume_uploaded_entry(socket, entry, fn %{} = _meta ->
+ {:ok, entry}
+ end)
+ end
+
+ {:noreply, socket}
+ end
+end
+```
+
+- when `mount/3`ing the LiveView,
+we are creating three socket assigns:
+`label` pertains to the model prediction;
+`running` is a boolean referring to whether the model is running or not;
+`task_ref` refers to the reference of the task that was created for image classification
+(we'll delve into this further later down the line).
+Additionally, we are using the `allow_upload/3` function to define our upload configuration.
+The most important settings here are `auto_upload` set to `true`
+and the `progress` fields.
+By configuring these two properties,
+we are telling `LiveView` that *whenever the person uploads a file*,
+**it is processed immediately and consumed**.
+
+- the `progress` field is handled by the `handle_progress/3` function.
+We consume the file in this function by using
+[`consume_uploaded_entry/3`](https://hexdocs.pm/phoenix_live_view/Phoenix.LiveView.html#consume_uploaded_entry/3).
+Whilst consuming the entry/file,
+we can access its path and then use its content.
+*For now*, we don't need to use it.
+But we will in the future to feed our image classifier with it!
+After the callback function is executed,
+this function "consumes the entry",
+essentially deleting the image from the temporary folder
+and removing it from the uploaded files list.
+
+- the `"validate"`, `"remove-selected"`, `"save"` event handlers
+are called whenever the person uploads the image,
+wants to remove it from the list of uploaded images
+and when wants to submit the form,
+respectively.
+You may see that we're not doing much with these handlers;
+we're simply replying with a `:noreply`
+because we don't need to do anything with them.
+
+And that's it!
+If you run `mix phx.server`,
+nothing will change.
+
+
+# 4. Integrating `Bumblebee` π
+
+Now here comes the fun part!
+It's time to do some image captioning! π
+
+
+## 4.1 `Nx` configuration βοΈ
+
+We first need to add some initial setup in the
+`lib/app/application.ex` file.
+Head over there and and change
+the `start` function like so:
+
+```elixir
+@impl true
+def start(_type, _args) do
+ children = [
+ # Start the Telemetry supervisor
+ AppWeb.Telemetry,
+ # Start the PubSub system
+ {Phoenix.PubSub, name: App.PubSub},
+ {Nx.Serving, serving: serving(), name: ImageClassifier},
+ # Start the Endpoint (http/https)
+ AppWeb.Endpoint
+ # Start a worker by calling: App.Worker.start_link(arg)
+ # {App.Worker, arg}
+ ]
+
+ # See https://hexdocs.pm/elixir/Supervisor.html
+ # for other strategies and supported options
+ opts = [strategy: :one_for_one, name: App.Supervisor]
+ Supervisor.start_link(children, opts)
+end
+
+def serving do
+ {:ok, model_info} = Bumblebee.load_model({:hf, "microsoft/resnet-50"})
+ {:ok, featurizer} = Bumblebee.load_featurizer({:hf, "microsoft/resnet-50"})
+
+ Bumblebee.Vision.image_classification(model_info, featurizer,
+ top_k: 1,
+ compile: [batch_size: 10],
+ defn_options: [compiler: EXLA]
+ )
+end
+```
+
+We are using
+[`Nx.Serving`](https://hexdocs.pm/nx/Nx.Serving.html),
+which simply allows us to encapsulates tasks,
+be it networking, machine learning, data processing or any other task.
+
+In this specific case,
+we are using it to **batch requests**.
+This is extremely useful and important
+because we are using models that typically run on
+[GPU](https://en.wikipedia.org/wiki/Graphics_processing_unit).
+The GPU is *really good* at **parallelizing tasks**.
+Therefore, instead of sending an image classification request one by one,
+we can *batch them*/bundle them together as much as we can
+and then send it over.
+
+We can define the `batch_size` and `batch_timeout` with `Nx.Serving`.
+We're going to use the default values,
+hence why we're not explicitly defining them.
+
+With `Nx.Serving`, we define a `serving/0` function
+that is then used by it,
+which in turn is executed in the supervision tree.
+
+In the `serving/0` function,
+we are loading the
+[`ResNet-50`](https://huggingface.co/microsoft/resnet-50)
+model
+and its featurizer.
+
+> **Note**:
+> A `featurizer` can be seen as a
+> [`Feature Extractor`](https://huggingface.co/docs/transformers/main_classes/feature_extractor).
+> It is essentially a component that is responsible for converting input data
+> into a format that can be processed by a pre-trained language model.
+>
+> It takes raw information and performs various transformations,
+> such as
+> [tokenization](https://neptune.ai/blog/tokenization-in-nlp),
+> [padding](https://www.baeldung.com/cs/deep-neural-networks-padding),
+> and encoding to prepare the data for model training or inference.
+
+Lastly, this function returns
+a builds serving for image classification
+by calling [`image_classification/3`](https://hexdocs.pm/bumblebee/Bumblebee.Vision.html#image_classification/3),
+where we can define our compiler and task batch size.
+We've given our serving function the name `ImageClassifier`.
+
+
+## 4.2 `Async` processing the image for classification
+
+Now we're ready to send the image to the model
+and get a prediction of it!
+
+Every time we upload an image,
+we are going to run **async processing**.
+This means that the task responsible for image classification
+will be created asynchronously,
+meaning that the LiveView *won't have to wait* for this task to finish
+to continue working.
+
+For this scenario,
+we are going to be using the
+[`Task` module](https://hexdocs.pm/elixir/1.14/Task.html)
+to spawn processes to complete this task.
+
+Go to `lib/app_web/live/page_live.ex`
+and change the following code.
+
+```elixir
+def handle_progress(:image_list, entry, socket) do
+ if entry.done? do
+
+ # Consume the entry and get the tensor to feed to classifier
+ tensor = consume_uploaded_entry(socket, entry, fn %{} = meta ->
+ {:ok, vimage} = Vix.Vips.Image.new_from_file(meta.path)
+ pre_process_image(vimage)
+ end)
+
+ # Create an async task to classify the image
+ task = Task.async(fn -> Nx.Serving.batched_run(ImageClassifier, tensor) end)
+
+ # Update socket assigns to show spinner whilst task is running
+ {:noreply, assign(socket, running: true, task_ref: task.ref)}
+ else
+ {:noreply, socket}
+ end
+end
+
+@impl true
+def handle_info({ref, result}, %{assigns: %{task_ref: ref}} = socket) do
+ # This is called everytime an Async Task is created.
+ # We flush it here.
+ Process.demonitor(ref, [:flush])
+
+ # And then destructure the result from the classifier.
+ %{predictions: [%{label: label}]} = result
+
+ # Update the socket assigns with result and stopping spinner.
+ {:noreply, assign(socket, label: label, running: false)}
+end
+```
+
+> **Note:** The `pre_process_image/1` function is yet to be defined.
+> We'll do that in the following section.
+
+In the `handle_progress/3` function,
+whilst we are consuming the image,
+we are first converting it to a
+[`Vix.Vips.Image`](https://hexdocs.pm/vix/Vix.Vips.Image.html) `Struct`
+using the file path.
+We then feed this image to the `pre_process_image/1` function that we'll implement later.
+
+What's important is to notice this line:
+
+```elixir
+task = Task.async(fn -> Nx.Serving.batched_run(ImageClassifier, tensor) end)
+```
+
+We are using
+[`Task.async/1`](https://hexdocs.pm/elixir/1.12/Task.html#async/1)
+to call our `Nx.Serving` build function `ImageClassifier` we've defined earlier,
+thus initiating a batched run with the image tensor.
+While the task is spawned,
+we update the socket assigns with the reference to the task (`:task_ref`)
+and update the `:running` assign to `true`,
+so we can show a spinner or a loading animation.
+
+When the task is spawned using `Task.async/1`,
+a couple of things happen in the background.
+The new process is monitored by the caller (our `LiveView`),
+which means that the caller will receive a
+`{:DOWN, ref, :process, object, reason}`
+message once the process it is monitoring dies.
+And, a link is created between both processes.
+
+Therefore,
+we **don't need to use**
+[**`Task.await/2`**](https://hexdocs.pm/elixir/1.12/Task.html#await/2).
+Instead, we create a new handler to receive the aforementioned.
+That's what we're doing in the
+`handle_info({ref, result}, %{assigns: %{task_ref: ref}} = socket)` function.
+The received message contains a `{ref, result}` tuple,
+where `ref` is the monitorβs reference.
+We use this reference to stop monitoring the task,
+since we received the result we needed from our task
+and we can discard an exit message.
+
+In this same function, we destructure the prediction
+from the model and assign it to the socket assign `:label`
+and set `:running` to `false`.
+
+Quite beautiful, isn't it?
+With this, we don't have to worry if the person closes the browser tab.
+The process dies (as does our `LiveView`),
+and the work is automatically cancelled,
+meaning no resources are spent
+on a process for which nobody expects a result anymore.
+
+
+### 4.2.1 Considerations regarding `async` processes
+
+When a task is spawned using `Task.async/2`,
+**it is linked to the caller**.
+Which means that they're related:
+if one dies, the other does too.
+
+We ought to take this into account when developing our application.
+If we don't have control over the result of the task,
+and we don't want our `LiveView` to crash if the task crashes,
+we must use a different alternative to spawn our task -
+[`Task.Supervisor.async_nolink/3`](https://hexdocs.pm/elixir/1.14/Task.Supervisor.html#async_nolink/3)
+can be used for this effect,
+meaning we can use it if we want to make sure
+our `LiveView` won't die and the error is reported,
+even if the task crashes.
+
+We've chosen `Task.async/2` for this very reason.
+We are doing something **that takes time/is expensive**
+and we **want to stop the task if `LiveView` is closed/crashes**.
+However, if you are building something
+like a report that has to be generated even if the person closes the browser tab,
+this is not the right solution.
+
+
+### 4.2.2 Alternative for better testing
+
+We are spawning async tasks by calling `Task.async/1`.
+This is creating an **_unsupervised_ task**.
+Although it's plausible for this simple app,
+it's best for us to create a
+[**`Supervisor`**](https://hexdocs.pm/elixir/1.15.7/Supervisor.html)
+that manages their child tasks.
+This gives more control over the execution
+and lifetime of the child tasks.
+
+Additionally, it's better to have these tasks supervised
+because it makes it possible to create tests for our `LiveView`.
+For this, we need to make a couple of changes.
+
+First, head over to `lib/app/application.ex`
+and add a supervisor to the `start/2` function children array.
+
+```elixir
+def start(_type, _args) do
+ children = [
+ AppWeb.Telemetry,
+ {Phoenix.PubSub, name: App.PubSub},
+ {Nx.Serving, serving: serving(), name: ImageClassifier},
+ {Task.Supervisor, name: App.TaskSupervisor}, # add this line
+ AppWeb.Endpoint
+ ]
+
+ opts = [strategy: :one_for_one, name: App.Supervisor]
+ Supervisor.start_link(children, opts)
+end
+```
+
+We are creating a [`Task.Supervisor`](https://hexdocs.pm/elixir/Supervisor.html)
+with the name `App.TaskSupervisor`.
+
+Now, in `lib/app_web/live/page_live.ex`,
+we create the async task like so:
+
+```elixir
+task = Task.Supervisor.async(App.TaskSupervisor, fn -> Nx.Serving.batched_run(ImageClassifier, tensor) end)
+```
+
+We are now using
+[`Task.Supervisor.async`](https://hexdocs.pm/elixir/1.15.7/Task.Supervisor.html#async/3),
+passing the name of the supervisor defined earlier.
+
+And that's it!
+We are creating async tasks like before,
+the only difference is that they're now **supervised**.
+
+In tests, you can create a small module that waits for the tasks to be completed.
+
+```elixir
+defmodule AppWeb.SupervisorSupport do
+
+ @moduledoc """
+ This is a support module helper that is meant to wait for all the children of a supervisor to complete.
+ If you go to `lib/app/application.ex`, you'll see that we created a `TaskSupervisor`, where async tasks are spawned.
+ This module helps us to wait for all the children to finish during tests.
+ """
+
+ @doc """
+ Find all children spawned by this supervisor and wait until they finish.
+ """
+ def wait_for_completion() do
+ pids = Task.Supervisor.children(App.TaskSupervisor)
+ Enum.each(pids, &Process.monitor/1)
+ wait_for_pids(pids)
+ end
+
+ defp wait_for_pids([]), do: nil
+ defp wait_for_pids(pids) do
+ receive do
+ {:DOWN, _ref, :process, pid, _reason} -> wait_for_pids(List.delete(pids, pid))
+ end
+ end
+end
+```
+
+You can call `AppWeb.SupervisorSupport.wait_for_completion()`
+in unit tests so they wait for the tasks to complete.
+In our case,
+we do that until the *prediction is made*.
+
+
+## 4.3 Image pre-processing
+
+As we've noted before,
+we need to **pre-process the image before passing it to the model**.
+For this, we have three main steps:
+
+- removing the [`alpha` ](https://en.wikipedia.org/wiki/Alpha_compositing)
+out of the image, flattening it out.
+- convert the image to `sRGB` [colourspace](https://en.wikipedia.org/wiki/Color_space).
+This is needed to ensure that the image is consistent
+and aligns with the model's training data images.
+- set the representation of the image as a `Tensor`
+to `height, width, bands`.
+The image tensor will then be organized as a three-dimensional array,
+where the first dimension represents the height of the image,
+the second refers to the width of the image,
+and the third pertains to the different
+[spectral bands/channels of the image](https://en.wikipedia.org/wiki/Multispectral_imaging).
+
+Our `pre_process_image/1` function will implement these three steps.
+Let's implement it now!
+In `lib/app_web/live/page_live.ex`,
+add the following:
+
+```elixir
+ defp pre_process_image(%Vimage{} = image) do
+
+ # If the image has an alpha channel, flatten it:
+ {:ok, flattened_image} = case Vix.Vips.Image.has_alpha?(image) do
+ true -> Vix.Vips.Operation.flatten(image)
+ false -> {:ok, image}
+ end
+
+ # Convert the image to sRGB colourspace ----------------
+ {:ok, srgb_image} = Vix.Vips.Operation.colourspace(flattened_image, :VIPS_INTERPRETATION_sRGB)
+
+ # Converting image to tensor ----------------
+ {:ok, tensor} = Vix.Vips.Image.write_to_tensor(srgb_image)
+
+ # We reshape the tensor given a specific format.
+ # In this case, we are using {height, width, channels/bands}.
+ %Vix.Tensor{data: binary, type: type, shape: {x, y, bands}} = tensor
+ format = [:height, :width, :bands]
+ shape = {x, y, bands}
+
+ final_tensor =
+ binary
+ |> Nx.from_binary(type)
+ |> Nx.reshape(shape, names: format)
+
+ {:ok, final_tensor}
+ end
+```
+
+The function receives a `Vix` image,
+as detailed earlier.
+We use [`flatten/1`](https://hexdocs.pm/vix/Vix.Vips.Operation.html#flatten/2)
+to flatten the alpha out of the image.
+
+The resulting image has its colourspaced changed
+by calling [`colourspace/3`](https://hexdocs.pm/vix/Vix.Vips.Operation.html#colourspace/3),
+where we change the to `sRGB`.
+
+The colourspace-altered image is then converted to a
+[tensor](https://hexdocs.pm/vix/Vix.Tensor.html),
+by calling
+[`write_to_tensor/1`](https://hexdocs.pm/vix/Vix.Vips.Image.html#write_to_tensor/1).
+
+We then
+[reshape](https://hexdocs.pm/nx/Nx.html#reshape/3)
+the tensor according to the format that was previously mentioned.
+
+This function returns the processed tensor,
+that is then used as input to the model.
+
+
+## 4.4 Updating the view
+
+All that's left is updating the view
+to reflect these changes we've made to the `LiveView`.
+Head over to `lib/app_web/live/page_live.html.heex`
+and change it to this.
+
+```html
+<.flash_group flash={@flash} />
+
+
+
+
+
Image Classification
+
+ Do simple classification with this LiveView
+ demo, powered by Bumblebee.
+
+
+
+
+
+
+
+
+
+
+
or drag and drop
+
+
PNG, JPG, GIF up to 5MB
+
+
+
+
+
+
+
+ Description:
+
+ <%= if @running do %>
+
+
+
+
+
+ <% else %>
+ <%= if @label do %>
+ <%= @label %>
+ <% else %>
+ Waiting for image input.
+ <% end %>
+ <% end %>
+
+
+
+
+
+
+```
+
+In these changes,
+we've added the output of the model in the form of text.
+We are rendering a spinner
+if the `:running` socket assign is set to true.
+Otherwise,
+we add the `:label`, which holds the prediction made by the model.
+
+You may have also noticed that
+we've changed the `phx` event handlers
+to `noop`.
+This is simply to simplify the `LiveView`.
+
+Head over to `lib/app_web/live/page_live.ex`.
+You can now remove the `"validate"`, `"save"`
+and `"remove-selected"` handlers,
+because we're not going to be needing them.
+Replace them with this handler:
+
+```elixir
+ @impl true
+ def handle_event("noop", _params, socket) do
+ {:noreply, socket}
+ end
+```
+
+
+## 4.5 Check it out!
+
+And that's it!
+Our app is now *functional* π.
+
+If you run the app,
+you can drag and drop or select an image.
+After this, a task will be spawned that will run the model
+against the image that was submitted.
+
+Once a prediction is made, display it!
+
+
+
+
+
+You can and **should** try other models.
+`ResNet-50` is just one of the many that are supported by `Bumblebee`.
+You can see the supported models in https://github.com/elixir-nx/bumblebee#model-support.
+
+
+## 4.6 Considerations on user images
+
+To keep the app as simple as possible,
+we are receiving the image from the person as is.
+Although we are processing the image,
+we are doing it so **it is processable by the model**.
+
+We have to understand that:
+- in most cases, **full-resolution images are not necessary**,
+because neural networks work on much smaller inputs
+(e.g. `ResNet-50` works with `224px x 224px` images).
+This means that a lot of data is unnecessarily uploaded over the network,
+increasing workload on the server to potentially downsize a large image.
+- decoding an image requires an additional package,
+meaning more work on the server.
+
+We can avoid both of these downsides by moving this work to the client.
+We can leverage the
+[`Canvas API` ](https://developer.mozilla.org/en-US/docs/Web/API/Canvas_API)
+to decode and downsize this image in the client-side,
+reducing server workload.
+
+You can see an example implementation of this technique
+in `Bumblebee`'s repository
+at https://github.com/elixir-nx/bumblebee/blob/main/examples/phoenix/image_classification.exs
+
+However, since we are not using `JavaScript` for anything,
+we can (and *should*!) properly downsize our images
+so they better fit the training dataset of the model we use.
+This will allow the model to process faster,
+since larger images carry over more data that is ultimately unnecessary
+for models to make predictions.
+
+Open
+`lib/app_web/live/page_live.ex`,
+find the `handle_progress/3` function
+and change resize the image *before processing it*.
+
+```elixir
+ file_binary = File.read!(meta.path)
+
+ # Get image and resize
+ # This is dependant on the resolution of the model's dataset.
+ # In our case, we want the width to be closer to 640, whilst maintaining aspect ratio.
+ width = 640
+ {:ok, thumbnail_vimage} = Vix.Vips.Operation.thumbnail(meta.path, width, size: :VIPS_SIZE_DOWN)
+
+ # Pre-process it
+ {:ok, tensor} = pre_process_image(thumbnail_vimage)
+
+ #...
+```
+
+We are using
+[`Vix.Vips.Operation.thumbnail/3`](https://hexdocs.pm/vix/Vix.Vips.Operation.html#thumbnail/3)
+to resize our image to a fixed width
+whilst maintaining aspect ratio.
+The `width` variable can be dependent on the model that you use.
+For example, `ResNet-50` is trained on `224px224` pictures,
+so you may want to resize the image to this width.
+
+> **Note**: We are using the `thumbnail/3` function
+> instead of `resize/3` because it's _much_ faster.
+> Check
+> https://github.com/libvips/libvips/wiki/HOWTO----Image-shrinking
+> to know why.
+
+# 5. Final Touches
+
+Although our app is functional,
+we can make it **better**. π¨
+
+## 5.1 Setting max file size
+
+In order to better control user input,
+we should add a limit to the size of the image that is being uploaded.
+It will be easier on our server and ultimately save costs.
+
+Let's add a cap of `5MB` to our app!
+Fortunately for you, this is super simple!
+You just need to add the `max_file_size`
+to the `allow_uploads/2` function
+when mounting the `LiveView`!
+
+```elixir
+ def mount(_params, _session, socket) do
+ {:ok,
+ socket
+ |> assign(label: nil, running: false, task_ref: nil)
+ |> allow_upload(:image_list,
+ accept: ~w(image/*),
+ auto_upload: true,
+ progress: &handle_progress/3,
+ max_entries: 1,
+ chunk_size: 64_000,
+ max_file_size: 5_000_000 # add this
+ )}
+ end
+```
+
+And that's it!
+The number is in `bytes`,
+hence why we set it as `5_000_000`.
+
+
+## 5.2 Show errors
+
+In case a person uploads an image that is too large,
+we should show this feedback to the person!
+
+For this, we can leverage the
+[`upload_errors/2`](https://hexdocs.pm/phoenix_live_view/Phoenix.Component.html#upload_errors/2)
+function.
+This function will return the entry errors for an upload.
+We need to add an handler for one of these errors to show it first.
+
+Head over `lib/app_web/live/page_live.ex`
+and add the following line.
+
+```elixir
+ def error_to_string(:too_large), do: "Image too large. Upload a smaller image up to 5MB."
+```
+
+Now, add the following section below the upload form
+inside `lib/app_web/live/page_live.html.heex`.
+
+```html
+
+<%= for entry <- @uploads.image_list.entries do %>
+
+ <%= for err <- upload_errors(@uploads.image_list, entry) do %>
+
+
+
+
+
+
+
+ <%= error_to_string(err) %>
+
+
+
+
+ <% end %>
+
+<% end %>
+```
+
+We are iterating over the errors returned by `upload_errors/2`
+and invoking `error_to_string/1`,
+which we've just defined in our `LiveView`.
+
+Now, if you run the app
+and try to upload an image that is too large,
+an error will show up.
+
+Awesome! π
+
+
+
+
+
+
+
+## 5.3 Show image preview
+
+As of now, even though our app predicts the given images,
+it is not showing a preview of the image the person submitted.
+Let's fix this π οΈ.
+
+Let's add a new socket assign variable
+pertaining to the [base64](https://en.wikipedia.org/wiki/Base64) representation
+of the image in `lib/app_web/live_page/live.ex`
+
+```elixir
+ |> assign(label: nil, running: false, task_ref: nil, image_preview_base64: nil)
+```
+
+We've added `image_preview_base64`
+as a new socket assign,
+initializing it as `nil`.
+
+Next, we need to *read the file while consuming it*,
+and properly update the socket assign
+so we can show it to the person.
+
+In the same file,
+change the `handle_progress/3` function to the following.
+
+```elixir
+ def handle_progress(:image_list, entry, socket) do
+ if entry.done? do
+
+ # Consume the entry and get the tensor to feed to classifier
+ %{tensor: tensor, file_binary: file_binary} = consume_uploaded_entry(socket, entry, fn %{} = meta ->
+ file_binary = File.read!(meta.path)
+
+ {:ok, vimage} = Vix.Vips.Image.new_from_file(meta.path)
+ {:ok, tensor} = pre_process_image(vimage)
+ {:ok, %{tensor: tensor, file_binary: file_binary}}
+ end)
+
+ # Create an async task to classify the image
+ task = Task.Supervisor.async(App.TaskSupervisor, fn -> Nx.Serving.batched_run(ImageClassifier, tensor) end)
+
+ # Encode the image to base64
+ base64 = "data:image/png;base64, " <> Base.encode64(file_binary)
+
+ # Update socket assigns to show spinner whilst task is running
+ {:noreply, assign(socket, running: true, task_ref: task.ref, image_preview_base64: base64)}
+ else
+ {:noreply, socket}
+ end
+ end
+```
+
+We're using [`File.read!/1`](https://hexdocs.pm/elixir/1.13/File.html#read/1)
+to retrieve the binary representation of the image that was uploaded.
+We use [`Base.encode64/2`](https://hexdocs.pm/elixir/1.12/Base.html#encode64/2)
+to encode this file binary
+and assign the newly created `image_preview_base64` socket assign
+with this base64 representation of the image.
+
+Now, all that's left to do
+is to *render the image on our view*.
+In `lib/app_web/live/page_live.html.heex`,
+locate the line:
+
+```html
+
+```
+
+We are going to update this `
`
+to show the image with the `image_preview_base64` socket assign.
+
+```html
+
+
+ <%= if @image_preview_base64 do %>
+
+ <% else %>
+
+
+
+
or drag and drop
+
+
PNG, JPG, GIF up to 5MB
+ <% end %>
+
+```
+
+As you can see,
+we are checking if `@image_preview_base64` is defined.
+If so, we simply show the image with it as `src` π.
+
+Now, if you run the application,
+you'll see that after dragging the image,
+it is previewed and shown to the person!
+
+
+
+
+
+
+
+# 6. What about other models?
+
+Maybe you weren't happy with the results from this model.
+
+That's fair. `ResNet-50` is a smaller, "older" model compared to other
+image captioning/classification models.
+
+What if you wanted to use others?
+Well, as we've mentioned before,
+`Bumblebee` uses
+[**Transformer models from `HuggingFace`**](https://huggingface.co/docs/transformers/index).
+To know if one is supported
+(as shown in [`Bumblebee`'s docs](https://github.com/elixir-nx/bumblebee#model-support)),
+we need to check the `config.json` file
+in the model repository
+and copy the class name under `"architectures"`
+and search it on `Bumblebee`'s codebase.
+
+For example,
+here's one of the more popular image captioning models -
+Salesforce's `BLIP` -
+https://huggingface.co/Salesforce/blip-image-captioning-large/blob/main/config.json.
+
+
+
+
+
+
+
+If you visit `Bumblebee`'s codebase
+and search for the class name,
+you'll find it is supported.
+
+
+
+
+
+Awesome!
+Now we can use it!
+
+If you dig around `Bumblebee`'s docs as well
+(https://hexdocs.pm/bumblebee/Bumblebee.Vision.html#image_to_text/5),
+you'll see that we've got to use `image_to_text/5` with this model.
+It needs a `tokenizer`, `featurizer` and a `generation-config`
+so we can use it.
+
+Let's do it!
+Head over to `lib/app/application.ex`,
+and change the `serving/0` function.
+
+```elixir
+ def serving do
+ {:ok, model_info} = Bumblebee.load_model({:hf, "Salesforce/blip-image-captioning-base"})
+ {:ok, featurizer} = Bumblebee.load_featurizer({:hf, "Salesforce/blip-image-captioning-base"})
+ {:ok, tokenizer} = Bumblebee.load_tokenizer({:hf, "Salesforce/blip-image-captioning-base"})
+ {:ok, generation_config} = Bumblebee.load_generation_config({:hf, "Salesforce/blip-image-captioning-base"})
+
+ Bumblebee.Vision.image_to_text(model_info, featurizer, tokenizer, generation_config,
+ compile: [batch_size: 10],
+ defn_options: [compiler: EXLA]
+ )
+ end
+```
+
+As you can see, we're using the repository name of `BLIP`'s model
+from the HuggingFace website.
+
+If you run `mix phx.server`,
+you'll see that it will download the new models,
+tokenizers, featurizer and configs to run the model.
+
+```sh
+|======================================================================| 100% (989.82 MB)
+[info] TfrtCpuClient created.
+|======================================================================| 100% (711.39 KB)
+[info] Running AppWeb.Endpoint with cowboy 2.10.0 at 127.0.0.1:4000 (http)
+[info] Access AppWeb.Endpoint at http://localhost:4000
+[watch] build finished, watching for changes...
+```
+
+You may think we're done here.
+But we are not! β
+
+The **destructuring of the output of the model may not be the same**.
+If you try to submit a photo,
+you'll get this error:
+
+```sh
+no match of right hand side value:
+%{results: [%{text: "a person holding a large blue ball on a beach"}]}
+```
+
+This means that we need to make some changes
+when parsing the output of the model π.
+
+Head over to `lib/app_web/live/page_live.ex`
+and change the `handle_info/3` function
+that is called after the async task is completed.
+
+```elixir
+ def handle_info({ref, result}, %{assigns: %{task_ref: ref}} = socket) do
+ Process.demonitor(ref, [:flush])
+
+ %{results: [%{text: label}]} = result # change this line
+
+ {:noreply, assign(socket, label: label, running: false)}
+ end
+```
+
+As you can see, we are now correctly destructuring the result from the model.
+And that's it!
+
+If you run `mix phx.server`,
+you'll see that we got far more accurate results!
+
+
+
+
+
+Awesome! π
+
+> **Note**: Be aware that `BLIP`
+> is a _much_ larger model than `ResNet-50`.
+> There are more accurate and even larger models out there
+> (e.g:
+> [`blip-image-captioning-large`](https://huggingface.co/Salesforce/blip-image-captioning-large),
+> the larger version of the model we've just used).
+> This is a balancing act: the larger the model, the longer a prediction may take
+> and more resources your server will need to have to handle this heavier workload.
+
+
+# _Please_ Star the repo! βοΈ
+
+If you find this package/repo useful,
+please star on GitHub, so that we know! β
+
+Thank you! π
\ No newline at end of file
diff --git a/assets/css/app.css b/assets/css/app.css
new file mode 100644
index 0000000..d001c18
--- /dev/null
+++ b/assets/css/app.css
@@ -0,0 +1,17 @@
+@import "tailwindcss/base";
+@import "tailwindcss/components";
+@import "tailwindcss/utilities";
+
+/* This file is for your main application CSS */
+/* override browser default */
+html,
+body {
+ margin: 0;
+ padding: 0;
+}
+
+/* use viewport-relative units to cover page fully */
+body {
+ height: 100vh;
+ width: 100vw;
+}
diff --git a/assets/js/app.js b/assets/js/app.js
new file mode 100644
index 0000000..df0cdd9
--- /dev/null
+++ b/assets/js/app.js
@@ -0,0 +1,41 @@
+// If you want to use Phoenix channels, run `mix help phx.gen.channel`
+// to get started and then uncomment the line below.
+// import "./user_socket.js"
+
+// You can include dependencies in two ways.
+//
+// The simplest option is to put them in assets/vendor and
+// import them using relative paths:
+//
+// import "../vendor/some-package.js"
+//
+// Alternatively, you can `npm install some-package --prefix assets` and import
+// them using a path starting with the package name:
+//
+// import "some-package"
+//
+
+// Include phoenix_html to handle method=PUT/DELETE in forms and buttons.
+import "phoenix_html"
+// Establish Phoenix Socket and LiveView configuration.
+import {Socket} from "phoenix"
+import {LiveSocket} from "phoenix_live_view"
+import topbar from "../vendor/topbar"
+
+let csrfToken = document.querySelector("meta[name='csrf-token']").getAttribute("content")
+let liveSocket = new LiveSocket("/live", Socket, {params: {_csrf_token: csrfToken}})
+
+// Show progress bar on live navigation and form submits
+topbar.config({barColors: {0: "#29d"}, shadowColor: "rgba(0, 0, 0, .3)"})
+window.addEventListener("phx:page-loading-start", _info => topbar.show(300))
+window.addEventListener("phx:page-loading-stop", _info => topbar.hide())
+
+// connect if there are any LiveViews on the page
+liveSocket.connect()
+
+// expose liveSocket on window for web console debug logs and latency simulation:
+// >> liveSocket.enableDebug()
+// >> liveSocket.enableLatencySim(1000) // enabled for duration of browser session
+// >> liveSocket.disableLatencySim()
+window.liveSocket = liveSocket
+
diff --git a/assets/tailwind.config.js b/assets/tailwind.config.js
new file mode 100644
index 0000000..e3bf241
--- /dev/null
+++ b/assets/tailwind.config.js
@@ -0,0 +1,26 @@
+// See the Tailwind configuration guide for advanced usage
+// https://tailwindcss.com/docs/configuration
+
+const plugin = require("tailwindcss/plugin")
+
+module.exports = {
+ content: [
+ "./js/**/*.js",
+ "../lib/*_web.ex",
+ "../lib/*_web/**/*.*ex"
+ ],
+ theme: {
+ extend: {
+ colors: {
+ brand: "#FD4F00",
+ }
+ },
+ },
+ plugins: [
+ require("@tailwindcss/forms"),
+ plugin(({addVariant}) => addVariant("phx-no-feedback", [".phx-no-feedback&", ".phx-no-feedback &"])),
+ plugin(({addVariant}) => addVariant("phx-click-loading", [".phx-click-loading&", ".phx-click-loading &"])),
+ plugin(({addVariant}) => addVariant("phx-submit-loading", [".phx-submit-loading&", ".phx-submit-loading &"])),
+ plugin(({addVariant}) => addVariant("phx-change-loading", [".phx-change-loading&", ".phx-change-loading &"]))
+ ]
+}
diff --git a/assets/vendor/topbar.js b/assets/vendor/topbar.js
new file mode 100644
index 0000000..4195727
--- /dev/null
+++ b/assets/vendor/topbar.js
@@ -0,0 +1,165 @@
+/**
+ * @license MIT
+ * topbar 2.0.0, 2023-02-04
+ * https://buunguyen.github.io/topbar
+ * Copyright (c) 2021 Buu Nguyen
+ */
+(function (window, document) {
+ "use strict";
+
+ // https://gist.github.com/paulirish/1579671
+ (function () {
+ var lastTime = 0;
+ var vendors = ["ms", "moz", "webkit", "o"];
+ for (var x = 0; x < vendors.length && !window.requestAnimationFrame; ++x) {
+ window.requestAnimationFrame =
+ window[vendors[x] + "RequestAnimationFrame"];
+ window.cancelAnimationFrame =
+ window[vendors[x] + "CancelAnimationFrame"] ||
+ window[vendors[x] + "CancelRequestAnimationFrame"];
+ }
+ if (!window.requestAnimationFrame)
+ window.requestAnimationFrame = function (callback, element) {
+ var currTime = new Date().getTime();
+ var timeToCall = Math.max(0, 16 - (currTime - lastTime));
+ var id = window.setTimeout(function () {
+ callback(currTime + timeToCall);
+ }, timeToCall);
+ lastTime = currTime + timeToCall;
+ return id;
+ };
+ if (!window.cancelAnimationFrame)
+ window.cancelAnimationFrame = function (id) {
+ clearTimeout(id);
+ };
+ })();
+
+ var canvas,
+ currentProgress,
+ showing,
+ progressTimerId = null,
+ fadeTimerId = null,
+ delayTimerId = null,
+ addEvent = function (elem, type, handler) {
+ if (elem.addEventListener) elem.addEventListener(type, handler, false);
+ else if (elem.attachEvent) elem.attachEvent("on" + type, handler);
+ else elem["on" + type] = handler;
+ },
+ options = {
+ autoRun: true,
+ barThickness: 3,
+ barColors: {
+ 0: "rgba(26, 188, 156, .9)",
+ ".25": "rgba(52, 152, 219, .9)",
+ ".50": "rgba(241, 196, 15, .9)",
+ ".75": "rgba(230, 126, 34, .9)",
+ "1.0": "rgba(211, 84, 0, .9)",
+ },
+ shadowBlur: 10,
+ shadowColor: "rgba(0, 0, 0, .6)",
+ className: null,
+ },
+ repaint = function () {
+ canvas.width = window.innerWidth;
+ canvas.height = options.barThickness * 5; // need space for shadow
+
+ var ctx = canvas.getContext("2d");
+ ctx.shadowBlur = options.shadowBlur;
+ ctx.shadowColor = options.shadowColor;
+
+ var lineGradient = ctx.createLinearGradient(0, 0, canvas.width, 0);
+ for (var stop in options.barColors)
+ lineGradient.addColorStop(stop, options.barColors[stop]);
+ ctx.lineWidth = options.barThickness;
+ ctx.beginPath();
+ ctx.moveTo(0, options.barThickness / 2);
+ ctx.lineTo(
+ Math.ceil(currentProgress * canvas.width),
+ options.barThickness / 2
+ );
+ ctx.strokeStyle = lineGradient;
+ ctx.stroke();
+ },
+ createCanvas = function () {
+ canvas = document.createElement("canvas");
+ var style = canvas.style;
+ style.position = "fixed";
+ style.top = style.left = style.right = style.margin = style.padding = 0;
+ style.zIndex = 100001;
+ style.display = "none";
+ if (options.className) canvas.classList.add(options.className);
+ document.body.appendChild(canvas);
+ addEvent(window, "resize", repaint);
+ },
+ topbar = {
+ config: function (opts) {
+ for (var key in opts)
+ if (options.hasOwnProperty(key)) options[key] = opts[key];
+ },
+ show: function (delay) {
+ if (showing) return;
+ if (delay) {
+ if (delayTimerId) return;
+ delayTimerId = setTimeout(() => topbar.show(), delay);
+ } else {
+ showing = true;
+ if (fadeTimerId !== null) window.cancelAnimationFrame(fadeTimerId);
+ if (!canvas) createCanvas();
+ canvas.style.opacity = 1;
+ canvas.style.display = "block";
+ topbar.progress(0);
+ if (options.autoRun) {
+ (function loop() {
+ progressTimerId = window.requestAnimationFrame(loop);
+ topbar.progress(
+ "+" + 0.05 * Math.pow(1 - Math.sqrt(currentProgress), 2)
+ );
+ })();
+ }
+ }
+ },
+ progress: function (to) {
+ if (typeof to === "undefined") return currentProgress;
+ if (typeof to === "string") {
+ to =
+ (to.indexOf("+") >= 0 || to.indexOf("-") >= 0
+ ? currentProgress
+ : 0) + parseFloat(to);
+ }
+ currentProgress = to > 1 ? 1 : to;
+ repaint();
+ return currentProgress;
+ },
+ hide: function () {
+ clearTimeout(delayTimerId);
+ delayTimerId = null;
+ if (!showing) return;
+ showing = false;
+ if (progressTimerId != null) {
+ window.cancelAnimationFrame(progressTimerId);
+ progressTimerId = null;
+ }
+ (function loop() {
+ if (topbar.progress("+.1") >= 1) {
+ canvas.style.opacity -= 0.05;
+ if (canvas.style.opacity <= 0.05) {
+ canvas.style.display = "none";
+ fadeTimerId = null;
+ return;
+ }
+ }
+ fadeTimerId = window.requestAnimationFrame(loop);
+ })();
+ },
+ };
+
+ if (typeof module === "object" && typeof module.exports === "object") {
+ module.exports = topbar;
+ } else if (typeof define === "function" && define.amd) {
+ define(function () {
+ return topbar;
+ });
+ } else {
+ this.topbar = topbar;
+ }
+}.call(this, window, document));
diff --git a/config/config.exs b/config/config.exs
new file mode 100644
index 0000000..e3e1cfa
--- /dev/null
+++ b/config/config.exs
@@ -0,0 +1,56 @@
+# This file is responsible for configuring your application
+# and its dependencies with the aid of the Config module.
+#
+# This configuration file is loaded before any dependency and
+# is restricted to this project.
+
+# General application configuration
+import Config
+
+# Tells `NX` to use `EXLA` as backend
+config :nx, default_backend: EXLA.Backend
+
+# Configures the endpoint
+config :app, AppWeb.Endpoint,
+ url: [host: "localhost"],
+ render_errors: [
+ formats: [html: AppWeb.ErrorHTML, json: AppWeb.ErrorJSON],
+ layout: false
+ ],
+ pubsub_server: App.PubSub,
+ live_view: [signing_salt: "euyclMQ2"]
+
+# Configure esbuild (the version is required)
+config :esbuild,
+ version: "0.14.41",
+ default: [
+ args:
+ ~w(js/app.js --bundle --target=es2017 --outdir=../priv/static/assets --external:/fonts/* --external:/images/*),
+ cd: Path.expand("../assets", __DIR__),
+ env: %{"NODE_PATH" => Path.expand("../deps", __DIR__)}
+ ]
+
+# Configure tailwind (the version is required)
+config :tailwind,
+ version: "3.2.4",
+ default: [
+ args: ~w(
+ --config=tailwind.config.js
+ --input=css/app.css
+ --output=../priv/static/assets/app.css
+ ),
+ cd: Path.expand("../assets", __DIR__)
+ ]
+
+# Configures Elixir's Logger
+config :logger, :console,
+ format: "$time $metadata[$level] $message\n",
+ metadata: [:request_id]
+
+# Use Jason for JSON parsing in Phoenix
+config :phoenix, :json_library, Jason
+
+
+# Import environment specific config. This must remain at the bottom
+# of this file so it overrides the configuration defined above.
+import_config "#{config_env()}.exs"
diff --git a/config/dev.exs b/config/dev.exs
new file mode 100644
index 0000000..e0be8f8
--- /dev/null
+++ b/config/dev.exs
@@ -0,0 +1,65 @@
+import Config
+
+# For development, we disable any cache and enable
+# debugging and code reloading.
+#
+# The watchers configuration can be used to run external
+# watchers to your application. For example, we use it
+# with esbuild to bundle .js and .css sources.
+config :app, AppWeb.Endpoint,
+ # Binding to loopback ipv4 address prevents access from other machines.
+ # Change to `ip: {0, 0, 0, 0}` to allow access from other machines.
+ http: [ip: {127, 0, 0, 1}, port: 4000],
+ check_origin: false,
+ code_reloader: true,
+ debug_errors: true,
+ secret_key_base: "btTsEy6WVagm+4u+ZrbwVg6F48ZfgpePZx70twE9SSyPZKkZHiaYa77bFUWV4Vw5",
+ watchers: [
+ esbuild: {Esbuild, :install_and_run, [:default, ~w(--sourcemap=inline --watch)]},
+ tailwind: {Tailwind, :install_and_run, [:default, ~w(--watch)]}
+ ]
+
+# ## SSL Support
+#
+# In order to use HTTPS in development, a self-signed
+# certificate can be generated by running the following
+# Mix task:
+#
+# mix phx.gen.cert
+#
+# Run `mix help phx.gen.cert` for more information.
+#
+# The `http:` config above can be replaced with:
+#
+# https: [
+# port: 4001,
+# cipher_suite: :strong,
+# keyfile: "priv/cert/selfsigned_key.pem",
+# certfile: "priv/cert/selfsigned.pem"
+# ],
+#
+# If desired, both `http:` and `https:` keys can be
+# configured to run both http and https servers on
+# different ports.
+
+# Watch static and templates for browser reloading.
+config :app, AppWeb.Endpoint,
+ live_reload: [
+ patterns: [
+ ~r"priv/static/.*(js|css|png|jpeg|jpg|gif|svg)$",
+ ~r"lib/app_web/(controllers|live|components)/.*(ex|heex)$"
+ ]
+ ]
+
+# Enable dev routes for dashboard and mailbox
+config :app, dev_routes: true
+
+# Do not include metadata nor timestamps in development logs
+config :logger, :console, format: "[$level] $message\n"
+
+# Set a higher stacktrace during development. Avoid configuring such
+# in production as building large stacktraces may be expensive.
+config :phoenix, :stacktrace_depth, 20
+
+# Initialize plugs at runtime for faster development compilation
+config :phoenix, :plug_init_mode, :runtime
diff --git a/config/prod.exs b/config/prod.exs
new file mode 100644
index 0000000..608ef13
--- /dev/null
+++ b/config/prod.exs
@@ -0,0 +1,18 @@
+import Config
+
+# For production, don't forget to configure the url host
+# to something meaningful, Phoenix uses this information
+# when generating URLs.
+
+# Note we also include the path to a cache manifest
+# containing the digested version of static files. This
+# manifest is generated by the `mix phx.digest` task,
+# which you should run after static files are built and
+# before starting your production server.
+config :app, AppWeb.Endpoint, cache_static_manifest: "priv/static/cache_manifest.json"
+
+# Do not print debug messages in production
+config :logger, level: :info
+
+# Runtime production configuration, including reading
+# of environment variables, is done on config/runtime.exs.
diff --git a/config/runtime.exs b/config/runtime.exs
new file mode 100644
index 0000000..894e9a3
--- /dev/null
+++ b/config/runtime.exs
@@ -0,0 +1,82 @@
+import Config
+
+# config/runtime.exs is executed for all environments, including
+# during releases. It is executed after compilation and before the
+# system starts, so it is typically used to load production configuration
+# and secrets from environment variables or elsewhere. Do not define
+# any compile-time configuration in here, as it won't be applied.
+# The block below contains prod specific runtime configuration.
+
+# ## Using releases
+#
+# If you use `mix release`, you need to explicitly enable the server
+# by passing the PHX_SERVER=true when you start it:
+#
+# PHX_SERVER=true bin/app start
+#
+# Alternatively, you can use `mix phx.gen.release` to generate a `bin/server`
+# script that automatically sets the env var above.
+if System.get_env("PHX_SERVER") do
+ config :app, AppWeb.Endpoint, server: true
+end
+
+if config_env() == :prod do
+ # The secret key base is used to sign/encrypt cookies and other secrets.
+ # A default value is used in config/dev.exs and config/test.exs but you
+ # want to use a different value for prod and you most likely don't want
+ # to check this value into version control, so we use an environment
+ # variable instead.
+ secret_key_base =
+ System.get_env("SECRET_KEY_BASE") ||
+ raise """
+ environment variable SECRET_KEY_BASE is missing.
+ You can generate one by calling: mix phx.gen.secret
+ """
+
+ host = System.get_env("PHX_HOST") || "example.com"
+ port = String.to_integer(System.get_env("PORT") || "4000")
+
+ config :app, AppWeb.Endpoint,
+ url: [host: host, port: 443, scheme: "https"],
+ http: [
+ # Enable IPv6 and bind on all interfaces.
+ # Set it to {0, 0, 0, 0, 0, 0, 0, 1} for local network only access.
+ # See the documentation on https://hexdocs.pm/plug_cowboy/Plug.Cowboy.html
+ # for details about using IPv6 vs IPv4 and loopback vs public addresses.
+ ip: {0, 0, 0, 0, 0, 0, 0, 0},
+ port: port
+ ],
+ secret_key_base: secret_key_base
+
+ # ## SSL Support
+ #
+ # To get SSL working, you will need to add the `https` key
+ # to your endpoint configuration:
+ #
+ # config :app, AppWeb.Endpoint,
+ # https: [
+ # ...,
+ # port: 443,
+ # cipher_suite: :strong,
+ # keyfile: System.get_env("SOME_APP_SSL_KEY_PATH"),
+ # certfile: System.get_env("SOME_APP_SSL_CERT_PATH")
+ # ]
+ #
+ # The `cipher_suite` is set to `:strong` to support only the
+ # latest and more secure SSL ciphers. This means old browsers
+ # and clients may not be supported. You can set it to
+ # `:compatible` for wider support.
+ #
+ # `:keyfile` and `:certfile` expect an absolute path to the key
+ # and cert in disk or a relative path inside priv, for example
+ # "priv/ssl/server.key". For all supported SSL configuration
+ # options, see https://hexdocs.pm/plug/Plug.SSL.html#configure/1
+ #
+ # We also recommend setting `force_ssl` in your endpoint, ensuring
+ # no data is ever sent via http, always redirecting to https:
+ #
+ # config :app, AppWeb.Endpoint,
+ # force_ssl: [hsts: true]
+ #
+ # Check `Plug.SSL` for all available options in `force_ssl`.
+end
diff --git a/config/test.exs b/config/test.exs
new file mode 100644
index 0000000..20a13c9
--- /dev/null
+++ b/config/test.exs
@@ -0,0 +1,14 @@
+import Config
+
+# We don't run a server during test. If one is required,
+# you can enable the server option below.
+config :app, AppWeb.Endpoint,
+ http: [ip: {127, 0, 0, 1}, port: 4002],
+ secret_key_base: "d422JqbVTXef5vPy90SakC4QcPN76fRi6wLm+pUnC09eFxWUjPbTKe0dVmpGpI5N",
+ server: false
+
+# Print only warnings and errors during test
+config :logger, level: :warning
+
+# Initialize plugs at runtime for faster test compilation
+config :phoenix, :plug_init_mode, :runtime
diff --git a/coveralls.json b/coveralls.json
new file mode 100644
index 0000000..b1550d0
--- /dev/null
+++ b/coveralls.json
@@ -0,0 +1,17 @@
+{
+ "skip_files": [
+ "test/",
+ "lib/app.ex",
+ "lib/app/application.ex",
+ "lib/app_web.ex",
+ "lib/app/repo.ex",
+ "lib/app/release.ex",
+ "lib/app_web/views/app_view.ex",
+ "lib/app_web/views/init_view.ex",
+ "lib/app_web/views/layout_view.ex",
+ "lib/app_web/views/error_helpers.ex",
+ "lib/app_web/components",
+ "lib/app_web/endpoint.ex",
+ "lib/app_web/telemetry.ex"
+ ]
+ }
\ No newline at end of file
diff --git a/lib/app.ex b/lib/app.ex
new file mode 100644
index 0000000..a10dc06
--- /dev/null
+++ b/lib/app.ex
@@ -0,0 +1,9 @@
+defmodule App do
+ @moduledoc """
+ App keeps the contexts that define your domain
+ and business logic.
+
+ Contexts are also responsible for managing your data, regardless
+ if it comes from the database, an external API or others.
+ """
+end
diff --git a/lib/app/application.ex b/lib/app/application.ex
new file mode 100644
index 0000000..c5ecf1b
--- /dev/null
+++ b/lib/app/application.ex
@@ -0,0 +1,50 @@
+defmodule App.Application do
+ # See https://hexdocs.pm/elixir/Application.html
+ # for more information on OTP Applications
+ @moduledoc false
+
+ use Application
+
+ @impl true
+ def start(_type, _args) do
+ children = [
+ # Start the Telemetry supervisor
+ AppWeb.Telemetry,
+ # Start the PubSub system
+ {Phoenix.PubSub, name: App.PubSub},
+ # Nx serving for image classifier
+ {Nx.Serving, serving: serving(), name: ImageClassifier},
+ # Adding a supervisor
+ {Task.Supervisor, name: App.TaskSupervisor},
+ # Start the Endpoint (http/https)
+ AppWeb.Endpoint
+ # Start a worker by calling: App.Worker.start_link(arg)
+ # {App.Worker, arg}
+ ]
+
+ # See https://hexdocs.pm/elixir/Supervisor.html
+ # for other strategies and supported options
+ opts = [strategy: :one_for_one, name: App.Supervisor]
+ Supervisor.start_link(children, opts)
+ end
+
+ def serving do
+ {:ok, model_info} = Bumblebee.load_model({:hf, "Salesforce/blip-image-captioning-base"})
+ {:ok, featurizer} = Bumblebee.load_featurizer({:hf, "Salesforce/blip-image-captioning-base"})
+ {:ok, tokenizer} = Bumblebee.load_tokenizer({:hf, "Salesforce/blip-image-captioning-base"})
+ {:ok, generation_config} = Bumblebee.load_generation_config({:hf, "Salesforce/blip-image-captioning-base"})
+
+ Bumblebee.Vision.image_to_text(model_info, featurizer, tokenizer, generation_config,
+ compile: [batch_size: 10],
+ defn_options: [compiler: EXLA]
+ )
+ end
+
+ # Tell Phoenix to update the endpoint configuration
+ # whenever the application is updated.
+ @impl true
+ def config_change(changed, _new, removed) do
+ AppWeb.Endpoint.config_change(changed, removed)
+ :ok
+ end
+end
diff --git a/lib/app_web.ex b/lib/app_web.ex
new file mode 100644
index 0000000..512ce55
--- /dev/null
+++ b/lib/app_web.ex
@@ -0,0 +1,111 @@
+defmodule AppWeb do
+ @moduledoc """
+ The entrypoint for defining your web interface, such
+ as controllers, components, channels, and so on.
+
+ This can be used in your application as:
+
+ use AppWeb, :controller
+ use AppWeb, :html
+
+ The definitions below will be executed for every controller,
+ component, etc, so keep them short and clean, focused
+ on imports, uses and aliases.
+
+ Do NOT define functions inside the quoted expressions
+ below. Instead, define additional modules and import
+ those modules here.
+ """
+
+ def static_paths, do: ~w(assets fonts images favicon.ico robots.txt)
+
+ def router do
+ quote do
+ use Phoenix.Router, helpers: false
+
+ # Import common connection and controller functions to use in pipelines
+ import Plug.Conn
+ import Phoenix.Controller
+ import Phoenix.LiveView.Router
+ end
+ end
+
+ def channel do
+ quote do
+ use Phoenix.Channel
+ end
+ end
+
+ def controller do
+ quote do
+ use Phoenix.Controller,
+ formats: [:html, :json],
+ layouts: [html: AppWeb.Layouts]
+
+ import Plug.Conn
+
+ unquote(verified_routes())
+ end
+ end
+
+ def live_view do
+ quote do
+ use Phoenix.LiveView,
+ layout: {AppWeb.Layouts, :app}
+
+ unquote(html_helpers())
+ end
+ end
+
+ def live_component do
+ quote do
+ use Phoenix.LiveComponent
+
+ unquote(html_helpers())
+ end
+ end
+
+ def html do
+ quote do
+ use Phoenix.Component
+
+ # Import convenience functions from controllers
+ import Phoenix.Controller,
+ only: [get_csrf_token: 0, view_module: 1, view_template: 1]
+
+ # Include general helpers for rendering HTML
+ unquote(html_helpers())
+ end
+ end
+
+ defp html_helpers do
+ quote do
+ # HTML escaping functionality
+ import Phoenix.HTML
+ # Core UI components and translation
+ import AppWeb.CoreComponents
+
+ # Shortcut for generating JS commands
+ alias Phoenix.LiveView.JS
+
+ # Routes generation with the ~p sigil
+ unquote(verified_routes())
+ end
+ end
+
+ def verified_routes do
+ quote do
+ use Phoenix.VerifiedRoutes,
+ endpoint: AppWeb.Endpoint,
+ router: AppWeb.Router,
+ statics: AppWeb.static_paths()
+ end
+ end
+
+ @doc """
+ When used, dispatch to the appropriate controller/view/etc.
+ """
+ defmacro __using__(which) when is_atom(which) do
+ apply(__MODULE__, which, [])
+ end
+end
diff --git a/lib/app_web/components/core_components.ex b/lib/app_web/components/core_components.ex
new file mode 100644
index 0000000..dad6364
--- /dev/null
+++ b/lib/app_web/components/core_components.ex
@@ -0,0 +1,650 @@
+defmodule AppWeb.CoreComponents do
+ @moduledoc """
+ Provides core UI components.
+
+ The components in this module use Tailwind CSS, a utility-first CSS framework.
+ See the [Tailwind CSS documentation](https://tailwindcss.com) to learn how to
+ customize the generated components in this module.
+
+ Icons are provided by [heroicons](https://heroicons.com), using the
+ [heroicons_elixir](https://github.com/mveytsman/heroicons_elixir) project.
+ """
+ use Phoenix.Component
+
+ alias Phoenix.LiveView.JS
+
+ @doc """
+ Renders a modal.
+
+ ## Examples
+
+ <.modal id="confirm-modal">
+ Are you sure?
+ <:confirm>OK
+ <:cancel>Cancel
+
+
+ JS commands may be passed to the `:on_cancel` and `on_confirm` attributes
+ for the caller to react to each button press, for example:
+
+ <.modal id="confirm" on_confirm={JS.push("delete")} on_cancel={JS.navigate(~p"/posts")}>
+ Are you sure you?
+ <:confirm>OK
+ <:cancel>Cancel
+
+ """
+ attr :id, :string, required: true
+ attr :show, :boolean, default: false
+ attr :on_cancel, JS, default: %JS{}
+ attr :on_confirm, JS, default: %JS{}
+
+ slot :inner_block, required: true
+ slot :title
+ slot :subtitle
+ slot :confirm
+ slot :cancel
+
+ def modal(assigns) do
+ ~H"""
+