diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 0a28a11112251..367df07a7e497 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -515,6 +515,31 @@ jobs:
- name: Build Xcode project
run: xcodebuild -project examples/llama.swiftui/llama.swiftui.xcodeproj -scheme llama.swiftui -sdk iphoneos CODE_SIGNING_REQUIRED=NO CODE_SIGN_IDENTITY= -destination 'generic/platform=iOS' build
+ android-build:
+ runs-on: ubuntu-latest
+
+ steps:
+ - name: Clone
+ uses: actions/checkout@v3
+
+ - name: Set up JDK
+ uses: actions/setup-java@v3
+ with:
+ java-version: 17
+ distribution: zulu
+
+ - name: Setup Android SDK
+ uses: android-actions/setup-android@v3
+ with:
+ log-accepted-android-sdk-licenses: false
+
+ - name: Build
+ run: |
+ cd examples/llama.android
+
+ # Skip armeabi-v7a for now (https://github.com/llvm/llvm-project/issues/65820).
+ ./gradlew build --no-daemon -Pskip-armeabi-v7a
+
# freeBSD-latest:
# runs-on: macos-12
# steps:
diff --git a/examples/llama.android/.gitignore b/examples/llama.android/.gitignore
new file mode 100644
index 0000000000000..347e252ef10e9
--- /dev/null
+++ b/examples/llama.android/.gitignore
@@ -0,0 +1,33 @@
+# Gradle files
+.gradle/
+build/
+
+# Local configuration file (sdk path, etc)
+local.properties
+
+# Log/OS Files
+*.log
+
+# Android Studio generated files and folders
+captures/
+.externalNativeBuild/
+.cxx/
+*.apk
+output.json
+
+# IntelliJ
+*.iml
+.idea/
+misc.xml
+deploymentTargetDropDown.xml
+render.experimental.xml
+
+# Keystore files
+*.jks
+*.keystore
+
+# Google Services (e.g. APIs or Firebase)
+google-services.json
+
+# Android Profiling
+*.hprof
diff --git a/examples/llama.android/README.md b/examples/llama.android/README.md
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/examples/llama.android/app/.gitignore b/examples/llama.android/app/.gitignore
new file mode 100644
index 0000000000000..796b96d1c4023
--- /dev/null
+++ b/examples/llama.android/app/.gitignore
@@ -0,0 +1 @@
+/build
diff --git a/examples/llama.android/app/build.gradle.kts b/examples/llama.android/app/build.gradle.kts
new file mode 100644
index 0000000000000..7815a802593ba
--- /dev/null
+++ b/examples/llama.android/app/build.gradle.kts
@@ -0,0 +1,91 @@
+plugins {
+ id("com.android.application")
+ id("org.jetbrains.kotlin.android")
+}
+
+android {
+ namespace = "com.example.llama"
+ compileSdk = 34
+
+ ndkVersion = "26.1.10909125"
+
+ defaultConfig {
+ applicationId = "com.example.llama"
+ minSdk = 33
+ targetSdk = 34
+ versionCode = 1
+ versionName = "1.0"
+
+ testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner"
+ vectorDrawables {
+ useSupportLibrary = true
+ }
+ ndk {
+ // Workaround for https://github.com/llvm/llvm-project/issues/65820
+ // affecting armeabi-v7a. Skip armeabi-v7a when invoked with
+ // -Pskip-armeabi-v7a (e.g., ./gradlew build -Pskip-armeabi-v7a).
+ if (project.hasProperty("skip-armeabi-v7a")) {
+ abiFilters += listOf("arm64-v8a", "x86_64", "x86")
+ }
+ }
+ externalNativeBuild {
+ cmake {
+ cppFlags += listOf()
+ arguments += listOf()
+ }
+ }
+ }
+
+ buildTypes {
+ release {
+ isMinifyEnabled = false
+ proguardFiles(
+ getDefaultProguardFile("proguard-android-optimize.txt"),
+ "proguard-rules.pro"
+ )
+ }
+ }
+ compileOptions {
+ sourceCompatibility = JavaVersion.VERSION_1_8
+ targetCompatibility = JavaVersion.VERSION_1_8
+ }
+ kotlinOptions {
+ jvmTarget = "1.8"
+ }
+ buildFeatures {
+ compose = true
+ }
+ composeOptions {
+ kotlinCompilerExtensionVersion = "1.5.1"
+ }
+ packaging {
+ resources {
+ excludes += "/META-INF/{AL2.0,LGPL2.1}"
+ }
+ }
+ externalNativeBuild {
+ cmake {
+ path = file("src/main/cpp/CMakeLists.txt")
+ version = "3.22.1"
+ }
+ }
+}
+
+dependencies {
+
+ implementation("androidx.core:core-ktx:1.12.0")
+ implementation("androidx.lifecycle:lifecycle-runtime-ktx:2.6.2")
+ implementation("androidx.activity:activity-compose:1.8.2")
+ implementation(platform("androidx.compose:compose-bom:2023.08.00"))
+ implementation("androidx.compose.ui:ui")
+ implementation("androidx.compose.ui:ui-graphics")
+ implementation("androidx.compose.ui:ui-tooling-preview")
+ implementation("androidx.compose.material3:material3")
+ testImplementation("junit:junit:4.13.2")
+ androidTestImplementation("androidx.test.ext:junit:1.1.5")
+ androidTestImplementation("androidx.test.espresso:espresso-core:3.5.1")
+ androidTestImplementation(platform("androidx.compose:compose-bom:2023.08.00"))
+ androidTestImplementation("androidx.compose.ui:ui-test-junit4")
+ debugImplementation("androidx.compose.ui:ui-tooling")
+ debugImplementation("androidx.compose.ui:ui-test-manifest")
+}
diff --git a/examples/llama.android/app/proguard-rules.pro b/examples/llama.android/app/proguard-rules.pro
new file mode 100644
index 0000000000000..f1b424510da51
--- /dev/null
+++ b/examples/llama.android/app/proguard-rules.pro
@@ -0,0 +1,21 @@
+# Add project specific ProGuard rules here.
+# You can control the set of applied configuration files using the
+# proguardFiles setting in build.gradle.
+#
+# For more details, see
+# http://developer.android.com/guide/developing/tools/proguard.html
+
+# If your project uses WebView with JS, uncomment the following
+# and specify the fully qualified class name to the JavaScript interface
+# class:
+#-keepclassmembers class fqcn.of.javascript.interface.for.webview {
+# public *;
+#}
+
+# Uncomment this to preserve the line number information for
+# debugging stack traces.
+#-keepattributes SourceFile,LineNumberTable
+
+# If you keep the line number information, uncomment this to
+# hide the original source file name.
+#-renamesourcefileattribute SourceFile
diff --git a/examples/llama.android/app/src/main/AndroidManifest.xml b/examples/llama.android/app/src/main/AndroidManifest.xml
new file mode 100644
index 0000000000000..41a358a299154
--- /dev/null
+++ b/examples/llama.android/app/src/main/AndroidManifest.xml
@@ -0,0 +1,30 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/examples/llama.android/app/src/main/cpp/CMakeLists.txt b/examples/llama.android/app/src/main/cpp/CMakeLists.txt
new file mode 100644
index 0000000000000..85139329aa082
--- /dev/null
+++ b/examples/llama.android/app/src/main/cpp/CMakeLists.txt
@@ -0,0 +1,50 @@
+
+# For more information about using CMake with Android Studio, read the
+# documentation: https://d.android.com/studio/projects/add-native-code.html.
+# For more examples on how to use CMake, see https://github.com/android/ndk-samples.
+
+# Sets the minimum CMake version required for this project.
+cmake_minimum_required(VERSION 3.22.1)
+
+# Declares the project name. The project name can be accessed via ${ PROJECT_NAME},
+# Since this is the top level CMakeLists.txt, the project name is also accessible
+# with ${CMAKE_PROJECT_NAME} (both CMake variables are in-sync within the top level
+# build script scope).
+project("llama-android")
+
+include(FetchContent)
+FetchContent_Declare(
+ llama
+ GIT_REPOSITORY https://github.com/ggerganov/llama.cpp
+ GIT_TAG master
+)
+
+# Also provides "common"
+FetchContent_MakeAvailable(llama)
+
+# Creates and names a library, sets it as either STATIC
+# or SHARED, and provides the relative paths to its source code.
+# You can define multiple libraries, and CMake builds them for you.
+# Gradle automatically packages shared libraries with your APK.
+#
+# In this top level CMakeLists.txt, ${CMAKE_PROJECT_NAME} is used to define
+# the target library name; in the sub-module's CMakeLists.txt, ${PROJECT_NAME}
+# is preferred for the same purpose.
+#
+# In order to load a library into your app from Java/Kotlin, you must call
+# System.loadLibrary() and pass the name of the library defined here;
+# for GameActivity/NativeActivity derived applications, the same library name must be
+# used in the AndroidManifest.xml file.
+add_library(${CMAKE_PROJECT_NAME} SHARED
+ # List C/C++ source files with relative paths to this CMakeLists.txt.
+ llama-android.cpp)
+
+# Specifies libraries CMake should link to your target library. You
+# can link libraries from various origins, such as libraries defined in this
+# build script, prebuilt third-party libraries, or Android system libraries.
+target_link_libraries(${CMAKE_PROJECT_NAME}
+ # List libraries link to the target library
+ llama
+ common
+ android
+ log)
diff --git a/examples/llama.android/app/src/main/cpp/llama-android.cpp b/examples/llama.android/app/src/main/cpp/llama-android.cpp
new file mode 100644
index 0000000000000..d5e705dce6ca0
--- /dev/null
+++ b/examples/llama.android/app/src/main/cpp/llama-android.cpp
@@ -0,0 +1,394 @@
+#include
+#include
+#include
+#include
+#include
+#include
+#include "llama.h"
+#include "common/common.h"
+
+// Write C++ code here.
+//
+// Do not forget to dynamically load the C++ library into your application.
+//
+// For instance,
+//
+// In MainActivity.java:
+// static {
+// System.loadLibrary("llama-android");
+// }
+//
+// Or, in MainActivity.kt:
+// companion object {
+// init {
+// System.loadLibrary("llama-android")
+// }
+// }
+
+#define TAG "llama-android.cpp"
+#define LOGi(...) __android_log_print(ANDROID_LOG_INFO, TAG, __VA_ARGS__)
+#define LOGe(...) __android_log_print(ANDROID_LOG_ERROR, TAG, __VA_ARGS__)
+
+jclass la_int_var;
+jmethodID la_int_var_value;
+jmethodID la_int_var_inc;
+
+static void log_callback(ggml_log_level level, const char * fmt, void * data) {
+ if (level == GGML_LOG_LEVEL_ERROR) __android_log_print(ANDROID_LOG_ERROR, TAG, fmt, data);
+ else if (level == GGML_LOG_LEVEL_INFO) __android_log_print(ANDROID_LOG_INFO, TAG, fmt, data);
+ else if (level == GGML_LOG_LEVEL_WARN) __android_log_print(ANDROID_LOG_WARN, TAG, fmt, data);
+ else __android_log_print(ANDROID_LOG_DEFAULT, TAG, fmt, data);
+}
+
+extern "C"
+JNIEXPORT jlong JNICALL
+Java_com_example_llama_Llm_load_1model(JNIEnv *env, jobject, jstring filename) {
+ llama_model_params model_params = llama_model_default_params();
+
+ auto path_to_model = env->GetStringUTFChars(filename, 0);
+ LOGi("Loading model from %s", path_to_model);
+
+ auto model = llama_load_model_from_file(path_to_model, model_params);
+ env->ReleaseStringUTFChars(filename, path_to_model);
+
+ if (!model) {
+ LOGe("load_model() failed");
+ env->ThrowNew(env->FindClass("java/lang/IllegalStateException"), "load_model() failed");
+ return 0;
+ }
+
+ return reinterpret_cast(model);
+}
+
+extern "C"
+JNIEXPORT void JNICALL
+Java_com_example_llama_Llm_free_1model(JNIEnv *, jobject, jlong model) {
+ llama_free_model(reinterpret_cast(model));
+}
+
+extern "C"
+JNIEXPORT jlong JNICALL
+Java_com_example_llama_Llm_new_1context(JNIEnv *env, jobject, jlong jmodel) {
+ auto model = reinterpret_cast(jmodel);
+
+ if (!model) {
+ LOGe("new_context(): model cannot be null");
+ env->ThrowNew(env->FindClass("java/lang/IllegalArgumentException"), "Model cannot be null");
+ return 0;
+ }
+
+ int n_threads = std::max(1, std::min(8, (int) sysconf(_SC_NPROCESSORS_ONLN) - 2));
+ LOGi("Using %d threads", n_threads);
+
+ llama_context_params ctx_params = llama_context_default_params();
+ ctx_params.seed = 1234;
+ ctx_params.n_ctx = 2048;
+ ctx_params.n_threads = n_threads;
+ ctx_params.n_threads_batch = n_threads;
+
+ llama_context * context = llama_new_context_with_model(model, ctx_params);
+
+ if (!context) {
+ LOGe("llama_new_context_with_model() returned null)");
+ env->ThrowNew(env->FindClass("java/lang/IllegalStateException"),
+ "llama_new_context_with_model() returned null)");
+ return 0;
+ }
+
+ return reinterpret_cast(context);
+}
+
+extern "C"
+JNIEXPORT void JNICALL
+Java_com_example_llama_Llm_free_1context(JNIEnv *, jobject, jlong context) {
+ llama_free(reinterpret_cast(context));
+}
+
+extern "C"
+JNIEXPORT void JNICALL
+Java_com_example_llama_Llm_backend_1free(JNIEnv *, jobject) {
+ llama_backend_free();
+}
+
+extern "C"
+JNIEXPORT void JNICALL
+Java_com_example_llama_Llm_log_1to_1android(JNIEnv *, jobject) {
+ llama_log_set(log_callback, NULL);
+}
+
+extern "C"
+JNIEXPORT jstring JNICALL
+Java_com_example_llama_Llm_bench_1model(
+ JNIEnv *env,
+ jobject,
+ jlong context_pointer,
+ jlong model_pointer,
+ jlong batch_pointer,
+ jint pp,
+ jint tg,
+ jint pl,
+ jint nr
+ ) {
+ auto pp_avg = 0.0;
+ auto tg_avg = 0.0;
+ auto pp_std = 0.0;
+ auto tg_std = 0.0;
+
+ const auto context = reinterpret_cast(context_pointer);
+ const auto model = reinterpret_cast(model_pointer);
+ const auto batch = reinterpret_cast(batch_pointer);
+
+ const int n_ctx = llama_n_ctx(context);
+
+ LOGi("n_ctx = %d", n_ctx);
+
+ int i, j;
+ int nri;
+ for (nri = 0; nri < nr; nri++) {
+ LOGi("Benchmark prompt processing (pp)");
+
+ llama_batch_clear(*batch);
+
+ const int n_tokens = pp;
+ for (i = 0; i < n_tokens; i++) {
+ llama_batch_add(*batch, 0, i, { 0 }, false);
+ }
+
+ batch->logits[batch->n_tokens - 1] = true;
+ llama_kv_cache_clear(context);
+
+ const auto t_pp_start = ggml_time_us();
+ if (llama_decode(context, *batch) != 0) {
+ LOGi("llama_decode() failed during prompt processing");
+ }
+ const auto t_pp_end = ggml_time_us();
+
+ // bench text generation
+
+ LOGi("Benchmark text generation (tg)");
+
+ llama_kv_cache_clear(context);
+ const auto t_tg_start = ggml_time_us();
+ for (i = 0; i < tg; i++) {
+
+ llama_batch_clear(*batch);
+ for (j = 0; j < pl; j++) {
+ llama_batch_add(*batch, 0, i, { j }, true);
+ }
+
+ LOGi("llama_decode() text generation: %d", i);
+ if (llama_decode(context, *batch) != 0) {
+ LOGi("llama_decode() failed during text generation");
+ }
+ }
+
+ const auto t_tg_end = ggml_time_us();
+
+ llama_kv_cache_clear(context);
+
+ const auto t_pp = double(t_pp_end - t_pp_start) / 1000000.0;
+ const auto t_tg = double(t_tg_end - t_tg_start) / 1000000.0;
+
+ const auto speed_pp = double(pp) / t_pp;
+ const auto speed_tg = double(pl * tg) / t_tg;
+
+ pp_avg += speed_pp;
+ tg_avg += speed_tg;
+
+ pp_std += speed_pp * speed_pp;
+ tg_std += speed_tg * speed_tg;
+
+ LOGi("pp %f t/s, tg %f t/s", speed_pp, speed_tg);
+ }
+
+ pp_avg /= double(nr);
+ tg_avg /= double(nr);
+
+ if (nr > 1) {
+ pp_std = sqrt(pp_std / double(nr - 1) - pp_avg * pp_avg * double(nr) / double(nr - 1));
+ tg_std = sqrt(tg_std / double(nr - 1) - tg_avg * tg_avg * double(nr) / double(nr - 1));
+ } else {
+ pp_std = 0;
+ tg_std = 0;
+ }
+
+ char model_desc[128];
+ llama_model_desc(model, model_desc, sizeof(model_desc));
+
+ const auto model_size = double(llama_model_size(model)) / 1024.0 / 1024.0 / 1024.0;
+ const auto model_n_params = double(llama_model_n_params(model)) / 1e9;
+
+ const auto backend = "(Android)"; // TODO: What should this be?
+
+ std::stringstream result;
+ result << std::setprecision(2);
+ result << "| model | size | params | backend | test | t/s |\n";
+ result << "| --- | --- | --- | --- | --- | --- |\n";
+ result << "| " << model_desc << " | " << model_size << "GiB | " << model_n_params << "B | " << backend << " | pp " << pp << " | " << pp_avg << " ± " << pp_std << " |\n";
+ result << "| " << model_desc << " | " << model_size << "GiB | " << model_n_params << "B | " << backend << " | tg " << tg << " | " << tg_avg << " ± " << tg_std << " |\n";
+
+ return env->NewStringUTF(result.str().c_str());
+}
+
+extern "C"
+JNIEXPORT void JNICALL
+Java_com_example_llama_Llm_free_1batch(JNIEnv *, jobject, jlong batch_pointer) {
+ llama_batch_free(*reinterpret_cast(batch_pointer));
+}
+
+extern "C"
+JNIEXPORT jlong JNICALL
+Java_com_example_llama_Llm_new_1batch(JNIEnv *, jobject, jint n_tokens, jint embd, jint n_seq_max) {
+
+ // Source: Copy of llama.cpp:llama_batch_init but heap-allocated.
+
+ llama_batch *batch = new llama_batch {
+ 0,
+ nullptr,
+ nullptr,
+ nullptr,
+ nullptr,
+ nullptr,
+ nullptr,
+ 0,
+ 0,
+ 0,
+ };
+
+ if (embd) {
+ batch->embd = (float *) malloc(sizeof(float) * n_tokens * embd);
+ } else {
+ batch->token = (llama_token *) malloc(sizeof(llama_token) * n_tokens);
+ }
+
+ batch->pos = (llama_pos *) malloc(sizeof(llama_pos) * n_tokens);
+ batch->n_seq_id = (int32_t *) malloc(sizeof(int32_t) * n_tokens);
+ batch->seq_id = (llama_seq_id **) malloc(sizeof(llama_seq_id *) * n_tokens);
+ for (int i = 0; i < n_tokens; ++i) {
+ batch->seq_id[i] = (llama_seq_id *) malloc(sizeof(llama_seq_id) * n_seq_max);
+ }
+ batch->logits = (int8_t *) malloc(sizeof(int8_t) * n_tokens);
+
+ return reinterpret_cast(batch);
+}
+
+extern "C"
+JNIEXPORT void JNICALL
+Java_com_example_llama_Llm_backend_1init(JNIEnv *, jobject, jboolean numa) {
+ llama_backend_init(numa);
+}
+
+extern "C"
+JNIEXPORT jstring JNICALL
+Java_com_example_llama_Llm_system_1info(JNIEnv *env, jobject) {
+ return env->NewStringUTF(llama_print_system_info());
+}
+
+extern "C"
+JNIEXPORT jint JNICALL
+Java_com_example_llama_Llm_completion_1init(
+ JNIEnv *env,
+ jobject,
+ jlong context_pointer,
+ jlong batch_pointer,
+ jstring jtext,
+ jint n_len
+ ) {
+
+ const auto text = env->GetStringUTFChars(jtext, 0);
+ const auto context = reinterpret_cast(context_pointer);
+ const auto batch = reinterpret_cast(batch_pointer);
+
+ const auto tokens_list = llama_tokenize(context, text, 1);
+
+ auto n_ctx = llama_n_ctx(context);
+ auto n_kv_req = tokens_list.size() + (n_len - tokens_list.size());
+
+ LOGi("n_len = %d, n_ctx = %d, n_kv_req = %d", n_len, n_ctx, n_kv_req);
+
+ if (n_kv_req > n_ctx) {
+ LOGe("error: n_kv_req > n_ctx, the required KV cache size is not big enough");
+ }
+
+ for (auto id : tokens_list) {
+ LOGi("%s", llama_token_to_piece(context, id).c_str());
+ }
+
+ llama_batch_clear(*batch);
+
+ // evaluate the initial prompt
+ for (auto i = 0; i < tokens_list.size(); i++) {
+ llama_batch_add(*batch, tokens_list[i], i, { 0 }, false);
+ }
+
+ // llama_decode will output logits only for the last token of the prompt
+ batch->logits[batch->n_tokens - 1] = true;
+
+ if (llama_decode(context, *batch) != 0) {
+ LOGe("llama_decode() failed");
+ }
+
+ env->ReleaseStringUTFChars(jtext, text);
+
+ return batch->n_tokens;
+}
+
+extern "C"
+JNIEXPORT jstring JNICALL
+Java_com_example_llama_Llm_completion_1loop(
+ JNIEnv * env,
+ jobject,
+ jlong context_pointer,
+ jlong batch_pointer,
+ jint n_len,
+ jobject intvar_ncur
+) {
+ const auto context = reinterpret_cast(context_pointer);
+ const auto batch = reinterpret_cast(batch_pointer);
+ const auto model = llama_get_model(context);
+
+ if (!la_int_var) la_int_var = env->GetObjectClass(intvar_ncur);
+ if (!la_int_var_value) la_int_var_value = env->GetMethodID(la_int_var, "getValue", "()I");
+ if (!la_int_var_inc) la_int_var_inc = env->GetMethodID(la_int_var, "inc", "()V");
+
+ auto n_vocab = llama_n_vocab(model);
+ auto logits = llama_get_logits_ith(context, batch->n_tokens - 1);
+
+ std::vector candidates;
+ candidates.reserve(n_vocab);
+
+ for (llama_token token_id = 0; token_id < n_vocab; token_id++) {
+ candidates.emplace_back(llama_token_data{ token_id, logits[token_id], 0.0f });
+ }
+
+ llama_token_data_array candidates_p = { candidates.data(), candidates.size(), false };
+
+ // sample the most likely token
+ const auto new_token_id = llama_sample_token_greedy(context, &candidates_p);
+
+ const auto n_cur = env->CallIntMethod(intvar_ncur, la_int_var_value);
+ if (new_token_id == llama_token_eos(model) || n_cur == n_len) {
+ return env->NewStringUTF("");
+ }
+
+ auto new_token_chars = llama_token_to_piece(context, new_token_id);
+ LOGi("new_token_chars: `%s`", new_token_chars.c_str());
+ auto new_token = env->NewStringUTF(new_token_chars.c_str());
+
+ llama_batch_clear(*batch);
+ llama_batch_add(*batch, new_token_id, n_cur, { 0 }, true);
+
+ env->CallVoidMethod(intvar_ncur, la_int_var_inc);
+
+ if (llama_decode(context, *batch) != 0) {
+ LOGe("llama_decode() returned null");
+ }
+
+ return new_token;
+}
+
+extern "C"
+JNIEXPORT void JNICALL
+Java_com_example_llama_Llm_kv_1cache_1clear(JNIEnv *, jobject, jlong context) {
+ llama_kv_cache_clear(reinterpret_cast(context));
+}
diff --git a/examples/llama.android/app/src/main/java/com/example/llama/Downloadable.kt b/examples/llama.android/app/src/main/java/com/example/llama/Downloadable.kt
new file mode 100644
index 0000000000000..78c231ae55d8c
--- /dev/null
+++ b/examples/llama.android/app/src/main/java/com/example/llama/Downloadable.kt
@@ -0,0 +1,119 @@
+package com.example.llama
+
+import android.app.DownloadManager
+import android.net.Uri
+import android.util.Log
+import androidx.compose.material3.Button
+import androidx.compose.material3.Text
+import androidx.compose.runtime.Composable
+import androidx.compose.runtime.getValue
+import androidx.compose.runtime.mutableDoubleStateOf
+import androidx.compose.runtime.mutableStateOf
+import androidx.compose.runtime.remember
+import androidx.compose.runtime.rememberCoroutineScope
+import androidx.compose.runtime.setValue
+import androidx.core.database.getLongOrNull
+import androidx.core.net.toUri
+import kotlinx.coroutines.delay
+import kotlinx.coroutines.launch
+import java.io.File
+
+data class Downloadable(val name: String, val source: Uri, val destination: File) {
+ companion object {
+ @JvmStatic
+ private val tag: String? = this::class.qualifiedName
+
+ sealed interface State
+ data object Ready: State
+ data class Downloading(val id: Long): State
+ data class Downloaded(val downloadable: Downloadable): State
+ data class Error(val message: String): State
+
+ @JvmStatic
+ @Composable
+ fun Button(viewModel: MainViewModel, dm: DownloadManager, item: Downloadable) {
+ var status: State by remember {
+ mutableStateOf(
+ if (item.destination.exists()) Downloaded(item)
+ else Ready
+ )
+ }
+ var progress by remember { mutableDoubleStateOf(0.0) }
+
+ val coroutineScope = rememberCoroutineScope()
+
+ suspend fun waitForDownload(result: Downloading, item: Downloadable): State {
+ while (true) {
+ val cursor = dm.query(DownloadManager.Query().setFilterById(result.id))
+
+ if (cursor == null) {
+ Log.e(tag, "dm.query() returned null")
+ return Error("dm.query() returned null")
+ }
+
+ if (!cursor.moveToFirst() || cursor.count < 1) {
+ cursor.close()
+ Log.i(tag, "cursor.moveToFirst() returned false or cursor.count < 1, download canceled?")
+ return Ready
+ }
+
+ val pix = cursor.getColumnIndex(DownloadManager.COLUMN_BYTES_DOWNLOADED_SO_FAR)
+ val tix = cursor.getColumnIndex(DownloadManager.COLUMN_TOTAL_SIZE_BYTES)
+ val sofar = cursor.getLongOrNull(pix) ?: 0
+ val total = cursor.getLongOrNull(tix) ?: 1
+ cursor.close()
+
+ if (sofar == total) {
+ return Downloaded(item)
+ }
+
+ progress = (sofar * 1.0) / total
+
+ delay(1000L)
+ }
+ }
+
+ fun onClick() {
+ when (val s = status) {
+ is Downloaded -> {
+ viewModel.load(item.destination.path)
+ }
+
+ is Downloading -> {
+ coroutineScope.launch {
+ status = waitForDownload(s, item)
+ }
+ }
+
+ else -> {
+ item.destination.delete()
+
+ val request = DownloadManager.Request(item.source).apply {
+ setTitle("Downloading model")
+ setDescription("Downloading model: ${item.name}")
+ setAllowedNetworkTypes(DownloadManager.Request.NETWORK_WIFI)
+ setDestinationUri(item.destination.toUri())
+ }
+
+ viewModel.log("Saving ${item.name} to ${item.destination.path}")
+ Log.i(tag, "Saving ${item.name} to ${item.destination.path}")
+
+ val id = dm.enqueue(request)
+ status = Downloading(id)
+ onClick()
+ }
+ }
+ }
+
+ Button(onClick = { onClick() }, enabled = status !is Downloading) {
+ when (status) {
+ is Downloading -> Text(text = "Downloading ${(progress * 100).toInt()}%")
+ is Downloaded -> Text("Load ${item.name}")
+ is Ready -> Text("Download ${item.name}")
+ is Error -> Text("Download ${item.name}")
+ }
+ }
+ }
+
+ }
+}
diff --git a/examples/llama.android/app/src/main/java/com/example/llama/Llm.kt b/examples/llama.android/app/src/main/java/com/example/llama/Llm.kt
new file mode 100644
index 0000000000000..5f32703724a49
--- /dev/null
+++ b/examples/llama.android/app/src/main/java/com/example/llama/Llm.kt
@@ -0,0 +1,172 @@
+package com.example.llama
+
+import android.util.Log
+import kotlinx.coroutines.CoroutineDispatcher
+import kotlinx.coroutines.asCoroutineDispatcher
+import kotlinx.coroutines.flow.Flow
+import kotlinx.coroutines.flow.flow
+import kotlinx.coroutines.flow.flowOn
+import kotlinx.coroutines.withContext
+import java.util.concurrent.Executors
+import kotlin.concurrent.thread
+
+class Llm {
+ private val tag: String? = this::class.simpleName
+
+ private val threadLocalState: ThreadLocal = ThreadLocal.withInitial { State.Idle }
+
+ private val runLoop: CoroutineDispatcher = Executors.newSingleThreadExecutor {
+ thread(start = false, name = "Llm-RunLoop") {
+ Log.d(tag, "Dedicated thread for native code: ${Thread.currentThread().name}")
+
+ // No-op if called more than once.
+ System.loadLibrary("llama-android")
+
+ // Set llama log handler to Android
+ log_to_android()
+ backend_init(false)
+
+ Log.d(tag, system_info())
+
+ it.run()
+ }.apply {
+ uncaughtExceptionHandler = Thread.UncaughtExceptionHandler { _, exception: Throwable ->
+ Log.e(tag, "Unhandled exception", exception)
+ }
+ }
+ }.asCoroutineDispatcher()
+
+ private val nlen: Int = 64
+
+ private external fun log_to_android()
+ private external fun load_model(filename: String): Long
+ private external fun free_model(model: Long)
+ private external fun new_context(model: Long): Long
+ private external fun free_context(context: Long)
+ private external fun backend_init(numa: Boolean)
+ private external fun backend_free()
+ private external fun free_batch(batch: Long)
+ private external fun new_batch(nTokens: Int, embd: Int, nSeqMax: Int): Long
+ private external fun bench_model(
+ context: Long,
+ model: Long,
+ batch: Long,
+ pp: Int,
+ tg: Int,
+ pl: Int,
+ nr: Int
+ ): String
+
+ private external fun system_info(): String
+
+ private external fun completion_init(
+ context: Long,
+ batch: Long,
+ text: String,
+ nLen: Int
+ ): Int
+
+ private external fun completion_loop(
+ context: Long,
+ batch: Long,
+ nLen: Int,
+ ncur: IntVar
+ ): String
+
+ private external fun kv_cache_clear(context: Long)
+
+ suspend fun bench(pp: Int, tg: Int, pl: Int, nr: Int = 1): String {
+ return withContext(runLoop) {
+ when (val state = threadLocalState.get()) {
+ is State.Loaded -> {
+ Log.d(tag, "bench(): $state")
+ bench_model(state.context, state.model, state.batch, pp, tg, pl, nr)
+ }
+
+ else -> throw IllegalStateException("No model loaded")
+ }
+ }
+ }
+
+ suspend fun load(pathToModel: String) {
+ withContext(runLoop) {
+ when (threadLocalState.get()) {
+ is State.Idle -> {
+ val model = load_model(pathToModel)
+ if (model == 0L) throw IllegalStateException("load_model() failed")
+
+ val context = new_context(model)
+ if (context == 0L) throw IllegalStateException("new_context() failed")
+
+ val batch = new_batch(512, 0, 1)
+ if (batch == 0L) throw IllegalStateException("new_batch() failed")
+
+ Log.i(tag, "Loaded model $pathToModel")
+ threadLocalState.set(State.Loaded(model, context, batch))
+ }
+ else -> throw IllegalStateException("Model already loaded")
+ }
+ }
+ }
+
+ fun send(message: String): Flow = flow {
+ when (val state = threadLocalState.get()) {
+ is State.Loaded -> {
+ val ncur = IntVar(completion_init(state.context, state.batch, message, nlen))
+ while (ncur.value <= nlen) {
+ val str = completion_loop(state.context, state.batch, nlen, ncur)
+ if (str.isEmpty()) {
+ break
+ }
+ emit(str)
+ }
+ kv_cache_clear(state.context)
+ }
+ else -> {}
+ }
+ }.flowOn(runLoop)
+
+ /**
+ * Unloads the model and frees resources.
+ *
+ * This is a no-op if there's no model loaded.
+ */
+ suspend fun unload() {
+ withContext(runLoop) {
+ when (val state = threadLocalState.get()) {
+ is State.Loaded -> {
+ free_context(state.context)
+ free_model(state.model)
+ free_batch(state.batch)
+
+ threadLocalState.set(State.Idle)
+ }
+ else -> {}
+ }
+ }
+ }
+
+ companion object {
+ private class IntVar(value: Int) {
+ @Volatile
+ var value: Int = value
+ private set
+
+ fun inc() {
+ synchronized(this) {
+ value += 1
+ }
+ }
+ }
+
+ private sealed interface State {
+ data object Idle: State
+ data class Loaded(val model: Long, val context: Long, val batch: Long): State
+ }
+
+ // Enforce only one instance of Llm.
+ private val _instance: Llm = Llm()
+
+ fun instance(): Llm = _instance
+ }
+}
diff --git a/examples/llama.android/app/src/main/java/com/example/llama/MainActivity.kt b/examples/llama.android/app/src/main/java/com/example/llama/MainActivity.kt
new file mode 100644
index 0000000000000..9da04f7d3c32e
--- /dev/null
+++ b/examples/llama.android/app/src/main/java/com/example/llama/MainActivity.kt
@@ -0,0 +1,154 @@
+package com.example.llama
+
+import android.app.ActivityManager
+import android.app.DownloadManager
+import android.content.ClipData
+import android.content.ClipboardManager
+import android.net.Uri
+import android.os.Bundle
+import android.os.StrictMode
+import android.os.StrictMode.VmPolicy
+import android.text.format.Formatter
+import androidx.activity.ComponentActivity
+import androidx.activity.compose.setContent
+import androidx.activity.viewModels
+import androidx.compose.foundation.layout.Box
+import androidx.compose.foundation.layout.Column
+import androidx.compose.foundation.layout.Row
+import androidx.compose.foundation.layout.fillMaxSize
+import androidx.compose.foundation.layout.padding
+import androidx.compose.foundation.lazy.LazyColumn
+import androidx.compose.foundation.lazy.items
+import androidx.compose.foundation.lazy.rememberLazyListState
+import androidx.compose.material3.Button
+import androidx.compose.material3.LocalContentColor
+import androidx.compose.material3.MaterialTheme
+import androidx.compose.material3.OutlinedTextField
+import androidx.compose.material3.Surface
+import androidx.compose.material3.Text
+import androidx.compose.runtime.Composable
+import androidx.compose.ui.Modifier
+import androidx.compose.ui.unit.dp
+import androidx.core.content.getSystemService
+import com.example.llama.ui.theme.LlamaAndroidTheme
+import java.io.File
+
+class MainActivity(
+ activityManager: ActivityManager? = null,
+ downloadManager: DownloadManager? = null,
+ clipboardManager: ClipboardManager? = null,
+): ComponentActivity() {
+ private val tag: String? = this::class.simpleName
+
+ private val activityManager by lazy { activityManager ?: getSystemService()!! }
+ private val downloadManager by lazy { downloadManager ?: getSystemService()!! }
+ private val clipboardManager by lazy { clipboardManager ?: getSystemService()!! }
+
+ private val viewModel: MainViewModel by viewModels()
+
+ // Get a MemoryInfo object for the device's current memory status.
+ private fun availableMemory(): ActivityManager.MemoryInfo {
+ return ActivityManager.MemoryInfo().also { memoryInfo ->
+ activityManager.getMemoryInfo(memoryInfo)
+ }
+ }
+
+ override fun onCreate(savedInstanceState: Bundle?) {
+ super.onCreate(savedInstanceState)
+
+ StrictMode.setVmPolicy(
+ VmPolicy.Builder(StrictMode.getVmPolicy())
+ .detectLeakedClosableObjects()
+ .build()
+ )
+
+ val free = Formatter.formatFileSize(this, availableMemory().availMem)
+ val total = Formatter.formatFileSize(this, availableMemory().totalMem)
+
+ viewModel.log("Current memory: $free / $total")
+ viewModel.log("Downloads directory: ${getExternalFilesDir(null)}")
+
+ val extFilesDir = getExternalFilesDir(null)
+
+ val models = listOf(
+ Downloadable(
+ "Phi-2 7B (Q4_0, 1.6 GiB)",
+ Uri.parse("https://huggingface.co/ggml-org/models/resolve/main/phi-2/ggml-model-q4_0.gguf?download=true"),
+ File(extFilesDir, "phi-2-q4_0.gguf"),
+ ),
+ Downloadable(
+ "TinyLlama 1.1B (f16, 2.2 GiB)",
+ Uri.parse("https://huggingface.co/ggml-org/models/resolve/main/tinyllama-1.1b/ggml-model-f16.gguf?download=true"),
+ File(extFilesDir, "tinyllama-1.1-f16.gguf"),
+ ),
+ Downloadable(
+ "Phi 2 DPO (Q3_K_M, 1.48 GiB)",
+ Uri.parse("https://huggingface.co/TheBloke/phi-2-dpo-GGUF/resolve/main/phi-2-dpo.Q3_K_M.gguf?download=true"),
+ File(extFilesDir, "phi-2-dpo.Q3_K_M.gguf")
+ ),
+ )
+
+ setContent {
+ LlamaAndroidTheme {
+ // A surface container using the 'background' color from the theme
+ Surface(
+ modifier = Modifier.fillMaxSize(),
+ color = MaterialTheme.colorScheme.background
+ ) {
+ MainCompose(
+ viewModel,
+ clipboardManager,
+ downloadManager,
+ models,
+ )
+ }
+
+ }
+ }
+ }
+}
+
+@Composable
+fun MainCompose(
+ viewModel: MainViewModel,
+ clipboard: ClipboardManager,
+ dm: DownloadManager,
+ models: List
+) {
+ Column {
+ val scrollState = rememberLazyListState()
+
+ Box(modifier = Modifier.weight(1f)) {
+ LazyColumn(state = scrollState) {
+ items(viewModel.messages) {
+ Text(
+ it,
+ style = MaterialTheme.typography.bodyLarge.copy(color = LocalContentColor.current),
+ modifier = Modifier.padding(16.dp)
+ )
+ }
+ }
+ }
+ OutlinedTextField(
+ value = viewModel.message,
+ onValueChange = { viewModel.updateMessage(it) },
+ label = { Text("Message") },
+ )
+ Row {
+ Button({ viewModel.send() }) { Text("Send") }
+ Button({ viewModel.bench(8, 4, 1) }) { Text("Bench") }
+ Button({ viewModel.clear() }) { Text("Clear") }
+ Button({
+ viewModel.messages.joinToString("\n").let {
+ clipboard.setPrimaryClip(ClipData.newPlainText("", it))
+ }
+ }) { Text("Copy") }
+ }
+
+ Column {
+ for (model in models) {
+ Downloadable.Button(viewModel, dm, model)
+ }
+ }
+ }
+}
diff --git a/examples/llama.android/app/src/main/java/com/example/llama/MainViewModel.kt b/examples/llama.android/app/src/main/java/com/example/llama/MainViewModel.kt
new file mode 100644
index 0000000000000..be95e22218332
--- /dev/null
+++ b/examples/llama.android/app/src/main/java/com/example/llama/MainViewModel.kt
@@ -0,0 +1,104 @@
+package com.example.llama
+
+import android.util.Log
+import androidx.compose.runtime.getValue
+import androidx.compose.runtime.mutableStateOf
+import androidx.compose.runtime.setValue
+import androidx.lifecycle.ViewModel
+import androidx.lifecycle.viewModelScope
+import kotlinx.coroutines.flow.catch
+import kotlinx.coroutines.launch
+
+class MainViewModel(private val llm: Llm = Llm.instance()): ViewModel() {
+ companion object {
+ @JvmStatic
+ private val NanosPerSecond = 1_000_000_000.0
+ }
+
+ private val tag: String? = this::class.simpleName
+
+ var messages by mutableStateOf(listOf("Initializing..."))
+ private set
+
+ var message by mutableStateOf("")
+ private set
+
+ override fun onCleared() {
+ super.onCleared()
+
+ viewModelScope.launch {
+ try {
+ llm.unload()
+ } catch (exc: IllegalStateException) {
+ messages += exc.message!!
+ }
+ }
+ }
+
+ fun send() {
+ val text = message
+ message = ""
+
+ // Add to messages console.
+ messages += text
+ messages += ""
+
+ viewModelScope.launch {
+ llm.send(text)
+ .catch {
+ Log.e(tag, "send() failed", it)
+ messages += it.message!!
+ }
+ .collect { messages = messages.dropLast(1) + (messages.last() + it) }
+ }
+ }
+
+ fun bench(pp: Int, tg: Int, pl: Int, nr: Int = 1) {
+ viewModelScope.launch {
+ try {
+ val start = System.nanoTime()
+ val warmupResult = llm.bench(pp, tg, pl, nr)
+ val end = System.nanoTime()
+
+ messages += warmupResult
+
+ val warmup = (end - start).toDouble() / NanosPerSecond
+ messages += "Warm up time: $warmup seconds, please wait..."
+
+ if (warmup > 5.0) {
+ messages += "Warm up took too long, aborting benchmark"
+ return@launch
+ }
+
+ messages += llm.bench(512, 128, 1, 3)
+ } catch (exc: IllegalStateException) {
+ Log.e(tag, "bench() failed", exc)
+ messages += exc.message!!
+ }
+ }
+ }
+
+ fun load(pathToModel: String) {
+ viewModelScope.launch {
+ try {
+ llm.load(pathToModel)
+ messages += "Loaded $pathToModel"
+ } catch (exc: IllegalStateException) {
+ Log.e(tag, "load() failed", exc)
+ messages += exc.message!!
+ }
+ }
+ }
+
+ fun updateMessage(newMessage: String) {
+ message = newMessage
+ }
+
+ fun clear() {
+ messages = listOf()
+ }
+
+ fun log(message: String) {
+ messages += message
+ }
+}
diff --git a/examples/llama.android/app/src/main/java/com/example/llama/ui/theme/Color.kt b/examples/llama.android/app/src/main/java/com/example/llama/ui/theme/Color.kt
new file mode 100644
index 0000000000000..40c30e8d97077
--- /dev/null
+++ b/examples/llama.android/app/src/main/java/com/example/llama/ui/theme/Color.kt
@@ -0,0 +1,11 @@
+package com.example.llama.ui.theme
+
+import androidx.compose.ui.graphics.Color
+
+val Purple80 = Color(0xFFD0BCFF)
+val PurpleGrey80 = Color(0xFFCCC2DC)
+val Pink80 = Color(0xFFEFB8C8)
+
+val Purple40 = Color(0xFF6650a4)
+val PurpleGrey40 = Color(0xFF625b71)
+val Pink40 = Color(0xFF7D5260)
diff --git a/examples/llama.android/app/src/main/java/com/example/llama/ui/theme/Theme.kt b/examples/llama.android/app/src/main/java/com/example/llama/ui/theme/Theme.kt
new file mode 100644
index 0000000000000..e742220a8d719
--- /dev/null
+++ b/examples/llama.android/app/src/main/java/com/example/llama/ui/theme/Theme.kt
@@ -0,0 +1,70 @@
+package com.example.llama.ui.theme
+
+import android.app.Activity
+import android.os.Build
+import androidx.compose.foundation.isSystemInDarkTheme
+import androidx.compose.material3.MaterialTheme
+import androidx.compose.material3.darkColorScheme
+import androidx.compose.material3.dynamicDarkColorScheme
+import androidx.compose.material3.dynamicLightColorScheme
+import androidx.compose.material3.lightColorScheme
+import androidx.compose.runtime.Composable
+import androidx.compose.runtime.SideEffect
+import androidx.compose.ui.graphics.toArgb
+import androidx.compose.ui.platform.LocalContext
+import androidx.compose.ui.platform.LocalView
+import androidx.core.view.WindowCompat
+
+private val DarkColorScheme = darkColorScheme(
+ primary = Purple80,
+ secondary = PurpleGrey80,
+ tertiary = Pink80
+)
+
+private val LightColorScheme = lightColorScheme(
+ primary = Purple40,
+ secondary = PurpleGrey40,
+ tertiary = Pink40
+
+ /* Other default colors to override
+ background = Color(0xFFFFFBFE),
+ surface = Color(0xFFFFFBFE),
+ onPrimary = Color.White,
+ onSecondary = Color.White,
+ onTertiary = Color.White,
+ onBackground = Color(0xFF1C1B1F),
+ onSurface = Color(0xFF1C1B1F),
+ */
+)
+
+@Composable
+fun LlamaAndroidTheme(
+ darkTheme: Boolean = isSystemInDarkTheme(),
+ // Dynamic color is available on Android 12+
+ dynamicColor: Boolean = true,
+ content: @Composable () -> Unit
+) {
+ val colorScheme = when {
+ dynamicColor && Build.VERSION.SDK_INT >= Build.VERSION_CODES.S -> {
+ val context = LocalContext.current
+ if (darkTheme) dynamicDarkColorScheme(context) else dynamicLightColorScheme(context)
+ }
+
+ darkTheme -> DarkColorScheme
+ else -> LightColorScheme
+ }
+ val view = LocalView.current
+ if (!view.isInEditMode) {
+ SideEffect {
+ val window = (view.context as Activity).window
+ window.statusBarColor = colorScheme.primary.toArgb()
+ WindowCompat.getInsetsController(window, view).isAppearanceLightStatusBars = darkTheme
+ }
+ }
+
+ MaterialTheme(
+ colorScheme = colorScheme,
+ typography = Typography,
+ content = content
+ )
+}
diff --git a/examples/llama.android/app/src/main/java/com/example/llama/ui/theme/Type.kt b/examples/llama.android/app/src/main/java/com/example/llama/ui/theme/Type.kt
new file mode 100644
index 0000000000000..0b87946ca3ab1
--- /dev/null
+++ b/examples/llama.android/app/src/main/java/com/example/llama/ui/theme/Type.kt
@@ -0,0 +1,34 @@
+package com.example.llama.ui.theme
+
+import androidx.compose.material3.Typography
+import androidx.compose.ui.text.TextStyle
+import androidx.compose.ui.text.font.FontFamily
+import androidx.compose.ui.text.font.FontWeight
+import androidx.compose.ui.unit.sp
+
+// Set of Material typography styles to start with
+val Typography = Typography(
+ bodyLarge = TextStyle(
+ fontFamily = FontFamily.Default,
+ fontWeight = FontWeight.Normal,
+ fontSize = 16.sp,
+ lineHeight = 24.sp,
+ letterSpacing = 0.5.sp
+ )
+ /* Other default text styles to override
+ titleLarge = TextStyle(
+ fontFamily = FontFamily.Default,
+ fontWeight = FontWeight.Normal,
+ fontSize = 22.sp,
+ lineHeight = 28.sp,
+ letterSpacing = 0.sp
+ ),
+ labelSmall = TextStyle(
+ fontFamily = FontFamily.Default,
+ fontWeight = FontWeight.Medium,
+ fontSize = 11.sp,
+ lineHeight = 16.sp,
+ letterSpacing = 0.5.sp
+ )
+ */
+)
diff --git a/examples/llama.android/app/src/main/res/drawable/ic_launcher_background.xml b/examples/llama.android/app/src/main/res/drawable/ic_launcher_background.xml
new file mode 100644
index 0000000000000..07d5da9cbf141
--- /dev/null
+++ b/examples/llama.android/app/src/main/res/drawable/ic_launcher_background.xml
@@ -0,0 +1,170 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/examples/llama.android/app/src/main/res/drawable/ic_launcher_foreground.xml b/examples/llama.android/app/src/main/res/drawable/ic_launcher_foreground.xml
new file mode 100644
index 0000000000000..7706ab9e6d407
--- /dev/null
+++ b/examples/llama.android/app/src/main/res/drawable/ic_launcher_foreground.xml
@@ -0,0 +1,30 @@
+
+
+
+
+