From b06a0349a918baa562718b29f88d5ad64b290238 Mon Sep 17 00:00:00 2001 From: juli Date: Sun, 9 Aug 2026 22:06:30 +0200 Subject: [PATCH] On-device fixes: optimized native build, 16 KB align, disable thinking Validated on a Pixel 7 (adb). Key fixes: - Force Release/-O3 for native even in the debug variant (AGP defaulted to -O0, making llama.cpp ~10x too slow / effectively unusable). - 16 KB-align all native LOAD segments (Play requirement; 16 KB-page devices). - Disable Qwen3 thinking (empty prefill + /no_think) so replies are fast and punchy instead of spending the whole budget on hidden reasoning. - Cap replies at 220 tokens. Verified: model downloads, loads (n_ctx=4096, 6 threads), streams an on-persona reply with emoji intact, and "new tsjet" clears the conversation. Co-Authored-By: Claude Opus 4.8 --- README.md | 12 +++++++++--- app/build.gradle.kts | 3 +++ app/src/main/cpp/CMakeLists.txt | 13 +++++++++++++ .../java/monster/autisme/tsjetpiti/MainActivity.kt | 8 +++++--- 4 files changed, 30 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index ee02fb2..0dc421b 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,9 @@ conversation and start fresh. **downloaded on first launch** from Hugging Face into the app's private storage. It is *not* bundled in the APK. (Swap the quant in `ModelDownloader.kt`.) - **Persona:** set via `SYSTEM_PROMPT` in `MainActivity.kt`; sampling is a little - hot (temp 0.9) for playful answers. + hot (temp 0.9) for playful answers. The model is a Qwen3 "thinking" model, so the + prompt disables reasoning (empty `` prefill + `/no_think`) to keep + replies fast and punchy instead of burning the token budget on hidden thoughts. - **Conversation:** the web layer holds the full history and sends it each turn; native rebuilds the ChatML prompt and clears the KV cache before every reply, so "new tsjet" is just: clear JS state + reset cache. @@ -85,5 +87,9 @@ manager, and tap it. First launch downloads the ~1.9 GB model over Wi-Fi. display and from history (`stripThink` in `app.js`). - Bumping the llama.cpp tag? Re-check the C API calls in `llama-jni.cpp` against that tag's `include/llama.h` — it uses the raw C API directly. -- **Build verified:** `./gradlew assembleDebug` produces a ~13 MB `arm64-v8a` APK - (native libs stripped; the ~1.9 GB model downloads on first launch). +- **Native is always built optimized.** AGP compiles the debug variant's C/C++ at + `-O0` by default, which makes llama.cpp ~10x too slow; `CMakeLists.txt` forces + `Release`/`-O3` regardless of variant. Native libs are also 16 KB page-aligned. +- **Verified on-device** (Pixel 7, Android, 4 KB pages): downloads the model, loads + it (`n_ctx=4096`, 6 threads), streams a reply, and "new tsjet" clears the chat. + `./gradlew assembleDebug` → ~13 MB `arm64-v8a` APK (model downloads on first launch). diff --git a/app/build.gradle.kts b/app/build.gradle.kts index dc8829b..02729bc 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -24,7 +24,10 @@ android { } externalNativeBuild { cmake { + // Force an optimized native build even for the debug APK — otherwise + // AGP compiles ggml/llama at -O0 and inference is ~10x too slow. arguments += "-DANDROID_STL=c++_shared" + arguments += "-DCMAKE_BUILD_TYPE=Release" cppFlags += "-std=c++17" } } diff --git a/app/src/main/cpp/CMakeLists.txt b/app/src/main/cpp/CMakeLists.txt index 0809eba..8982cd8 100644 --- a/app/src/main/cpp/CMakeLists.txt +++ b/app/src/main/cpp/CMakeLists.txt @@ -2,6 +2,19 @@ cmake_minimum_required(VERSION 3.22.1) project(tsjet LANGUAGES C CXX) +# AGP compiles the debug variant with CMAKE_BUILD_TYPE=Debug (-O0), which makes +# llama.cpp ~10x too slow. Force an optimized build regardless of the variant. +if(NOT CMAKE_BUILD_TYPE OR CMAKE_BUILD_TYPE STREQUAL "Debug") + set(CMAKE_BUILD_TYPE Release CACHE STRING "" FORCE) +endif() +set(CMAKE_C_FLAGS_DEBUG "-O3 -DNDEBUG") +set(CMAKE_CXX_FLAGS_DEBUG "-O3 -DNDEBUG") + +# Align all native LOAD segments to 16 KB. Required for Android 15+ / devices with +# 16 KB memory pages and for Google Play uploads. Set BEFORE llama.cpp is added so +# its targets (libllama, libggml*) inherit it too. +add_link_options("-Wl,-z,max-page-size=16384" "-Wl,-z,common-page-size=16384") + include(FetchContent) # --- llama.cpp (pinned) ----------------------------------------------------- diff --git a/app/src/main/java/monster/autisme/tsjetpiti/MainActivity.kt b/app/src/main/java/monster/autisme/tsjetpiti/MainActivity.kt index dc746e0..fd46986 100644 --- a/app/src/main/java/monster/autisme/tsjetpiti/MainActivity.kt +++ b/app/src/main/java/monster/autisme/tsjetpiti/MainActivity.kt @@ -34,7 +34,7 @@ class MainActivity : Activity() { companion object { private const val N_CTX = 4096 - private const val MAX_TOKENS = 384 + private const val MAX_TOKENS = 220 // tsjet's persona: confidently, hilariously wrong. private const val SYSTEM_PROMPT = @@ -45,7 +45,7 @@ class MainActivity : Activity() { "fake-but-believable facts, mix up cause and effect, and give reasoning " + "that falls apart the moment you think about it. Keep replies to a few " + "sentences. Never say you are unsure, never refuse, never admit you are " + - "joking, and never give the truly correct answer. Stay in character at all times." + "joking, and never give the truly correct answer. Stay in character at all times. /no_think" } @SuppressLint("SetJavaScriptEnabled") @@ -113,7 +113,9 @@ class MainActivity : Activity() { sb.append("<|im_start|>").append(role).append('\n') .append(content).append("<|im_end|>\n") } - sb.append("<|im_start|>assistant\n") + // Prefill a closed (empty) reasoning block so the model skips and + // answers immediately — tsjet is meant to be fast and punchy, not thoughtful. + sb.append("<|im_start|>assistant\n\n\n\n\n") return sb.toString().toByteArray(Charsets.UTF_8) }