diff --git a/app/build.gradle.kts b/app/build.gradle.kts index 2b8535e..673eb9d 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -21,7 +21,7 @@ android { // versionCode/versionName are overridable from Gradle properties so the release CI // can drive them straight from the git tag (e.g. -PappVersionCode=5 -PappVersionName=1.0.0). // Local builds fall back to the literals below. - versionCode = (project.findProperty("appVersionCode") as String?)?.toIntOrNull() ?: 37 + versionCode = (project.findProperty("appVersionCode") as String?)?.toIntOrNull() ?: 38 versionName = (project.findProperty("appVersionName") as String?) ?: "2.6.0" testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner" diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index 99f6ff4..cd67289 100644 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -62,6 +62,7 @@ android:allowBackup="false" android:label="@string/app_name" android:supportsRtl="true" + android:networkSecurityConfig="@xml/network_security_config" android:icon="@mipmap/ic_launcher" android:roundIcon="@mipmap/ic_launcher_round" android:theme="@android:style/Theme.Material.Light.NoActionBar"> diff --git a/app/src/main/java/com/pulseloop/coach/config/CoachClientResolver.kt b/app/src/main/java/com/pulseloop/coach/config/CoachClientResolver.kt index 1401116..5d036d8 100644 --- a/app/src/main/java/com/pulseloop/coach/config/CoachClientResolver.kt +++ b/app/src/main/java/com/pulseloop/coach/config/CoachClientResolver.kt @@ -1,6 +1,8 @@ package com.pulseloop.coach.config import com.pulseloop.coach.gemini.GeminiClient +import com.pulseloop.coach.local.LocalEndpoint +import com.pulseloop.coach.local.LocalOpenAICompatClient import com.pulseloop.coach.minimax.MiniMaxClient import com.pulseloop.coach.openai.OpenAIResponsesClient import com.pulseloop.coach.openai.ResponsesClient @@ -18,6 +20,11 @@ import com.pulseloop.settings.ApiKeyStore * run (used to gate `CoachFeatureFlags.coachEnabled`). A client is returned * even when the key is absent (`key == null`); the feature-flags gate prevents * an empty-key call. + * + * For [CoachProviderMode.LOCAL_OPENAI_COMPAT] the sentinel is the **base URL**, not a key: a + * self-hosted Ollama/llama.cpp/vLLM/SGLang server is unauthenticated by default, so requiring a + * key there would leave the coach permanently disabled for the normal setup. See + * `docs/local-llm-coach.md`. */ object CoachClientResolver { @@ -30,6 +37,7 @@ object CoachClientResolver { geminiKey: String?, openRouterKey: String?, minimaxKey: String? = null, + localKey: String? = null, openAIClientFactory: (String) -> ResponsesClient = { OpenAIResponsesClient(it) }, ): Resolution = when (settings.providerMode) { CoachProviderMode.USER_GEMINI_KEY -> { @@ -48,6 +56,24 @@ object CoachClientResolver { val key = minimaxKey?.takeIf { it.isNotBlank() } Resolution(key, MiniMaxClient(apiKey = key ?: "", model = settings.resolvedMinimaxModel)) } + CoachProviderMode.LOCAL_OPENAI_COMPAT -> { + // Readiness is a base URL that would actually work — `validate`, not `isNotBlank`. + // Settings persists the field on every keystroke, so a blank-check flips the coach to + // "Active" on the first character typed, and every turn then fails inside `send()` + // with the same URL-validation text the Settings field is already showing inline. + // `localKey` may legitimately be blank and is passed through as null so the client + // omits the Authorization header entirely. + val baseUrl = settings.resolvedLocalBaseUrl + Resolution(baseUrl.takeIf { LocalEndpoint.validate(it) == null }, LocalOpenAICompatClient( + baseUrl = baseUrl, + model = settings.resolvedLocalModel, + apiKey = localKey?.takeIf { it.isNotBlank() }, + toolCallingEnabled = settings.localToolCalling, + structuredOutput = settings.localStructuredOutput, + maxOutputTokens = settings.localMaxTokens.takeIf { it > 0 }, + readTimeoutSeconds = settings.localTimeoutSeconds, + )) + } else -> { // USER_OPENAI_KEY / OFFLINE_STUB / BACKEND_PROXY all use the OpenAI // key + factory, mirroring the iOS resolver. @@ -67,6 +93,7 @@ object CoachClientResolver { geminiKey = store.geminiApiKey, openRouterKey = store.openRouterApiKey, minimaxKey = store.minimaxApiKey, + localKey = store.localApiKey, openAIClientFactory = openAIClientFactory, ) @@ -101,6 +128,9 @@ object CoachClientResolver { CoachProviderMode.USER_GEMINI_KEY -> settings.geminiModel CoachProviderMode.USER_OPENROUTER_KEY -> settings.resolvedOpenRouterModel CoachProviderMode.USER_MINIMAX_KEY -> settings.resolvedMinimaxModel + // Reported verbatim for attribution — including blank, which is a valid llama.cpp + // setup. `CoachPricingCatalog` prices this provider at $0 regardless of the string. + CoachProviderMode.LOCAL_OPENAI_COMPAT -> settings.resolvedLocalModel else -> openAIModel.ifEmpty { OpenAIModel.DEFAULT.slug } } } diff --git a/app/src/main/java/com/pulseloop/coach/config/CoachProviderSettings.kt b/app/src/main/java/com/pulseloop/coach/config/CoachProviderSettings.kt index 294cc96..d7445a5 100644 --- a/app/src/main/java/com/pulseloop/coach/config/CoachProviderSettings.kt +++ b/app/src/main/java/com/pulseloop/coach/config/CoachProviderSettings.kt @@ -13,6 +13,10 @@ enum class CoachProviderMode(val rawValue: String, val label: String) { USER_GEMINI_KEY("userGeminiKey", "Gemini (your key)"), USER_OPENROUTER_KEY("userOpenRouterKey", "OpenRouter (your key)"), USER_MINIMAX_KEY("userMiniMaxKey", "MiniMax (your key)"), + /** Any OpenAI-Chat-Completions-compatible server the user runs themselves — Ollama, + * llama.cpp, vLLM, SGLang, LM Studio. The API key is optional here; the readiness sentinel + * is the base URL. See `docs/local-llm-coach.md`. */ + LOCAL_OPENAI_COMPAT("localOpenAICompat", "Local / self-hosted"), BACKEND_PROXY("backendProxy", "Backend proxy"); companion object { @@ -48,6 +52,21 @@ data class CoachProviderSettings( val reasoningEffort: String? = null, /** When true, the coach composer shows an attach-image button. */ val imageInputEnabled: Boolean = false, + /** Local-only: base URL of the self-hosted server, as typed (`http://192.168.1.50:11434`). + * Blank means the provider isn't configured — this, not the key, gates readiness. */ + val localBaseUrl: String = "", + /** Local-only: model name the server expects. Free-form; `/v1/models` populates the picker. */ + val localModel: String = "", + /** Local-only: send `tools`. Off for a server started without tool-call support (vLLM without + * `--enable-auto-tool-choice` returns HTTP 400) or a model that can't call them. */ + val localToolCalling: Boolean = true, + /** Local-only: how hard to constrain the output shape. Default OFF — the only mode every + * backend supports. */ + val localStructuredOutput: LocalStructuredOutput = LocalStructuredOutput.OFF, + /** Local-only: `max_tokens`; 0 = omit and let the server decide. */ + val localMaxTokens: Int = 0, + /** Local-only: read timeout in seconds. Long, because CPU inference is slow. */ + val localTimeoutSeconds: Int = com.pulseloop.coach.local.LocalOpenAICompatClient.DEFAULT_READ_TIMEOUT_SECONDS, ) { /** The OpenRouter model slug to use; falls back to the default preset only * when the stored slug is blank. */ @@ -58,4 +77,33 @@ data class CoachProviderSettings( * the stored slug is blank. */ val resolvedMinimaxModel: String get() = minimaxModel.trim().ifEmpty { MiniMaxModel.DEFAULT.slug } + + /** The local base URL with surrounding whitespace gone; blank when unconfigured. There's no + * default to fall back to — every engine listens on a different port. */ + val resolvedLocalBaseUrl: String get() = localBaseUrl.trim() + + /** The local model name. Blank is sent as-is rather than substituted: llama.cpp ignores the + * field entirely, so an empty value is legitimate there, and inventing a slug would turn a + * working setup into a 404 on the servers that do read it. */ + val resolvedLocalModel: String get() = localModel.trim() +} + +/** + * How hard to constrain a local model's output shape — see `docs/local-llm-coach.md` §5. + * + * [OFF] is the default because it's the only mode implemented by every backend: the coach's shape + * is carried by `CoachResponseSchema.promptInstruction` in the system message, with the + * orchestrator's JSON-repair loop as the backstop. The other two are opt-in because the support + * matrix is genuinely uneven — LM Studio implements `json_schema` but not `json_object`, and some + * llama.cpp builds error when `json_schema` collides with a server-side `grammar`. + */ +enum class LocalStructuredOutput(val rawValue: String, val label: String, val blurb: String) { + OFF("off", "Prompt only", "Works everywhere (default)"), + JSON_OBJECT("jsonObject", "JSON mode", "response_format: json_object — not on LM Studio"), + JSON_SCHEMA("jsonSchema", "Strict schema", "response_format: json_schema — best when supported"); + + companion object { + fun fromRaw(raw: String?): LocalStructuredOutput = + entries.firstOrNull { it.rawValue == raw } ?: OFF + } } diff --git a/app/src/main/java/com/pulseloop/coach/config/CoachProviderSettingsStore.kt b/app/src/main/java/com/pulseloop/coach/config/CoachProviderSettingsStore.kt index 2154eda..3928791 100644 --- a/app/src/main/java/com/pulseloop/coach/config/CoachProviderSettingsStore.kt +++ b/app/src/main/java/com/pulseloop/coach/config/CoachProviderSettingsStore.kt @@ -83,6 +83,52 @@ class CoachProviderSettingsStore(context: Context) { else prefs.edit().putString(KEY_REASONING_EFFORT, value).apply() } + // ── Local / self-hosted provider (docs/local-llm-coach.md) ────────── + // The base URL lives in the same encrypted file as the keys: it's a private network address, + // and it's what gates readiness in place of a key. + + /** Base URL of the self-hosted OpenAI-compatible server, as the user typed it. */ + var localBaseUrl: String + get() = prefs.getString(KEY_LOCAL_BASE_URL, "") ?: "" + set(value) { prefs.edit().putString(KEY_LOCAL_BASE_URL, value).apply() } + + val hasLocalBaseUrl: Boolean get() = localBaseUrl.isNotBlank() + + /** OPTIONAL bearer token. Blank is the normal case — llama.cpp/vLLM/SGLang only require one + * when started with `--api-key`, and Ollama ignores the header entirely. */ + var localApiKey: String + get() = prefs.getString(KEY_LOCAL_API_KEY, "") ?: "" + set(value) { prefs.edit().putString(KEY_LOCAL_API_KEY, value).apply() } + + val hasLocalKey: Boolean get() = localApiKey.isNotBlank() + + /** Model name the server expects; free-form (the `/v1/models` picker just fills it in). */ + var localModel: String + get() = prefs.getString(KEY_LOCAL_MODEL, "") ?: "" + set(value) { prefs.edit().putString(KEY_LOCAL_MODEL, value).apply() } + + /** Send `tools` to the local server. Default on. */ + var localToolCalling: Boolean + get() = prefs.getBoolean(KEY_LOCAL_TOOL_CALLING, true) + set(value) { prefs.edit().putBoolean(KEY_LOCAL_TOOL_CALLING, value).apply() } + + /** How hard to constrain the output shape; default [LocalStructuredOutput.OFF]. */ + var localStructuredOutput: LocalStructuredOutput + get() = LocalStructuredOutput.fromRaw(prefs.getString(KEY_LOCAL_STRUCTURED_OUTPUT, null)) + set(value) { prefs.edit().putString(KEY_LOCAL_STRUCTURED_OUTPUT, value.rawValue).apply() } + + /** `max_tokens` for local requests; 0 = omit the field. */ + var localMaxTokens: Int + get() = prefs.getInt(KEY_LOCAL_MAX_TOKENS, 0) + set(value) { prefs.edit().putInt(KEY_LOCAL_MAX_TOKENS, maxOf(0, value)).apply() } + + /** Read timeout for local requests, in seconds. Clamped to a sane floor so a stray `1` can't + * make every turn time out with no way back except reinstalling. */ + var localTimeoutSeconds: Int + get() = prefs.getInt(KEY_LOCAL_TIMEOUT_SECONDS, + com.pulseloop.coach.local.LocalOpenAICompatClient.DEFAULT_READ_TIMEOUT_SECONDS) + set(value) { prefs.edit().putInt(KEY_LOCAL_TIMEOUT_SECONDS, value.coerceIn(10, 1800)).apply() } + /** Master toggle for the coach composer's attach-image button. */ var imageInputEnabled: Boolean get() = prefs.getBoolean(KEY_IMAGE_INPUT_ENABLED, false) @@ -98,6 +144,12 @@ class CoachProviderSettingsStore(context: Context) { orProviderSort = orProviderSort, reasoningEffort = reasoningEffort, imageInputEnabled = imageInputEnabled, + localBaseUrl = localBaseUrl, + localModel = localModel, + localToolCalling = localToolCalling, + localStructuredOutput = localStructuredOutput, + localMaxTokens = localMaxTokens, + localTimeoutSeconds = localTimeoutSeconds, ) companion object { @@ -112,5 +164,12 @@ class CoachProviderSettingsStore(context: Context) { private const val KEY_OR_PROVIDER_SORT = "openrouter_provider_sort" private const val KEY_REASONING_EFFORT = "coach_reasoning_effort" private const val KEY_IMAGE_INPUT_ENABLED = "coach_image_input_enabled" + private const val KEY_LOCAL_BASE_URL = "local_llm_base_url" + private const val KEY_LOCAL_API_KEY = "local_llm_api_key" + private const val KEY_LOCAL_MODEL = "local_llm_model" + private const val KEY_LOCAL_TOOL_CALLING = "local_llm_tool_calling" + private const val KEY_LOCAL_STRUCTURED_OUTPUT = "local_llm_structured_output" + private const val KEY_LOCAL_MAX_TOKENS = "local_llm_max_tokens" + private const val KEY_LOCAL_TIMEOUT_SECONDS = "local_llm_timeout_seconds" } } diff --git a/app/src/main/java/com/pulseloop/coach/local/LocalCapabilityProbe.kt b/app/src/main/java/com/pulseloop/coach/local/LocalCapabilityProbe.kt new file mode 100644 index 0000000..7e50883 --- /dev/null +++ b/app/src/main/java/com/pulseloop/coach/local/LocalCapabilityProbe.kt @@ -0,0 +1,494 @@ +package com.pulseloop.coach.local + +import com.pulseloop.coach.config.LocalStructuredOutput +import com.pulseloop.coach.openai.ResponsesError +import com.pulseloop.coach.openai.ResponsesHttp +import kotlinx.serialization.json.* + +/** + * Self-configuration for the local provider: given only a base URL, work out which engine is + * behind it, what model it serves, and which optional request fields it will actually accept — + * then hand back settings the user doesn't have to reason about. + * + * **Why capabilities have to be probed rather than looked up.** `/v1/models` describes the model, + * not the server's request surface, and the two things most likely to fail a coach turn are + * decided at *launch time* by flags that endpoint never mentions: vLLM rejects `tools` unless it + * was started with `--enable-auto-tool-choice --tool-call-parser`, and structured-output support + * varies by backend and build (LM Studio implements `json_schema` but not `json_object`; some + * llama.cpp builds error when `json_schema` meets a server-side `grammar`). The only honest test + * is to send the field and see whether the server takes it — so [run] sends two deliberately tiny + * chat requests, one carrying a throwaway tool and one carrying a throwaway `response_format`. + * + * Cost: three generations of at most a few tokens — a plain baseline request first, then the two + * carrying the fields under test. The baseline is what makes a 4xx readable as "this field is + * refused" rather than "this request is refused". On a server that has to page the model in first + * (Ollama, LM Studio) the first of them can take tens of seconds — hence [PROBE_TIMEOUT_SECONDS]. + * + * Failure is never destructive. A probe that errors for an unrelated reason (network blip, model + * still loading) leaves that capability at its safe default rather than switching it off, and the + * report says the probe was inconclusive so the UI can say so too. + */ +object LocalCapabilityProbe { + + /** Which server is behind the URL. Cosmetic — it drives the summary line and the hints, not + * the request body; every real decision comes from [Report]'s probed capabilities. */ + enum class Engine(val label: String) { + OLLAMA("Ollama"), + LLAMA_CPP("llama.cpp"), + VLLM("vLLM"), + SGLANG("SGLang"), + LM_STUDIO("LM Studio"), + UNKNOWN("OpenAI-compatible server"), + } + + /** A probed capability. [UNKNOWN] means the probe itself failed, so don't change the setting. */ + enum class Support { YES, NO, UNKNOWN } + + data class Report( + val engine: Engine, + /** Engine version when it advertises one. */ + val version: String? = null, + val models: List = emptyList(), + /** The model to use: the sole served model, else the previously-chosen one if the server + * still lists it, else blank for the user to pick. */ + val suggestedModel: String = "", + val toolCalling: Support = Support.UNKNOWN, + val jsonSchema: Support = Support.UNKNOWN, + val jsonObject: Support = Support.UNKNOWN, + /** The server's **context window** for the chosen model (prompt + completion), when it + * reports one. This is NOT an output budget — see [suggestedMaxTokens]. */ + val contextWindow: Int? = null, + /** Per-probe detail, shown under the summary so an inconclusive result is explainable. */ + val notes: List = emptyList(), + ) { + /** The structured-output mode to store: the strongest one the server accepted. Falls back + * to OFF, which needs nothing from the server. */ + val suggestedStructuredOutput: LocalStructuredOutput = when { + jsonSchema == Support.YES -> LocalStructuredOutput.JSON_SCHEMA + jsonObject == Support.YES -> LocalStructuredOutput.JSON_OBJECT + else -> LocalStructuredOutput.OFF + } + + /** Tool calling stays ON unless the server actively refused it — an inconclusive probe + * must not silently strip the coach of its ability to read the user's data. */ + val suggestedToolCalling: Boolean get() = toolCalling != Support.NO + + /** + * Whether the probes actually reached a verdict, so a suggestion may **overwrite a setting + * the user chose by hand**. + * + * [suggestedToolCalling] and [suggestedStructuredOutput] both have a safe default for the + * inconclusive case, which is right for a first-time setup and wrong for a re-detect: a + * user who turned tools off for a vLLM server without `--enable-auto-tool-choice`, then + * pressed Detect to refresh the model list, would have them switched back on and every + * turn would 400. Probes are skipped entirely when the model comes back blank + * ([pickModel] on a multi-model server) or when the baseline request fails — neither says + * anything about capabilities. + */ + val toolCallingConclusive: Boolean get() = toolCalling != Support.UNKNOWN + + /** As [toolCallingConclusive]. One conclusive probe is enough: a `YES` on the strict + * schema deliberately leaves the weaker JSON mode untested. */ + val structuredOutputConclusive: Boolean + get() = jsonSchema != Support.UNKNOWN || jsonObject != Support.UNKNOWN + + /** + * The value to store in **Max tokens**, derived from [contextWindow]; 0 means "leave + * blank and let the server decide". + * + * A context window is not an output budget, and copying it across would be actively + * harmful: `max_tokens` is checked against what's *left* after the prompt, so a request + * with `prompt + max_tokens > context` is rejected outright. We therefore reserve + * [PROMPT_RESERVE_TOKENS] for the coach's own prompt — measured at ~3.3k for a plain turn, + * doubled to cover tool results and replayed history — and cap the remainder at + * [MAX_SUGGESTED_TOKENS], well past what a coach_response needs, so a huge context doesn't + * turn into a runaway generation budget. + * + * When the headroom is too small to be worth setting, this returns 0 and + * [contextTooSmall] carries the warning instead: a server whose context can't even hold + * the prompt (Ollama ships a 2048-token `num_ctx` default, smaller than our prompt) will + * silently truncate, and a wrong `max_tokens` would only mask that. + */ + val suggestedMaxTokens: Int get() { + val ctx = contextWindow ?: return 0 + val headroom = ctx - PROMPT_RESERVE_TOKENS + if (headroom < MIN_USEFUL_OUTPUT_TOKENS) return 0 + return minOf(headroom, MAX_SUGGESTED_TOKENS) + } + + /** True when the reported context can't comfortably hold the coach's prompt, so the user + * needs to raise it on the server (Ollama `num_ctx`, llama.cpp `-c`, vLLM + * `--max-model-len`) rather than tune anything in the app. */ + val contextTooSmall: Boolean + get() = contextWindow != null && + contextWindow - PROMPT_RESERVE_TOKENS < MIN_USEFUL_OUTPUT_TOKENS + + /** One line for the Settings summary. */ + val summary: String get() = buildString { + append(engine.label) + version?.let { append(" $it") } + append(" · ") + append(if (suggestedModel.isNotBlank()) suggestedModel else "${models.size} model(s)") + append(" · tools ") + append(when (toolCalling) { + Support.YES -> "yes"; Support.NO -> "no"; Support.UNKNOWN -> "unknown" + }) + append(" · ") + append(when (suggestedStructuredOutput) { + LocalStructuredOutput.JSON_SCHEMA -> "strict schema" + LocalStructuredOutput.JSON_OBJECT -> "JSON mode" + LocalStructuredOutput.OFF -> "prompt-only" + }) + contextWindow?.let { append(" · ${formatTokens(it)} ctx") } + } + } + + /** Raised when the server can't be reached or isn't OpenAI-compatible — [run]'s only hard + * failure. Everything after model discovery degrades to [Support.UNKNOWN] instead. */ + class Unreachable(val reason: String) : Exception(reason) + + /** + * Discovers everything about [baseUrl] in one pass. [currentModel] is preserved when the + * server still lists it, so re-probing doesn't silently move a working setup to another model. + * + * @throws Unreachable when `/v1/models` fails — nothing else can be trusted after that. + */ + suspend fun run( + baseUrl: String, + apiKey: String? = null, + currentModel: String = "", + ): Report { + LocalEndpoint.validate(baseUrl)?.let { throw Unreachable(LocalEndpoint.message(it)) } + val headers = mutableMapOf() + apiKey?.takeIf { it.isNotBlank() }?.let { headers["Authorization"] = "Bearer $it" } + + // 1. Models — also the reachability check, so its failure is the one hard failure. + val entries = when (val r = LocalModelCatalog.fetch(baseUrl, apiKey)) { + is LocalModelCatalog.Result.Success -> r.entries + is LocalModelCatalog.Result.Failure -> throw Unreachable(r.message) + } + val models = entries.map { it.id } + val model = pickModel(models, currentModel) + + // 2. Engine identity — best-effort, from the engine-specific info routes. Never fatal. + val (engine, version) = identify(baseUrl, headers) + + // Context window: from the listing when the engine puts it there (vLLM, llama.cpp, + // LM Studio), else from that engine's own route. + val context = entries.firstOrNull { it.id == model }?.contextWindow + ?: contextFromEngine(baseUrl, headers, engine, model) + + val notes = mutableListOf() + if (context != null && context - PROMPT_RESERVE_TOKENS < MIN_USEFUL_OUTPUT_TOKENS) { + notes.add( + "Context is only ${formatTokens(context)} — the coach's prompt alone is around " + + "${formatTokens(PROMPT_RESERVE_TOKENS / 2)}. Raise it on the server " + + "(Ollama `num_ctx`, llama.cpp `-c`, vLLM `--max-model-len`) or replies will be " + + "truncated." + ) + } + if (model.isBlank()) { + // Capability probes need a model name on every engine except llama.cpp, and without + // one a 400 would be indistinguishable from "capability unsupported". + notes.add("Pick a model, then run this again to detect tools and response format.") + return Report(engine, version, models, model, contextWindow = context, notes = notes) + } + + // 3. Baseline. A plain chat request with no optional fields at all, so the 4xx-means-no + // reading below is about the probed field rather than about the request as a whole. + // Skipping this was how an unloadable model id or an auth-gated chat route turned into + // "tools: not supported" and a persisted `toolCalling = false`. + when (val baseline = send(baseUrl, headers, model, BASELINE_PROBE)) { + is Outcome.Accepted -> Unit + is Outcome.Refused -> { + notes.add( + "The server refused a plain chat request for `$model` (HTTP " + + "${baseline.status}) — ${shorten(baseline.body)}. Tools and response format " + + "couldn't be tested, so both are left unchanged. Check the model can actually " + + "load and that the chat route accepts the same key as /v1/models." + ) + return Report(engine, version, models, model, contextWindow = context, notes = notes) + } + is Outcome.Inconclusive -> { + notes.add( + "Couldn't complete a plain chat request (${baseline.reason}) — tools and " + + "response format are left unchanged." + ) + return Report(engine, version, models, model, contextWindow = context, notes = notes) + } + } + + // 4. Capability probes. + val tools = probe(baseUrl, headers, model, TOOL_PROBE, "Tool calling", notes) + val schema = probe(baseUrl, headers, model, SCHEMA_PROBE, "Strict schema", notes) + // Only worth asking about the weaker mode when the stronger one was refused. + val obj = if (schema == Support.YES) Support.UNKNOWN + else probe(baseUrl, headers, model, JSON_OBJECT_PROBE, "JSON mode", notes) + + return Report(engine, version, models, model, tools, schema, obj, context, notes) + } + + /** Sole model → use it. Otherwise keep the user's current pick when the server still has it; + * else blank, because guessing among several would silently switch a working setup. */ + internal fun pickModel(models: List, currentModel: String): String = when { + currentModel.isNotBlank() && currentModel in models -> currentModel + models.size == 1 -> models.first() + else -> "" + } + + // ── Engine identity ────────────────────────────────────────────────── + + /** + * Asks each engine's own info route in turn and stops at the first that answers. These are + * distinct paths rather than a single field because `owned_by` in `/v1/models` is unreliable + * (vLLM says "vllm", but Ollama says "library" and LM Studio says "organization_owner", and a + * proxy rewrites all of them). Every call is best-effort — an engine we can't name still works. + */ + private suspend fun identify(baseUrl: String, headers: Map): Pair { + val base = LocalEndpoint.normalize(baseUrl) ?: return Engine.UNKNOWN to null + // vLLM: GET /version -> {"version":"0.27.1"} + get("$base/version", headers)?.let { body -> + versionField(body)?.let { return Engine.VLLM to it } + } + // Ollama: GET /api/version -> {"version":"0.x.y"} + get("$base/api/version", headers)?.let { body -> + versionField(body)?.let { return Engine.OLLAMA to it } + } + // llama.cpp: GET /props -> build_info / default_generation_settings + get("$base/props", headers)?.let { body -> + val root = jsonObjectOrNull(body) + if (root != null && (root.containsKey("build_info") || root.containsKey("default_generation_settings"))) { + return Engine.LLAMA_CPP to (root["build_info"] as? JsonPrimitive)?.contentOrNull + } + } + // SGLang: GET /get_server_info -> model_path / version + get("$base/get_server_info", headers)?.let { body -> + val root = jsonObjectOrNull(body) + if (root != null && (root.containsKey("model_path") || root.containsKey("version"))) { + return Engine.SGLANG to (root["version"] as? JsonPrimitive)?.contentOrNull + } + } + // LM Studio: its richer native listing, absent everywhere else. + get("$base/api/v0/models", headers)?.let { return Engine.LM_STUDIO to null } + return Engine.UNKNOWN to null + } + + /** + * The context window from an engine's own route, for the two that don't put it in + * `/v1/models`. Best-effort: a null here just means the app won't suggest a budget. + * + * Ollama is the one that matters. Its `/v1/models` carries no context at all, and its default + * `num_ctx` is **2048** — smaller than the coach's own prompt — so without this a user would + * get silently truncated context and blame the model. + */ + private suspend fun contextFromEngine( + baseUrl: String, + headers: Map, + engine: Engine, + model: String, + ): Int? { + val base = LocalEndpoint.normalize(baseUrl) ?: return null + return when (engine) { + Engine.OLLAMA -> { + if (model.isBlank()) return null + val body = JsonObject(mapOf("model" to JsonPrimitive(model))) + val text = try { + ResponsesHttp.post( + "$base/api/show", + Json.encodeToString(JsonObject.serializer(), body).toByteArray(), + headers, + IDENTIFY_TIMEOUT_SECONDS, + followRedirects = false, + ) + } catch (_: Exception) { return null } + // `model_info` is keyed by architecture, e.g. "qwen3.context_length", so match on + // the suffix rather than guessing the family. + val info = jsonObjectOrNull(text)?.get("model_info") as? JsonObject ?: return null + info.entries.firstOrNull { it.key.endsWith(".context_length") } + ?.value?.let { (it as? JsonPrimitive)?.intOrNull }?.takeIf { it > 0 } + } + Engine.SGLANG -> { + val text = get("$base/get_model_info", headers) ?: return null + val root = jsonObjectOrNull(text) ?: return null + LocalModelCatalog.contextWindowOf(root) + } + Engine.LLAMA_CPP -> { + val text = get("$base/props", headers) ?: return null + val root = jsonObjectOrNull(text) ?: return null + LocalModelCatalog.contextWindowOf(root) + ?: (root["default_generation_settings"] as? JsonObject) + ?.let { LocalModelCatalog.contextWindowOf(it) } + } + else -> null + } + } + + /** "262,144" → "262k"; small values stay exact so a 2048 warning reads literally. */ + internal fun formatTokens(tokens: Int): String = + if (tokens >= 10_000) "${tokens / 1000}k" else tokens.toString() + + private suspend fun get(url: String, headers: Map): String? = try { + ResponsesHttp.get(url, headers, IDENTIFY_TIMEOUT_SECONDS, followRedirects = false) + } catch (_: Exception) { + null // A 404 here just means "not this engine". + } + + private fun jsonObjectOrNull(body: String): JsonObject? = try { + Json { ignoreUnknownKeys = true }.parseToJsonElement(body) as? JsonObject + } catch (_: Exception) { null } + + private fun versionField(body: String): String? = + (jsonObjectOrNull(body)?.get("version") as? JsonPrimitive)?.contentOrNull + + // ── Capability probes ──────────────────────────────────────────────── + + /** What one probe request actually got back, before it is read as a capability verdict. */ + private sealed interface Outcome { + object Accepted : Outcome + /** The server answered 4xx — it read the request and refused it. */ + data class Refused(val status: Int, val body: String) : Outcome + /** 5xx, transport failure, or an unusable URL: says nothing either way. */ + data class Inconclusive(val reason: String) : Outcome + } + + /** Sends a one-token chat request carrying [extra] and reports what came back. */ + private suspend fun send( + baseUrl: String, + headers: Map, + model: String, + extra: Map, + ): Outcome { + val url = LocalEndpoint.chatCompletionsUrl(baseUrl) + ?: return Outcome.Inconclusive("the server address couldn't be parsed") + val body = JsonObject(buildMap { + put("model", JsonPrimitive(model)) + put("messages", JsonArray(listOf(JsonObject(mapOf( + "role" to JsonPrimitive("user"), + "content" to JsonPrimitive("hi"), + ))))) + put("max_tokens", JsonPrimitive(PROBE_MAX_TOKENS)) + putAll(extra) + }) + return try { + ResponsesHttp.post( + url, + Json.encodeToString(JsonObject.serializer(), body).toByteArray(), + headers, + PROBE_TIMEOUT_SECONDS, + followRedirects = false, + ) + Outcome.Accepted + } catch (e: ResponsesError.Http) { + if (e.status in 400..499) Outcome.Refused(e.status, e.body) + else Outcome.Inconclusive("HTTP ${e.status}") + } catch (e: Exception) { + Outcome.Inconclusive(e.message ?: "no response") + } + } + + /** + * Sends [extra] alongside a one-token chat request and classifies the answer. + * + * A 4xx is the server telling us it won't take the field — that's [Support.NO], and the exact + * status doesn't matter (vLLM answers 400 for a disabled tool parser and 422 for a field its + * deserializer doesn't know). A 5xx or a transport failure says nothing about the capability, + * so it stays [Support.UNKNOWN] and the caller keeps its default. + * + * Reading a 4xx as "this field is unsupported" is only sound because [run] has already + * established with [BASELINE_PROBE] that a request carrying *no* optional fields succeeds. + * Without that, every whole-request rejection — a model id `/v1/models` lists but can't load + * (LM Studio with JIT off, a model pulled between the two calls), a chat route that wants auth + * when the listing didn't, a broken chat template — would come back as "tools: no" and + * silently persist `toolCalling = false`, which costs the coach all access to the user's data. + */ + private suspend fun probe( + baseUrl: String, + headers: Map, + model: String, + extra: Map, + label: String, + notes: MutableList, + ): Support = when (val outcome = send(baseUrl, headers, model, extra)) { + is Outcome.Accepted -> Support.YES + is Outcome.Refused -> { + notes.add("$label: not supported (HTTP ${outcome.status}) — ${shorten(outcome.body)}") + Support.NO + } + is Outcome.Inconclusive -> { + notes.add("$label: couldn't tell (${outcome.reason}) — left unchanged.") + Support.UNKNOWN + } + } + + /** Server error bodies are verbose; the first line is the part worth showing. */ + private fun shorten(body: String): String { + val message = try { + ((jsonObjectOrNull(body)?.get("error") as? JsonObject)?.get("message") as? JsonPrimitive) + ?.contentOrNull + } catch (_: Exception) { null } ?: body + return message.trim().lineSequence().firstOrNull().orEmpty().take(160) + } + + /** Nothing optional at all — the control the capability probes are measured against. */ + private val BASELINE_PROBE: Map = emptyMap() + + /** A throwaway tool. Named so it can't collide with a real coach tool in a server-side log. */ + private val TOOL_PROBE: Map = mapOf( + "tools" to JsonArray(listOf(JsonObject(mapOf( + "type" to JsonPrimitive("function"), + "function" to JsonObject(mapOf( + "name" to JsonPrimitive("pulseloop_probe"), + "description" to JsonPrimitive("Capability probe. Do not call."), + "parameters" to JsonObject(mapOf( + "type" to JsonPrimitive("object"), + "properties" to JsonObject(emptyMap()), + )), + )), + )))), + ) + + /** A minimal schema, not the coach's: we're testing whether the *field* is accepted, and a + * large schema risks a rejection about the schema itself rather than the capability. */ + private val SCHEMA_PROBE: Map = mapOf( + "response_format" to JsonObject(mapOf( + "type" to JsonPrimitive("json_schema"), + "json_schema" to JsonObject(mapOf( + "name" to JsonPrimitive("pulseloop_probe"), + "strict" to JsonPrimitive(true), + "schema" to JsonObject(mapOf( + "type" to JsonPrimitive("object"), + "properties" to JsonObject(mapOf( + "ok" to JsonObject(mapOf("type" to JsonPrimitive("string"))), + )), + "required" to JsonArray(listOf(JsonPrimitive("ok"))), + "additionalProperties" to JsonPrimitive(false), + )), + )), + )), + ) + + private val JSON_OBJECT_PROBE: Map = mapOf( + "response_format" to JsonObject(mapOf("type" to JsonPrimitive("json_object"))), + ) + + /** Long enough for a cold model to page in on Ollama/LM Studio. */ + private const val PROBE_TIMEOUT_SECONDS = 120 + /** Short: these routes either exist or 404 immediately. */ + private const val IDENTIFY_TIMEOUT_SECONDS = 10 + /** Enough that a grammar-constrained probe emits something, small enough to stay cheap. */ + private const val PROBE_MAX_TOKENS = 8 + + /** + * Context reserved for input before any of it is offered as output budget. A plain coach turn + * measured 3.1–3.3k input tokens on a real device; this doubles that so a turn that replays + * history and feeds back tool results still fits. + */ + internal const val PROMPT_RESERVE_TOKENS = 6144 + + /** Below this much headroom, suggesting a budget is worse than saying the context is too small. */ + internal const val MIN_USEFUL_OUTPUT_TOKENS = 512 + + /** A coach_response plus reasoning needs far less than this; the cap stops a 262k context from + * becoming a licence for a runaway generation. */ + internal const val MAX_SUGGESTED_TOKENS = 32_768 +} diff --git a/app/src/main/java/com/pulseloop/coach/local/LocalEndpoint.kt b/app/src/main/java/com/pulseloop/coach/local/LocalEndpoint.kt new file mode 100644 index 0000000..76f97aa --- /dev/null +++ b/app/src/main/java/com/pulseloop/coach/local/LocalEndpoint.kt @@ -0,0 +1,139 @@ +package com.pulseloop.coach.local + +import java.net.URI + +/** + * URL handling for the self-hosted ("local") coach provider — see + * `docs/local-llm-coach.md` §3. + * + * The user types a *base* URL (`http://192.168.1.50:11434`, `http://localhost:1234/v1`, + * `https://llm.example.com`), not a full endpoint path, because every engine serves the same + * OpenAI-compatible routes under a `/v1` prefix: Ollama on 11434, llama.cpp on 8080, vLLM on + * 8000, SGLang on 30000, LM Studio on 1234. This object turns whatever they typed into the two + * concrete URLs the app calls, and enforces the plaintext-host rule that Android's Network + * Security Config can't express. + */ +object LocalEndpoint { + + /** Why a base URL can't be used. `null` from [validate] means it's fine. */ + enum class Problem { BLANK, MALFORMED, UNSUPPORTED_SCHEME, PUBLIC_CLEARTEXT } + + /** + * Normalizes a user-typed base URL to its scheme+authority+path root, with any trailing `/` + * and any trailing `/v1` (or `/v1/chat/completions`, if they pasted the full endpoint) + * stripped — so [chatCompletionsUrl] and [modelsUrl] can append the canonical suffix without + * producing `/v1/v1`. A bare `host:port` with no scheme is assumed to be `http://` (the + * overwhelmingly common local case; a public host would be rejected by [validate] anyway). + * + * Returns null when the input can't be parsed into a scheme + host. + */ + fun normalize(raw: String): String? { + var text = raw.trim() + if (text.isEmpty()) return null + if (!text.contains("://")) text = "http://$text" + val uri = try { URI(text) } catch (_: Exception) { return null } + val scheme = uri.scheme?.lowercase() ?: return null + val host = uri.host ?: return null + if (host.isEmpty()) return null + + var path = (uri.path ?: "").trimEnd('/') + // Tolerate a pasted full endpoint or an explicit /v1 — both are re-appended by callers. + for (suffix in listOf("/v1/chat/completions", "/chat/completions", "/v1")) { + if (path.endsWith(suffix)) { path = path.dropLast(suffix.length); break } + } + path = path.trimEnd('/') + + val port = if (uri.port >= 0) ":${uri.port}" else "" + return "$scheme://$host$port$path" + } + + /** `POST` target for a chat turn. */ + fun chatCompletionsUrl(base: String): String? = normalize(base)?.let { "$it/v1/chat/completions" } + + /** `GET` target that lists the models the server currently has loaded/available. */ + fun modelsUrl(base: String): String? = normalize(base)?.let { "$it/v1/models" } + + /** + * The reason [raw] can't be used, or null if it's usable. + * + * `https://` is unrestricted — a self-hosted box with a real certificate is the user's call. + * Plaintext `http://` is confined to hosts that can't be on the public internet: loopback, + * RFC1918 / CGNAT / link-local addresses, and mDNS `*.local` names. The app permits cleartext + * app-wide in `network_security_config.xml` (Network Security Config has no CIDR syntax), so + * this is the check that actually keeps an API key and a stream of health data off the open + * internet in the clear. + */ + fun validate(raw: String): Problem? { + if (raw.isBlank()) return Problem.BLANK + val normalized = normalize(raw) ?: return Problem.MALFORMED + val uri = try { URI(normalized) } catch (_: Exception) { return Problem.MALFORMED } + val scheme = uri.scheme?.lowercase() ?: return Problem.MALFORMED + val host = uri.host?.lowercase() ?: return Problem.MALFORMED + return when (scheme) { + "https" -> null + "http" -> if (isPrivateHost(host)) null else Problem.PUBLIC_CLEARTEXT + else -> Problem.UNSUPPORTED_SCHEME + } + } + + /** A short, user-facing explanation for a [Problem], for the Settings field. */ + fun message(problem: Problem): String = when (problem) { + Problem.BLANK -> "Enter your server's address, e.g. http://192.168.1.50:11434" + Problem.MALFORMED -> "That doesn't look like a URL — use host:port or http://host:port" + Problem.UNSUPPORTED_SCHEME -> "Only http:// and https:// are supported." + Problem.PUBLIC_CLEARTEXT -> + "Plain http:// is only allowed for a server on this device or your local network " + + "(an IP address, a plain hostname, or a .local / .lan / .ts.net name). " + + "Use https:// to reach one over the internet." + } + + /** + * True when [host] can't be routed off the local network: loopback (incl. the `10.0.2.2` alias + * an emulator uses for the dev machine, which is RFC1918 anyway), RFC1918 (`10/8`, + * `172.16/12`, `192.168/16`), CGNAT `100.64/10` (Tailscale), link-local `169.254/16`, IPv6 + * loopback/ULA/link-local, mDNS `.local` names, and the name forms that only resolve on a + * local network. + * + * That last group is why this isn't purely an address test. Addressing an inference box by the + * name its router or mDNS hands out — `http://nas:11434`, `http://ollama.lan:8080`, a + * Tailscale MagicDNS `http://box.tail1234.ts.net:11434` — is an ordinary setup, and rejecting + * it told the user their server had to be on their local network, which is exactly where it + * was. A single-label host has no public TLD and cannot be resolved off-LAN; the suffixes in + * [LOCAL_SUFFIXES] are the reserved/local-scope ones (RFC 8375 `.home.arpa`, RFC 6762 + * `.local`, the `.lan`/`.home`/`.internal` conventions, and Tailscale's `.ts.net`). + */ + internal fun isPrivateHost(host: String): Boolean { + val h = host.trim('[', ']') + if (h == "localhost" || h.endsWith(".localhost")) return true + if (LOCAL_SUFFIXES.any { h.endsWith(it) }) return true + // A bare hostname with no dot at all: resolvable only via DNS search domain, mDNS or + // NetBIOS, i.e. on-link. `h.contains(':')` below still catches a bracket-less IPv6 form. + if (!h.contains('.') && !h.contains(':') && h.isNotEmpty()) return true + if (h.contains(':')) { // IPv6 + val v6 = h.lowercase() + return v6 == "::1" || v6.startsWith("fc") || v6.startsWith("fd") || v6.startsWith("fe80:") + } + val octets = h.split('.') + if (octets.size != 4) return false + val nums = octets.map { it.toIntOrNull() ?: return false } + if (nums.any { it !in 0..255 }) return false + val (a, b) = nums[0] to nums[1] + return when { + a == 127 -> true // loopback + a == 10 -> true // RFC1918 + a == 192 && b == 168 -> true // RFC1918 + a == 172 && b in 16..31 -> true // RFC1918 + a == 169 && b == 254 -> true // link-local + a == 100 && b in 64..127 -> true // CGNAT / Tailscale + else -> false + } + } + + /** Suffixes that are reserved for, or conventionally used on, a local network only. */ + private val LOCAL_SUFFIXES = listOf( + ".local", // RFC 6762 mDNS + ".home.arpa", // RFC 8375 + ".lan", ".home", ".internal", // common router defaults + ".ts.net", // Tailscale MagicDNS + ) +} diff --git a/app/src/main/java/com/pulseloop/coach/local/LocalModelCatalog.kt b/app/src/main/java/com/pulseloop/coach/local/LocalModelCatalog.kt new file mode 100644 index 0000000..8b31e1d --- /dev/null +++ b/app/src/main/java/com/pulseloop/coach/local/LocalModelCatalog.kt @@ -0,0 +1,118 @@ +package com.pulseloop.coach.local + +import com.pulseloop.coach.openai.ResponsesError +import com.pulseloop.coach.openai.ResponsesHttp +import kotlinx.serialization.json.* + +/** + * `GET {base}/v1/models` against a self-hosted server, so Settings can offer a real model picker + * instead of making the user type `qwen3:8b` from memory. + * + * Every engine in scope serves this route (it's how the OpenAI SDK enumerates models), and every + * one returns the same envelope: `{"object":"list","data":[{"id":"…"},…]}`. Ollama lists pulled + * models, LM Studio lists loaded ones, vLLM/SGLang list the single served model, and llama.cpp + * lists the loaded model under its `--alias`. The list is advisory — the stored model stays a free + * string, because a router in front of any of these can serve names the endpoint doesn't + * enumerate. + */ +object LocalModelCatalog { + + /** + * One entry from the listing. [contextWindow] is the model's **context window** (prompt + + * completion), when the server volunteers it — NOT an output budget; see + * [LocalCapabilityProbe] for the derivation. Null when the engine doesn't report it here. + */ + data class ModelInfo(val id: String, val contextWindow: Int? = null) + + /** The outcome of a refresh, kept as data so Settings can show the failure inline. */ + sealed class Result { + data class Success(val entries: List) : Result() { + val models: List get() = entries.map { it.id } + } + /** [message] is already user-facing. */ + data class Failure(val message: String) : Result() + } + + suspend fun fetch( + baseUrl: String, + apiKey: String? = null, + timeoutSeconds: Int = REFRESH_TIMEOUT_SECONDS, + ): Result { + LocalEndpoint.validate(baseUrl)?.let { return Result.Failure(LocalEndpoint.message(it)) } + val url = LocalEndpoint.modelsUrl(baseUrl) + ?: return Result.Failure(LocalEndpoint.message(LocalEndpoint.Problem.MALFORMED)) + + val headers = mutableMapOf() + apiKey?.takeIf { it.isNotBlank() }?.let { headers["Authorization"] = "Bearer $it" } + + return try { + // followRedirects = false: `validate` vets the typed URL, not where a redirect lands. + Result.Success(parseEntries(ResponsesHttp.get(url, headers, timeoutSeconds, followRedirects = false))) + } catch (e: ResponsesError.Http) { + Result.Failure("The server answered HTTP ${e.status} for /v1/models.") + } catch (e: ResponsesError.Transport) { + // Some exceptions carry no message at all (NetworkOnMainThreadException is the one + // that bit us), and "no connection" then sends the user hunting a network fault that + // isn't there. Fall back to the class name, which at least names the real failure. + val why = e.underlying.message?.takeIf { it.isNotBlank() } + ?: e.underlying::class.java.simpleName + Result.Failure("Couldn't reach $url — $why.") + } catch (e: Exception) { + Result.Failure(e.message ?: "Couldn't read the model list.") + } + } + + /** + * Pulls the `id`s out of the OpenAI list envelope, sorted and de-duplicated. Falls back to a + * bare top-level array, which a couple of thin proxies return instead of the envelope. + */ + internal fun parse(body: String): List = parseEntries(body).map { it.id } + + /** + * As [parse], but keeps each entry's context window when the server reports one alongside the + * id. The field name differs per engine and none of them is the OpenAI spec — vLLM writes + * `max_model_len`, llama.cpp `n_ctx` (with `n_ctx_train` as the model's trained maximum), and + * LM Studio `loaded_context_length` / `max_context_length` in its own richer listing. We take + * the first present, preferring what's actually *loaded* over what the model could support, + * because the loaded value is the one a request is measured against. + */ + internal fun parseEntries(body: String): List { + val root = Json { ignoreUnknownKeys = true }.parseToJsonElement(body) + val data = when (root) { + is JsonObject -> root["data"] as? JsonArray + is JsonArray -> root + else -> null + } ?: throw ResponsesError.Decoding("No `data` array in the /v1/models response.") + return data.mapNotNull { entry -> + when (entry) { + is JsonObject -> (entry["id"] as? JsonPrimitive)?.contentOrNull + ?.takeIf { it.isNotBlank() } + ?.let { ModelInfo(it, contextWindowOf(entry)) } + is JsonPrimitive -> entry.contentOrNull?.takeIf { it.isNotBlank() }?.let { ModelInfo(it) } + else -> null + } + }.distinctBy { it.id }.sortedBy { it.id } + } + + /** The context window an entry advertises, under whichever name its engine uses. */ + internal fun contextWindowOf(entry: JsonObject): Int? { + for (key in CONTEXT_KEYS) { + val v = (entry[key] as? JsonPrimitive)?.intOrNull + if (v != null && v > 0) return v + } + return null + } + + /** Ordered by preference: loaded-context first, then configured, then trained maximum. */ + private val CONTEXT_KEYS = listOf( + "loaded_context_length", // LM Studio (actually loaded) + "max_model_len", // vLLM + "n_ctx", // llama.cpp (as served) + "max_context_length", // LM Studio (model ceiling) + "context_length", // generic / proxies + "n_ctx_train", // llama.cpp (model ceiling) + ) + + /** Short: this is a list lookup behind a button, not a generation. */ + private const val REFRESH_TIMEOUT_SECONDS = 15 +} diff --git a/app/src/main/java/com/pulseloop/coach/local/LocalOpenAICompatClient.kt b/app/src/main/java/com/pulseloop/coach/local/LocalOpenAICompatClient.kt new file mode 100644 index 0000000..364666d --- /dev/null +++ b/app/src/main/java/com/pulseloop/coach/local/LocalOpenAICompatClient.kt @@ -0,0 +1,420 @@ +package com.pulseloop.coach.local + +import com.pulseloop.coach.config.LocalStructuredOutput +import com.pulseloop.coach.openai.FunctionCallOutput +import com.pulseloop.coach.openai.MessageOutput +import com.pulseloop.coach.openai.OpenAIResponse +import com.pulseloop.coach.openai.ResponseOutputItem +import com.pulseloop.coach.openai.ResponsesClient +import com.pulseloop.coach.openai.ResponsesError +import com.pulseloop.coach.openai.ResponsesHttp +import com.pulseloop.coach.openai.ResponsesToolSpecs +import com.pulseloop.coach.openai.TextContent +import com.pulseloop.coach.orchestration.CoachResponseSchema +import kotlinx.serialization.json.* +import java.util.UUID + +/** + * The self-hosted / local coach client — see `docs/local-llm-coach.md`. + * + * Adapts the app's [ResponsesClient] interface to the OpenAI **Chat Completions** API + * (`POST {base}/v1/chat/completions`) as implemented by Ollama, llama.cpp's `llama-server`, + * vLLM, SGLang, LM Studio and friends. Structurally this is + * [com.pulseloop.coach.minimax.MiniMaxClient] — same Responses→Chat translation, same + * accumulate-messages-across-`send` statefulness, same fresh-client-per-turn contract from the + * factory — with four deliberate differences, each forced by something a local backend does that + * a hosted one doesn't: + * + * 1. **The API key is optional.** Every one of these servers runs unauthenticated by default + * (`--api-key` is opt-in on llama.cpp/vLLM/SGLang; Ollama ignores the field entirely). A blank + * key omits the `Authorization` header rather than throwing — the readiness sentinel is the + * **base URL** instead (see `CoachClientResolver`). + * 2. **`developer` is folded into `system`, and all system turns are merged into one leading + * message.** SGLang validates roles against a pydantic `Literal` and raises (→ HTTP 400) for a + * role outside it; vLLM accepts `developer` and hands it to a Jinja chat template that usually + * has no branch for it. Many local templates additionally require the system turn to be first + * and singular. Folding + merging is lossless and works on all of them. + * 3. **Capabilities are user-declared, not assumed.** vLLM 400s on `tools` unless the server was + * started with `--enable-auto-tool-choice`; LM Studio has no `json_object` mode. Tool calling + * and structured output are therefore switches, defaulting to the combination that works + * everywhere (tools on, `response_format` off + prompt-injected schema). + * 4. **A long, configurable read timeout.** A 30B model on CPU can spend minutes on one round. + * + * Nothing else is sent: no `reasoning`, no `cache_control`, no provider-routing block. Only Ollama + * documents `reasoning_effort`, and vLLM/SGLang would warn or reject the rest. + */ +class LocalOpenAICompatClient( + /** Base URL as the user typed it; normalized via [LocalEndpoint]. */ + private val baseUrl: String, + private val model: String, + /** Optional — blank means send no `Authorization` header at all. */ + private val apiKey: String? = null, + private val toolCallingEnabled: Boolean = true, + private val structuredOutput: LocalStructuredOutput = LocalStructuredOutput.OFF, + /** `null`/0 = omit `max_tokens` and let the server decide. */ + private val maxOutputTokens: Int? = null, + private val readTimeoutSeconds: Int = DEFAULT_READ_TIMEOUT_SECONDS, +) : ResponsesClient { + private val json = Json { ignoreUnknownKeys = true } + + // Accumulated Chat Completions messages for this turn, minus the system block. + private var messages = mutableListOf() + // The merged leading system message (instructions + per-turn context + schema instruction). + private var systemPrompt: String = "" + // Maps generated response IDs → the assistant message (content + tool_calls) so a + // continuation turn can re-insert it before the matching tool results. + private val storedAssistantMessage = mutableMapOf() + + override suspend fun send(requestBody: ByteArray): OpenAIResponse { + // The base URL, not the key, is what makes this provider usable. + LocalEndpoint.validate(baseUrl)?.let { throw ResponsesError.Decoding(LocalEndpoint.message(it)) } + // Not MissingAPIKey — the key is optional on this provider, so blaming it sends the user + // to the one field that is allowed to be empty. A URL that survives `validate` but can't + // be normalized is malformed, which is what the message says. + val endpoint = LocalEndpoint.chatCompletionsUrl(baseUrl) + ?: throw ResponsesError.Decoding(LocalEndpoint.message(LocalEndpoint.Problem.MALFORMED)) + + val req = try { + json.parseToJsonElement(String(requestBody)).jsonObject + } catch (_: Exception) { + throw ResponsesError.Decoding("LocalOpenAICompatClient: invalid request body") + } + + val body = buildRequestBody(req) + val bodyBytes = json.encodeToString(JsonObject.serializer(), body).toByteArray() + + val headers = mutableMapOf() + apiKey?.takeIf { it.isNotBlank() }?.let { headers["Authorization"] = "Bearer $it" } + + // followRedirects = false: the body carries the user's health context, `validate` above + // vets only the typed URL, and cleartext is permitted app-wide — so a 307 off the LAN + // would resend it in the clear. See ResponsesHttp.clientFor. + val responseBody = ResponsesHttp.post( + endpoint, bodyBytes, headers, readTimeoutSeconds, followRedirects = false) + + val root = try { + json.parseToJsonElement(responseBody).jsonObject + } catch (_: Exception) { + throw ResponsesError.Decoding( + "The server at $endpoint did not return JSON — is it an OpenAI-compatible endpoint?") + } + return ingestResponse(root) + } + + // ── Request assembly (internal for unit tests) ─────────────────────── + + internal fun buildRequestBody(req: JsonObject): JsonObject { + val input = (req["input"] as? JsonArray)?.mapNotNull { it as? JsonObject } ?: emptyList() + val tools = (req["tools"] as? JsonArray)?.mapNotNull { it as? JsonObject } ?: emptyList() + val previousResponseId = (req["previous_response_id"] as? JsonPrimitive)?.contentOrNull + + if (previousResponseId == null) setupConversation(input) + else appendContinuation(previousResponseId, input) + + return buildChatBody(if (toolCallingEnabled) convertTools(tools) else emptyList()) + } + + // ── Conversation setup ─────────────────────────────────────────────── + + /** + * First turn. Every `system`/`developer` item is merged, in order, into a single leading + * system message; `user`/`assistant` items keep their order after it. The schema instruction + * joins the system block rather than trailing the conversation (where MiniMax puts it) because + * a system turn after a user turn raises in several local chat templates. + */ + private fun setupConversation(input: List) { + messages = mutableListOf() + storedAssistantMessage.clear() + + val systemParts = mutableListOf() + val conversation = mutableListOf() + for (item in input) { + val role = (item["role"] as? JsonPrimitive)?.contentOrNull ?: continue + if (item["content"] == null) continue + if (role == "system" || role == "developer") { + // A system turn is always plain instruction text; flatten any content parts. + systemParts.add(flattenText(item)) + } else { + conversation.add(JsonObject(mapOf( + "role" to JsonPrimitive(role), + "content" to chatContent(item), + ))) + } + } + // Only the prompt tells an unconstrained local model what shape to answer in. Even with + // `response_format` on, this stays — it's what the orchestrator's JSON-repair loop leans on + // when a small model ignores the grammar. + systemParts.add(CoachResponseSchema.promptInstruction) + systemPrompt = systemParts.filter { it.isNotBlank() }.joinToString("\n\n") + messages.addAll(conversation) + } + + /** + * Subsequent turns: replay the stored assistant message for [previousId] (Chat Completions + * requires the assistant `tool_calls` message to precede the `tool` results answering them), + * then append the new tool results / messages. A stray system/developer item here is folded + * into the leading system block rather than appended mid-conversation. + */ + private fun appendContinuation(previousId: String, input: List) { + storedAssistantMessage[previousId]?.let { messages.add(it) } + for (item in input) { + val type = (item["type"] as? JsonPrimitive)?.contentOrNull + val callId = (item["call_id"] as? JsonPrimitive)?.contentOrNull + val output = (item["output"] as? JsonPrimitive)?.contentOrNull + if (type == "function_call_output" && callId != null && output != null) { + messages.add(JsonObject(mapOf( + "role" to JsonPrimitive("tool"), + "tool_call_id" to JsonPrimitive(callId), + "content" to JsonPrimitive(output), + ))) + } else { + val role = (item["role"] as? JsonPrimitive)?.contentOrNull ?: continue + if (item["content"] == null) continue + if (role == "system" || role == "developer") { + systemPrompt = listOf(systemPrompt, flattenText(item)) + .filter { it.isNotBlank() }.joinToString("\n\n") + } else { + messages.add(JsonObject(mapOf( + "role" to JsonPrimitive(role), + "content" to chatContent(item), + ))) + } + } + } + } + + /** All text in a message item, whether `content` is a string or a content-part array. */ + private fun flattenText(item: JsonObject): String { + val content = item["content"] + if (content is JsonPrimitive && content.isString) return content.content + val parts = (content as? JsonArray)?.mapNotNull { it as? JsonObject } ?: return "" + return parts.mapNotNull { (it["text"] as? JsonPrimitive)?.contentOrNull }.joinToString("\n") + } + + /** + * Converts a Responses-API message item's `content` into Chat Completions `content`. Text + * stays a plain string; images map to `{type:image_url, image_url:{url}}` parts. Local vision + * backends take base64 `data:` URLs (Ollama explicitly rejects remote image URLs), which is + * exactly what `CoachImagePayload.dataURL` produces. + */ + private fun chatContent(item: JsonObject): JsonElement { + val content = item["content"] + if (content is JsonPrimitive && content.isString) return content + val parts = (content as? JsonArray)?.mapNotNull { it as? JsonObject } + ?: return JsonPrimitive("") + val out = mutableListOf() + for (part in parts) { + when ((part["type"] as? JsonPrimitive)?.contentOrNull) { + "input_text", "text" -> { + (part["text"] as? JsonPrimitive)?.contentOrNull?.let { + out.add(JsonObject(mapOf( + "type" to JsonPrimitive("text"), + "text" to JsonPrimitive(it), + ))) + } + } + "input_image" -> { + (part["image_url"] as? JsonPrimitive)?.contentOrNull?.let { + out.add(JsonObject(mapOf( + "type" to JsonPrimitive("image_url"), + "image_url" to JsonObject(mapOf("url" to JsonPrimitive(it))), + ))) + } + } + } + } + return JsonArray(out) + } + + // ── Tool conversion (Responses flat → Chat Completions nested) ─────── + + /** + * Flat Responses function specs → Chat Completions' nested `{type:function, function:{…}}`. + * The hosted `web_search` tool is dropped: no local engine has one. `strict` is dropped too — + * it's an OpenAI structured-outputs extension that vLLM/SGLang don't act on and some stricter + * proxies reject inside a function spec. + */ + private fun convertTools(tools: List): List = + ResponsesToolSpecs.parse(tools).functions.map { spec -> + val fn = mutableMapOf("name" to JsonPrimitive(spec.name)) + spec.description?.let { fn["description"] = JsonPrimitive(it) } + spec.parameters?.let { fn["parameters"] = it } + JsonObject(mapOf( + "type" to JsonPrimitive("function"), + "function" to JsonObject(fn), + )) + } + + // ── Build request body ─────────────────────────────────────────────── + + internal fun buildChatBody(tools: List): JsonObject { + val allMessages = mutableListOf() + if (systemPrompt.isNotBlank()) { + allMessages.add(JsonObject(mapOf( + "role" to JsonPrimitive("system"), + "content" to JsonPrimitive(systemPrompt), + ))) + } + allMessages.addAll(messages) + + val body = mutableMapOf( + // llama.cpp ignores `model` unless started with --alias; everyone else requires it. + // Sending it unconditionally is correct for both. + "model" to JsonPrimitive(model), + "messages" to JsonArray(allMessages), + ) + if (tools.isNotEmpty()) body["tools"] = JsonArray(tools) + responseFormat()?.let { body["response_format"] = it } + maxOutputTokens?.takeIf { it > 0 }?.let { body["max_tokens"] = JsonPrimitive(it) } + return JsonObject(body) + } + + /** + * The `response_format` block, or null when the user left structured output off (the default, + * and the only setting that works on every backend). `JSON_SCHEMA` uses the nested OpenAI + * shape — `{type:"json_schema", json_schema:{name, strict, schema}}` — which vLLM, SGLang, + * LM Studio and recent llama.cpp all accept. `JSON_OBJECT` is the older, weaker mode; LM + * Studio doesn't implement it, hence the choice. + */ + internal fun responseFormat(): JsonObject? = when (structuredOutput) { + LocalStructuredOutput.OFF -> null + LocalStructuredOutput.JSON_OBJECT -> JsonObject(mapOf("type" to JsonPrimitive("json_object"))) + LocalStructuredOutput.JSON_SCHEMA -> JsonObject(mapOf( + "type" to JsonPrimitive("json_schema"), + "json_schema" to JsonObject(mapOf( + "name" to JsonPrimitive("coach_response"), + "strict" to JsonPrimitive(true), + "schema" to CoachResponseSchema.schema, + )), + )) + } + + // ── Parse Chat Completions response → OpenAIResponse (internal for tests) ─ + + internal fun ingestResponse(root: JsonObject): OpenAIResponse { + // Some servers (and most reverse proxies in front of them) report errors in the body on an + // HTTP 200. `error` may be an object or, on llama.cpp, a bare string. + (root["error"] as? JsonObject)?.let { err -> + val msg = (err["message"] as? JsonPrimitive)?.contentOrNull ?: err.toString().take(200) + throw ResponsesError.Decoding("Server error: $msg") + } + (root["error"] as? JsonPrimitive)?.contentOrNull?.let { + throw ResponsesError.Decoding("Server error: $it") + } + + val first = (root["choices"] as? JsonArray)?.firstOrNull() as? JsonObject + val message = first?.get("message") as? JsonObject + ?: throw ResponsesError.Decoding( + "No `choices` in the response — the server may not be OpenAI-compatible. " + + "Got: ${root.toString().take(300)}") + + val responseId = (root["id"] as? JsonPrimitive)?.contentOrNull + ?.takeIf { it.isNotEmpty() } ?: UUID.randomUUID().toString() + val outputItems = mutableListOf() + val assistantMessage = mutableMapOf("role" to JsonPrimitive("assistant")) + + // Open reasoning models emit their chain of thought either as an inline `…` + // block (llama.cpp/Ollama without a reasoning parser) or split into a separate field — + // `reasoning` on vLLM 0.27+, `reasoning_content` on older builds and SGLang. Neither + // belongs in the coach_response JSON: the first is stripped, the others aren't read. + val content = (message["content"] as? JsonPrimitive)?.contentOrNull?.let { stripThinking(it) } + if (!content.isNullOrEmpty()) { + outputItems.add(MessageOutput(role = "assistant", content = listOf(TextContent(content)))) + assistantMessage["content"] = JsonPrimitive(content) + } else { + assistantMessage["content"] = JsonNull + } + + val toolCalls = (message["tool_calls"] as? JsonArray)?.mapNotNull { it as? JsonObject } + if (toolCalls != null) { + val storedCalls = mutableListOf() + for (call in toolCalls) { + val fn = call["function"] as? JsonObject ?: continue + val name = (fn["name"] as? JsonPrimitive)?.contentOrNull ?: continue + val callId = (call["id"] as? JsonPrimitive)?.contentOrNull?.takeIf { it.isNotEmpty() } + ?: ("local_call_" + UUID.randomUUID().toString().replace("-", "").take(12)) + // `arguments` is a JSON *string* per the spec, but several local tool-call parsers + // emit a JSON object instead. Re-encode that so the orchestrator's parse succeeds + // instead of failing the round on a well-formed-but-differently-typed field. + val args = when (val a = fn["arguments"]) { + is JsonPrimitive -> a.contentOrNull ?: "{}" + is JsonObject -> a.toString() + else -> "{}" + } + outputItems.add(FunctionCallOutput(id = callId, callId = callId, name = name, arguments = args)) + storedCalls.add(JsonObject(mapOf( + "id" to JsonPrimitive(callId), + "type" to JsonPrimitive("function"), + "function" to JsonObject(mapOf( + "name" to JsonPrimitive(name), + "arguments" to JsonPrimitive(args), + )), + ))) + } + if (storedCalls.isNotEmpty()) assistantMessage["tool_calls"] = JsonArray(storedCalls) + } + + if (outputItems.isEmpty()) { + // A reasoning model that ran out of budget mid-thought returns null content, no tool + // calls, and finish_reason "length". Bare "the model returned no output" sends the + // user looking for the wrong problem — the fix is the Max tokens field. + val finishReason = (first["finish_reason"] as? JsonPrimitive)?.contentOrNull + if (finishReason == "length") throw ResponsesError.Decoding( + "The model hit its token limit before producing an answer" + + (if (message["reasoning"] != null || message["reasoning_content"] != null) + " (it spent the budget reasoning)" else "") + + ". Raise Max tokens in Settings → AI Coach, or leave it blank.") + throw ResponsesError.EmptyOutput + } + + storedAssistantMessage[responseId] = JsonObject(assistantMessage) + return OpenAIResponse(id = responseId, output = outputItems, usage = usage(root)) + } + + /** Maps the `usage` block when present. Local servers all report the OpenAI split; a server + * that omits it leaves usage null, and the coach shows no token counts rather than zeros. */ + private fun usage(root: JsonObject): com.pulseloop.coach.usage.CoachTokenUsage? { + val usage = root["usage"] as? JsonObject ?: return null + val input = usage["prompt_tokens"]?.jsonPrimitive?.intOrNull ?: return null + val output = usage["completion_tokens"]?.jsonPrimitive?.intOrNull ?: return null + return com.pulseloop.coach.usage.CoachTokenUsage(inputTokens = input, outputTokens = output) + } + + /** + * Removes `…` reasoning blocks. Tolerant at both ends: an unterminated trailing + * `` (truncated output) drops its remainder, and a leading unmatched `` drops + * everything before it. + * + * That second case is the common one, not an edge case. R1-style distills served by llama.cpp + * and Ollama have the *opening* tag injected into the prompt by the chat template, so the + * completion starts mid-thought and the only tag in `content` is a bare closing one. Matching + * pairs only, the whole chain of thought used to reach `CoachResponseParser.parse`, which then + * burned `maxFinalAttempts` repair generations — each up to the 180 s read timeout — before + * the turn ended in "Bad response". + */ + private fun stripThinking(text: String): String { + var body = text + val firstClose = body.indexOf("") + if (firstClose >= 0) { + val firstOpen = body.indexOf("") + if (firstOpen < 0 || firstClose < firstOpen) body = body.substring(firstClose + 8) + } + val out = StringBuilder() + var scan = 0 + while (true) { + val open = body.indexOf("", scan) + if (open < 0) break + out.append(body, scan, open) + val close = body.indexOf("", open + 7) + if (close < 0) { scan = body.length; break } + scan = close + 8 + } + out.append(body, scan, body.length) + return out.toString().trim() + } + + companion object { + /** Generous by cloud standards, ordinary for a quantized model on consumer hardware. */ + const val DEFAULT_READ_TIMEOUT_SECONDS = 180 + } +} diff --git a/app/src/main/java/com/pulseloop/coach/openai/OpenAIResponsesClient.kt b/app/src/main/java/com/pulseloop/coach/openai/OpenAIResponsesClient.kt index 71b02d7..90f8224 100644 --- a/app/src/main/java/com/pulseloop/coach/openai/OpenAIResponsesClient.kt +++ b/app/src/main/java/com/pulseloop/coach/openai/OpenAIResponsesClient.kt @@ -1,7 +1,9 @@ package com.pulseloop.coach.openai import com.pulseloop.coach.attachments.CoachImagePayload +import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.delay +import kotlinx.coroutines.withContext import kotlinx.serialization.Serializable import kotlinx.serialization.json.* import okhttp3.MediaType.Companion.toMediaType @@ -38,6 +40,15 @@ class OpenAIResponsesClient( * (30 s connect / 60 s read) instead of a fresh connection pool per agent turn, * and one place for the [ResponsesError.Transport]/[ResponsesError.Http] mapping * all three providers share. + * + * Both entry points switch to [Dispatchers.IO] themselves. `execute()` is a BLOCKING call, and + * Android throws `NetworkOnMainThreadException` for one on the main thread — an exception whose + * `message` is **null**, so it surfaces to the user as a bare "no connection" that points at + * their network instead of at the code. `CoachOrchestrator` happens to wrap its turns in + * `withContext(Dispatchers.IO)` already, which is why this was invisible until a Compose + * `scope.launch` (Settings' server-detect button, which runs on `Dispatchers.Main`) became the + * second caller. Owning the dispatcher here rather than trusting every call site removes the + * whole class of bug; the redundant nesting for orchestrator calls costs nothing. */ internal object ResponsesHttp { private val jsonMediaType = "application/json".toMediaType() @@ -52,49 +63,112 @@ internal object ResponsesHttp { * [ResponsesError.Transport] on network failure and [ResponsesError.Http] * (with the error body) on a non-2xx status. */ - suspend fun post(url: String, body: ByteArray, headers: Map = emptyMap()): String { + suspend fun post( + url: String, + body: ByteArray, + headers: Map = emptyMap(), + readTimeoutSeconds: Int? = null, + followRedirects: Boolean = true, + ): String { val builder = okhttp3.Request.Builder() .url(url) .post(okhttp3.RequestBody.create(jsonMediaType, body)) for ((name, value) in headers) builder.header(name, value) val request = builder.build() + val call = clientFor(readTimeoutSeconds, followRedirects) - for (attempt in 0..MAX_UNSENT_RETRIES) { + return withContext(Dispatchers.IO) { + for (attempt in 0..MAX_UNSENT_RETRIES) { + try { + // Both the call and the body read live inside the try. A read timeout can fire + // while the body is still streaming, and if that escaped uncaught it would reach + // CoachTurnError as a bare SocketTimeoutException — bypassing the transport copy + // and printing the JDK's one-word "timeout" again, the exact bug this fixes. + // `use` closes the response on every path, including a mid-read failure. + return@withContext call.newCall(request).execute().use { response -> + val text = response.body?.string() ?: "" + if (!response.isSuccessful) throw ResponsesError.Http(response.code, text) + text + } + } catch (e: ResponsesError) { + // An HTTP status is an answer from the provider, not a transport failure. Never + // retried, and never re-wrapped as Transport by the catch below. + throw e + } catch (e: Exception) { + // Only retry failures that provably never reached the provider: DNS resolution and + // TCP connect. A momentary DNS miss (radio handover, a VPN or private-DNS resolver + // still coming up) otherwise kills the whole turn and burns the user's message. + // + // A read timeout is deliberately NOT retried. OkHttp reports connect and read + // timeouts as the same SocketTimeoutException, so we cannot tell "never sent" from + // "sent, answer lost" — and re-sending the latter bills the user's API key for a + // generation that already ran. + if (!isProvablyUnsent(e) || attempt == MAX_UNSENT_RETRIES) throw ResponsesError.Transport(e) + // delay(), not Thread.sleep(): the turn is cancellable (the user leaves the coach + // screen, WorkManager stops the summary worker), and a blocking sleep would keep an + // IO thread parked and then fire the remaining doomed attempts anyway. + delay(RETRY_BACKOFF_MS shl attempt) // 400ms, 800ms + } + } + // Unreachable — the final attempt either returns or throws — but keeps the + // compiler happy. + throw IllegalStateException("request never ran") + } + } + + /** + * A one-off `GET`, for the local provider's `/v1/models` discovery. No retry: unlike a chat + * turn this is a user-initiated refresh they can simply press again, and a failure here is + * informational rather than a burned message. + */ + suspend fun get( + url: String, + headers: Map = emptyMap(), + readTimeoutSeconds: Int? = null, + followRedirects: Boolean = true, + ): String { + val builder = okhttp3.Request.Builder().url(url).get() + for ((name, value) in headers) builder.header(name, value) + val request = builder.build() + return withContext(Dispatchers.IO) { try { - // Both the call and the body read live inside the try. A read timeout can fire - // while the body is still streaming, and if that escaped uncaught it would reach - // CoachTurnError as a bare SocketTimeoutException — bypassing the transport copy - // and printing the JDK's one-word "timeout" again, the exact bug this fixes. - // `use` closes the response on every path, including a mid-read failure. - return client.newCall(request).execute().use { response -> + clientFor(readTimeoutSeconds, followRedirects).newCall(request).execute().use { response -> val text = response.body?.string() ?: "" if (!response.isSuccessful) throw ResponsesError.Http(response.code, text) text } } catch (e: ResponsesError) { - // An HTTP status is an answer from the provider, not a transport failure. Never - // retried, and never re-wrapped as Transport by the catch below. throw e } catch (e: Exception) { - // Only retry failures that provably never reached the provider: DNS resolution and - // TCP connect. A momentary DNS miss (radio handover, a VPN or private-DNS resolver - // still coming up) otherwise kills the whole turn and burns the user's message. - // - // A read timeout is deliberately NOT retried. OkHttp reports connect and read - // timeouts as the same SocketTimeoutException, so we cannot tell "never sent" from - // "sent, answer lost" — and re-sending the latter bills the user's API key for a - // generation that already ran. - if (!isProvablyUnsent(e) || attempt == MAX_UNSENT_RETRIES) throw ResponsesError.Transport(e) - // delay(), not Thread.sleep(): the turn is cancellable (the user leaves the coach - // screen, WorkManager stops the summary worker), and a blocking sleep would keep an - // IO thread parked and then fire the remaining doomed attempts anyway. - delay(RETRY_BACKOFF_MS shl attempt) // 400ms, 800ms + throw ResponsesError.Transport(e) } } - // Unreachable — the final attempt either returns or throws — but keeps the compiler happy. - throw IllegalStateException("request never ran") } + /** + * The shared client, or a derived one with a longer read timeout. `newBuilder()` shares the + * connection pool and dispatcher, so a self-hosted model that thinks for three minutes doesn't + * cost us a second pool. Null (every cloud provider) keeps the 60 s default. + * + * [followRedirects] = false is the local provider's. `LocalEndpoint.validate` vets the URL the + * *user typed* — it cannot vet where a redirect lands, and the app permits cleartext app-wide + * (`network_security_config.xml`, which has no CIDR syntax), so a 307 from the validated LAN + * host to a public `http://` one would resend the health-context POST body in the clear with + * nothing left to stop it. Cloud providers keep redirects; their hosts are https:// constants. + */ + private fun clientFor( + readTimeoutSeconds: Int?, + followRedirects: Boolean = true, + ): okhttp3.OkHttpClient = + if ((readTimeoutSeconds == null || readTimeoutSeconds <= 0) && followRedirects) client + else client.newBuilder() + .apply { + if (readTimeoutSeconds != null && readTimeoutSeconds > 0) + readTimeout(readTimeoutSeconds.toLong(), java.util.concurrent.TimeUnit.SECONDS) + if (!followRedirects) { followRedirects(false); followSslRedirects(false) } + } + .build() + /** * True when [e] means the request never left the device, so re-sending it is side-effect free. * `UnknownHostException` is DNS; `ConnectException` is a refused/unreachable TCP connect. diff --git a/app/src/main/java/com/pulseloop/coach/usage/CoachModelPricing.kt b/app/src/main/java/com/pulseloop/coach/usage/CoachModelPricing.kt index 8b432de..4b033f1 100644 --- a/app/src/main/java/com/pulseloop/coach/usage/CoachModelPricing.kt +++ b/app/src/main/java/com/pulseloop/coach/usage/CoachModelPricing.kt @@ -62,6 +62,19 @@ object CoachPricingCatalog { /** The model strings that always cost $0 (local / scripted). */ private val freeModels: Set = setOf("offline-stub") + /** + * Cost for a turn, given the provider that ran it. A self-hosted model costs the user nothing + * per token no matter what it's called, and its name (`qwen3:8b`, `Llama-3.3-70B`, or blank on + * llama.cpp) would otherwise fall through to the longest-prefix match and, at worst, price a + * local run at a cloud rate. Provider-blind callers keep the two-arg form. + * + * [providerRaw] is `CoachProviderMode.rawValue` as persisted on the message; unknown/null + * values fall through to the catalog. + */ + fun cost(model: String, usage: CoachTokenUsage, providerRaw: String?): Double? = + if (providerRaw == com.pulseloop.coach.config.CoachProviderMode.LOCAL_OPENAI_COMPAT.rawValue) 0.0 + else cost(model, usage) + /** The estimated USD cost of [usage] for [model]. Returns 0 for offline * models, `null` for an unrecognized model, else the priced estimate. */ fun cost(model: String, usage: CoachTokenUsage): Double? { diff --git a/app/src/main/java/com/pulseloop/ui/screens/SettingsScreen.kt b/app/src/main/java/com/pulseloop/ui/screens/SettingsScreen.kt index ce7e13d..ea564f7 100644 --- a/app/src/main/java/com/pulseloop/ui/screens/SettingsScreen.kt +++ b/app/src/main/java/com/pulseloop/ui/screens/SettingsScreen.kt @@ -58,6 +58,10 @@ fun SettingsScreen( CoachProviderMode.OFFLINE_STUB -> "Offline" CoachProviderMode.USER_GEMINI_KEY -> providerStore.geminiModel CoachProviderMode.USER_OPENROUTER_KEY -> providerStore.openRouterModel + CoachProviderMode.USER_MINIMAX_KEY -> providerStore.minimaxModel + // Local: the model name, or just "Local" for a server (llama.cpp) that ignores the field. + CoachProviderMode.LOCAL_OPENAI_COMPAT -> + providerStore.localModel.ifBlank { "Local" } CoachProviderMode.BACKEND_PROXY -> "Backend proxy" else -> keyStore.model } diff --git a/app/src/main/java/com/pulseloop/ui/screens/SettingsSubScreens.kt b/app/src/main/java/com/pulseloop/ui/screens/SettingsSubScreens.kt index 42a5446..b1400a6 100644 --- a/app/src/main/java/com/pulseloop/ui/screens/SettingsSubScreens.kt +++ b/app/src/main/java/com/pulseloop/ui/screens/SettingsSubScreens.kt @@ -29,6 +29,7 @@ import androidx.compose.material3.* import androidx.compose.runtime.* import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier +import androidx.compose.ui.focus.onFocusChanged import androidx.compose.ui.draw.clip import androidx.compose.ui.hapticfeedback.HapticFeedbackType import androidx.compose.ui.platform.LocalContext @@ -42,6 +43,9 @@ import androidx.compose.ui.unit.sp import com.pulseloop.coach.config.CoachProviderMode import com.pulseloop.coach.config.CoachProviderSettingsStore import com.pulseloop.coach.config.GeminiModel +import com.pulseloop.coach.config.LocalStructuredOutput +import com.pulseloop.coach.local.LocalCapabilityProbe +import com.pulseloop.coach.local.LocalEndpoint import com.pulseloop.coach.config.MiniMaxModel import com.pulseloop.coach.config.OpenRouterModel import com.pulseloop.data.DemoDataSeeder @@ -205,8 +209,16 @@ fun CoachSettingsScreen(onBack: () -> Unit) { Column(Modifier.padding(16.dp)) { Text("AI Coach", style = MaterialTheme.typography.titleMedium, fontWeight = FontWeight.SemiBold) Spacer(Modifier.height(4.dp)) + // Readiness differs by provider: the local mode is gated by the server address, + // not a key, so the old key-only check told a correctly-configured user their + // "API key" was missing. + val isLocal = providerStore.providerMode == CoachProviderMode.LOCAL_OPENAI_COMPAT + val ready = if (isLocal) providerStore.hasLocalBaseUrl else keyStore.apiKey.isNotBlank() + val activeModel = if (isLocal) providerStore.localModel.ifBlank { "local server" } + else selectedModel Text( - if (coachEnabled && keyStore.apiKey.isNotBlank()) "Active — ${selectedModel}" + if (coachEnabled && ready) "Active — $activeModel" + else if (coachEnabled && isLocal) "Server address needed" else if (coachEnabled) "API key needed" else "Disabled", style = MaterialTheme.typography.bodySmall, @@ -260,12 +272,28 @@ fun CoachSettingsScreen(onBack: () -> Unit) { var orSort by remember { mutableStateOf(providerStore.orProviderSort) } var reasoningEffort by remember { mutableStateOf(providerStore.reasoningEffort) } var imageInput by remember { mutableStateOf(providerStore.imageInputEnabled) } + // Local / self-hosted (docs/local-llm-coach.md). + var localBaseUrl by remember { mutableStateOf(providerStore.localBaseUrl) } + var localKey by remember { mutableStateOf(providerStore.localApiKey) } + var localKeyVisible by remember { mutableStateOf(false) } + var localModel by remember { mutableStateOf(providerStore.localModel) } + var localToolCalling by remember { mutableStateOf(providerStore.localToolCalling) } + var localStructured by remember { mutableStateOf(providerStore.localStructuredOutput) } + var localMaxTokens by remember { mutableStateOf(providerStore.localMaxTokens.takeIf { it > 0 }?.toString() ?: "") } + var localTimeout by remember { mutableStateOf(providerStore.localTimeoutSeconds.toString()) } + // Discovered via GET /v1/models; advisory, the typed slug always wins. + var localDiscovered by remember { mutableStateOf>(emptyList()) } + var localProbeBusy by remember { mutableStateOf(false) } + var localProbeResult by remember { mutableStateOf(null) } + var localProbeNotes by remember { mutableStateOf>(emptyList()) } + var localProbeOk by remember { mutableStateOf(false) } val providerOptions = listOf( CoachProviderMode.USER_OPENAI_KEY to "OpenAI", CoachProviderMode.USER_GEMINI_KEY to "Google Gemini", CoachProviderMode.USER_OPENROUTER_KEY to "OpenRouter (100+ models)", CoachProviderMode.USER_MINIMAX_KEY to "MiniMax", + CoachProviderMode.LOCAL_OPENAI_COMPAT to "Local / self-hosted (no key needed)", ) val providerLabel = providerOptions.firstOrNull { it.first == providerMode }?.second ?: "OpenAI" @@ -421,6 +449,228 @@ fun CoachSettingsScreen(onBack: () -> Unit) { onRemove = { minimaxKey = ""; providerStore.minimaxApiKey = "" }, ) } + CoachProviderMode.LOCAL_OPENAI_COMPAT -> { + // Any OpenAI-Chat-Completions-compatible server the user runs: Ollama, + // llama.cpp, vLLM, SGLang, LM Studio. No key required — see + // docs/local-llm-coach.md. The base URL is what gates readiness. + val urlProblem = LocalEndpoint.validate(localBaseUrl) + .takeIf { localBaseUrl.isNotBlank() } + OutlinedTextField( + value = localBaseUrl, + onValueChange = { + localBaseUrl = it; providerStore.localBaseUrl = it + localProbeResult = null; localProbeOk = false + localProbeNotes = emptyList(); localDiscovered = emptyList() + }, + label = { Text("Server address") }, + placeholder = { Text("http://192.168.1.50:11434") }, + modifier = Modifier.fillMaxWidth(), + singleLine = true, + isError = urlProblem != null, + keyboardOptions = KeyboardOptions(keyboardType = KeyboardType.Uri, imeAction = ImeAction.Done), + ) + Text( + urlProblem?.let { LocalEndpoint.message(it) } + ?: "Default ports — Ollama 11434 · LM Studio 1234 · llama.cpp 8080 · vLLM 8000 · SGLang 30000. " + + "The /v1 path is added for you.", + style = MaterialTheme.typography.bodySmall, + color = if (urlProblem != null) MaterialTheme.colorScheme.error + else MaterialTheme.colorScheme.onSurfaceVariant, + modifier = Modifier.padding(top = 4.dp), + ) + + Spacer(Modifier.height(8.dp)) + // One press does the whole setup: identifies the engine, lists models, + // and probes whether the server actually accepts `tools` and + // `response_format` — which depend on launch flags that no metadata + // endpoint exposes. The detected values are applied straight to the + // controls below, which stay editable. + Button( + onClick = { + localProbeBusy = true; localProbeResult = null; localProbeNotes = emptyList() + scope.launch { + try { + val report = LocalCapabilityProbe.run( + baseUrl = localBaseUrl, + apiKey = localKey, + currentModel = localModel, + ) + localDiscovered = report.models + if (report.suggestedModel.isNotBlank()) { + localModel = report.suggestedModel + providerStore.localModel = localModel + } + // Only a probe that reached a verdict may overwrite + // these. Detect is also how you refresh the model + // list, so it gets pressed on a working setup — and + // the safe defaults behind `suggested*` (tools ON, + // structured OFF) would then undo a deliberate choice + // rather than leave it alone. The notes say what was + // skipped and why. + if (report.toolCallingConclusive) { + localToolCalling = report.suggestedToolCalling + providerStore.localToolCalling = localToolCalling + } + if (report.structuredOutputConclusive) { + localStructured = report.suggestedStructuredOutput + providerStore.localStructuredOutput = localStructured + } + // Derived from the server's context window, minus a + // reserve for the prompt — never a straight copy, or + // `prompt + max_tokens` would exceed the context and + // the server would reject the request. 0 means the + // server reported no context window at all (a thin + // proxy, an engine we can't identify, Ollama when + // `/api/show` doesn't parse) — "not detected", which + // is no reason to clear a value the user typed. + if (report.suggestedMaxTokens > 0) { + localMaxTokens = report.suggestedMaxTokens.toString() + providerStore.localMaxTokens = report.suggestedMaxTokens + } + localProbeOk = true + localProbeResult = report.summary + localProbeNotes = report.notes + } catch (e: LocalCapabilityProbe.Unreachable) { + localProbeOk = false; localProbeResult = e.reason + } catch (e: Exception) { + localProbeOk = false + localProbeResult = e.message ?: "Couldn't reach the server." + } + localProbeBusy = false + } + }, + enabled = localBaseUrl.isNotBlank() && urlProblem == null && !localProbeBusy, + modifier = Modifier.fillMaxWidth(), + ) { Text(if (localProbeBusy) "Detecting…" else "Detect server & configure") } + if (localProbeBusy) { + Text( + "Sending three tiny test messages. This can take a minute if the model " + + "still has to load.", + style = MaterialTheme.typography.bodySmall, + color = MaterialTheme.colorScheme.onSurfaceVariant, + modifier = Modifier.padding(top = 4.dp), + ) + } + localProbeResult?.let { + Text( + it, + style = MaterialTheme.typography.bodySmall, + color = if (localProbeOk) PulseColors.success else MaterialTheme.colorScheme.error, + modifier = Modifier.padding(top = 4.dp), + ) + } + // Only non-empty when a probe was inconclusive or refused — the + // difference between "off because your server said no" and "off + // because we couldn't tell" is the whole point of showing it. + localProbeNotes.forEach { note -> + Text( + "• $note", + style = MaterialTheme.typography.bodySmall, + color = MaterialTheme.colorScheme.onSurfaceVariant, + ) + } + + Spacer(Modifier.height(8.dp)) + if (localDiscovered.isNotEmpty()) { + ModelDropdown( + "Model", localModel.ifBlank { "Choose a model" }, + localDiscovered.map { it to "" }, + ) { localModel = it; providerStore.localModel = it } + } + // Always typeable: a router in front of the server can serve names + // /v1/models doesn't list, and llama.cpp ignores the field entirely. + OutlinedTextField( + value = localModel, + onValueChange = { localModel = it; providerStore.localModel = it }, + label = { Text("Model name") }, + placeholder = { Text("qwen3:8b") }, + modifier = Modifier.fillMaxWidth(), + singleLine = true, + ) + Spacer(Modifier.height(8.dp)) + + KeyField( + label = "API key (optional)", value = localKey, visible = localKeyVisible, + saved = providerStore.hasLocalKey, + onValue = { localKey = it }, onVisibility = { localKeyVisible = !localKeyVisible }, + onSave = { providerStore.localApiKey = localKey }, + onRemove = { localKey = ""; providerStore.localApiKey = "" }, + ) + Text( + "Only needed if you started the server with --api-key. Leave blank otherwise.", + style = MaterialTheme.typography.bodySmall, + color = MaterialTheme.colorScheme.onSurfaceVariant, + ) + + Spacer(Modifier.height(8.dp)) + Row(Modifier.fillMaxWidth(), horizontalArrangement = Arrangement.SpaceBetween, verticalAlignment = Alignment.CenterVertically) { + Column(Modifier.weight(1f)) { + Text("Tool calling") + Text( + "Lets the coach read your data. Detect sets this for you; turn it off " + + "if your server rejects tools (vLLM needs --enable-auto-tool-choice).", + style = MaterialTheme.typography.bodySmall, + color = MaterialTheme.colorScheme.onSurfaceVariant, + ) + } + Switch(checked = localToolCalling, onCheckedChange = { + localToolCalling = it; providerStore.localToolCalling = it + }) + } + + Spacer(Modifier.height(8.dp)) + ModelDropdown( + "Response format", localStructured.label, + LocalStructuredOutput.entries.map { it.label to it.blurb }, + ) { picked -> + val mode = LocalStructuredOutput.entries.first { it.label == picked } + localStructured = mode; providerStore.localStructuredOutput = mode + } + + Row(Modifier.fillMaxWidth(), horizontalArrangement = Arrangement.spacedBy(8.dp)) { + OutlinedTextField( + value = localMaxTokens, + onValueChange = { + localMaxTokens = it.filter { c -> c.isDigit() } + providerStore.localMaxTokens = localMaxTokens.toIntOrNull() ?: 0 + }, + label = { Text("Max tokens") }, + placeholder = { Text("auto") }, + modifier = Modifier.weight(1f), + singleLine = true, + keyboardOptions = KeyboardOptions(keyboardType = KeyboardType.Number, imeAction = ImeAction.Done), + ) + OutlinedTextField( + value = localTimeout, + onValueChange = { + localTimeout = it.filter { c -> c.isDigit() } + localTimeout.toIntOrNull()?.let { v -> providerStore.localTimeoutSeconds = v } + }, + label = { Text("Timeout (s)") }, + modifier = Modifier + .weight(1f) + // The store clamps to 10..1800 on write, and a blank field + // skips the write entirely, so the text can show a value + // that was never stored (type "5", store holds 10) or an + // empty box over a live setting. Reconciling per keystroke + // would fight the user — "1" would jump to "10" — so read + // the stored value back when the field loses focus. + .onFocusChanged { focus -> + if (!focus.isFocused) + localTimeout = providerStore.localTimeoutSeconds.toString() + }, + singleLine = true, + keyboardOptions = KeyboardOptions(keyboardType = KeyboardType.Number, imeAction = ImeAction.Done), + ) + } + Text( + "Leave max tokens blank to let the server decide. Raise the timeout for a large " + + "model on CPU — a slow reply is not retried.", + style = MaterialTheme.typography.bodySmall, + color = MaterialTheme.colorScheme.onSurfaceVariant, + modifier = Modifier.padding(top = 4.dp), + ) + } else -> { // OpenAI (and legacy modes): the original model picker + key field. ModelDropdown("Model", selectedModel, models.map { it to "" }) { @@ -459,9 +709,11 @@ fun CoachSettingsScreen(onBack: () -> Unit) { HorizontalDivider() - // Tool toggles. MiniMax's compat endpoint has no hosted web search, so hide - // the toggle for it (the client drops the tool anyway — this is the UI half). - if (providerMode != CoachProviderMode.USER_MINIMAX_KEY) { + // Tool toggles. Neither MiniMax's compat endpoint nor any self-hosted engine + // has a hosted web search, so hide the toggle for both (each client drops the + // tool anyway — this is the UI half). + if (providerMode != CoachProviderMode.USER_MINIMAX_KEY && + providerMode != CoachProviderMode.LOCAL_OPENAI_COMPAT) { Row(Modifier.fillMaxWidth(), horizontalArrangement = Arrangement.SpaceBetween, verticalAlignment = Alignment.CenterVertically) { Column(Modifier.weight(1f)) { Text("Web Search") diff --git a/app/src/main/java/com/pulseloop/ui/viewmodels/ViewModels.kt b/app/src/main/java/com/pulseloop/ui/viewmodels/ViewModels.kt index 3e9e9b6..bd7363b 100644 --- a/app/src/main/java/com/pulseloop/ui/viewmodels/ViewModels.kt +++ b/app/src/main/java/com/pulseloop/ui/viewmodels/ViewModels.kt @@ -864,7 +864,10 @@ class CoachViewModel( // estimate. Both are null when the model is unknown/on-device/no usage reported — // the usage sheet shows "cost unavailable" rather than a wrong number. val cost = usage?.reportedCostUSD - ?: usage?.let { com.pulseloop.coach.usage.CoachPricingCatalog.cost(result.modelUsed, it) } + ?: usage?.let { + com.pulseloop.coach.usage.CoachPricingCatalog.cost( + result.modelUsed, it, result.providerUsed) + } db.coachMessageDao().insert(CoachMessageEntity( id = assistantMessageId, conversationId = conversationId, role = persistedRole, body = reply.text, diff --git a/app/src/main/res/xml/network_security_config.xml b/app/src/main/res/xml/network_security_config.xml new file mode 100644 index 0000000..029a03d --- /dev/null +++ b/app/src/main/res/xml/network_security_config.xml @@ -0,0 +1,29 @@ + + + + + + + + + diff --git a/app/src/test/java/com/pulseloop/coach/CoachClientResolverTest.kt b/app/src/test/java/com/pulseloop/coach/CoachClientResolverTest.kt index 300ae9f..9572a57 100644 --- a/app/src/test/java/com/pulseloop/coach/CoachClientResolverTest.kt +++ b/app/src/test/java/com/pulseloop/coach/CoachClientResolverTest.kt @@ -1,6 +1,7 @@ package com.pulseloop.coach.config import com.pulseloop.coach.gemini.GeminiClient +import com.pulseloop.coach.local.LocalOpenAICompatClient import com.pulseloop.coach.minimax.MiniMaxClient import com.pulseloop.coach.openai.OpenAIResponse import com.pulseloop.coach.openai.ResponsesClient @@ -104,6 +105,55 @@ class CoachClientResolverTest { assertTrue(openRouter.client is OpenRouterClient) } + // ── Local / self-hosted (docs/local-llm-coach.md) ─────────────────── + + @Test + fun testLocalModeReadinessIsTheBaseUrlNotTheKey() { + // The whole point of the provider: a key-less server must still enable the coach. + val ready = CoachClientResolver.resolve( + CoachProviderSettings( + providerMode = CoachProviderMode.LOCAL_OPENAI_COMPAT, + localBaseUrl = "http://192.168.1.50:11434", + localModel = "qwen3:8b", + ), + openAIKey = null, geminiKey = null, openRouterKey = null, localKey = null, + ) + assertEquals("http://192.168.1.50:11434", ready.key) + assertTrue(ready.client is LocalOpenAICompatClient) + } + + @Test + fun testLocalModeWithNoBaseUrlIsNotReadyEvenWithAKey() { + val notReady = CoachClientResolver.resolve( + CoachProviderSettings(providerMode = CoachProviderMode.LOCAL_OPENAI_COMPAT), + openAIKey = "sk-test", geminiKey = null, openRouterKey = null, localKey = "sk-local", + ) + assertNull(notReady.key) + assertTrue(notReady.client is LocalOpenAICompatClient) + } + + @Test + fun testLocalBaseUrlIsTrimmed() { + val s = CoachProviderSettings(localBaseUrl = " http://localhost:8080 ", localModel = " m ") + assertEquals("http://localhost:8080", s.resolvedLocalBaseUrl) + assertEquals("m", s.resolvedLocalModel) + } + + @Test + fun testBlankLocalModelIsNotSubstituted() { + // llama.cpp ignores `model`; inventing a slug would 404 on the servers that read it. + assertEquals("", CoachProviderSettings(localModel = " ").resolvedLocalModel) + } + + @Test + fun testLocalStructuredOutputTolerantDecode() { + for (mode in LocalStructuredOutput.entries) { + assertEquals(mode, LocalStructuredOutput.fromRaw(mode.rawValue)) + } + assertEquals(LocalStructuredOutput.OFF, LocalStructuredOutput.fromRaw(null)) + assertEquals(LocalStructuredOutput.OFF, LocalStructuredOutput.fromRaw("grammar")) + } + // ── Active model ──────────────────────────────────────────────────── @Test @@ -119,6 +169,8 @@ class CoachClientResolverTest { s.copy(providerMode = CoachProviderMode.USER_OPENROUTER_KEY), "gpt-5.4")) assertEquals("MiniMax-M2", CoachClientResolver.activeModel( s.copy(providerMode = CoachProviderMode.USER_MINIMAX_KEY), "gpt-5.4")) + assertEquals("qwen3:8b", CoachClientResolver.activeModel( + s.copy(providerMode = CoachProviderMode.LOCAL_OPENAI_COMPAT, localModel = "qwen3:8b"), "gpt-5.4")) assertEquals("gpt-5.4", CoachClientResolver.activeModel( s.copy(providerMode = CoachProviderMode.USER_OPENAI_KEY), "gpt-5.4")) assertEquals(OpenAIModel.DEFAULT.slug, CoachClientResolver.activeModel( diff --git a/app/src/test/java/com/pulseloop/coach/LocalCapabilityProbeTest.kt b/app/src/test/java/com/pulseloop/coach/LocalCapabilityProbeTest.kt new file mode 100644 index 0000000..2431ff0 --- /dev/null +++ b/app/src/test/java/com/pulseloop/coach/LocalCapabilityProbeTest.kt @@ -0,0 +1,133 @@ +package com.pulseloop.coach.local + +import com.pulseloop.coach.config.LocalStructuredOutput +import org.junit.Assert.* +import org.junit.Test + +/** + * The pure decision logic of self-discovery — model choice and how a probe result maps onto a + * setting. The network sequencing in `run` is covered by the on-device check in + * `docs/local-llm-coach.md` §5a, not here. + */ +class LocalCapabilityProbeTest { + + private fun report( + tools: LocalCapabilityProbe.Support = LocalCapabilityProbe.Support.UNKNOWN, + schema: LocalCapabilityProbe.Support = LocalCapabilityProbe.Support.UNKNOWN, + obj: LocalCapabilityProbe.Support = LocalCapabilityProbe.Support.UNKNOWN, + ) = LocalCapabilityProbe.Report( + engine = LocalCapabilityProbe.Engine.VLLM, + version = "0.27.1", + models = listOf("qwen3.8-27b-int8-w8a16-mtp"), + suggestedModel = "qwen3.8-27b-int8-w8a16-mtp", + toolCalling = tools, jsonSchema = schema, jsonObject = obj, + ) + + // ── Model choice ───────────────────────────────────────────────────── + + @Test + fun `a sole served model is chosen automatically`() { + assertEquals("only", LocalCapabilityProbe.pickModel(listOf("only"), currentModel = "")) + } + + @Test + fun `an existing choice is kept when the server still lists it`() { + assertEquals("b", LocalCapabilityProbe.pickModel(listOf("a", "b", "c"), currentModel = "b")) + } + + @Test + fun `several models and no valid current pick leaves the choice to the user`() { + // Guessing here would silently move a working setup onto a different model. + assertEquals("", LocalCapabilityProbe.pickModel(listOf("a", "b"), currentModel = "")) + assertEquals("", LocalCapabilityProbe.pickModel(listOf("a", "b"), currentModel = "gone")) + } + + @Test + fun `an empty catalog yields no model`() { + assertEquals("", LocalCapabilityProbe.pickModel(emptyList(), currentModel = "x")) + } + + // ── Capability → setting ───────────────────────────────────────────── + + @Test + fun `tool calling turns off only on an explicit refusal`() { + assertFalse(report(tools = LocalCapabilityProbe.Support.NO).suggestedToolCalling) + // An inconclusive probe must not strip the coach of its ability to read the user's data. + assertTrue(report(tools = LocalCapabilityProbe.Support.UNKNOWN).suggestedToolCalling) + assertTrue(report(tools = LocalCapabilityProbe.Support.YES).suggestedToolCalling) + } + + @Test + fun `the strongest accepted response format wins`() { + assertEquals(LocalStructuredOutput.JSON_SCHEMA, + report(schema = LocalCapabilityProbe.Support.YES).suggestedStructuredOutput) + assertEquals(LocalStructuredOutput.JSON_OBJECT, + report(schema = LocalCapabilityProbe.Support.NO, obj = LocalCapabilityProbe.Support.YES) + .suggestedStructuredOutput) + assertEquals(LocalStructuredOutput.OFF, + report(schema = LocalCapabilityProbe.Support.NO, obj = LocalCapabilityProbe.Support.NO) + .suggestedStructuredOutput) + } + + @Test + fun `an inconclusive format probe falls back to the mode that needs nothing`() { + assertEquals(LocalStructuredOutput.OFF, report().suggestedStructuredOutput) + } + + // ── Whether a suggestion may overwrite a hand-set value ────────────── + + @Test + fun `an unrun probe is not conclusive, so Detect leaves the setting alone`() { + // The state after a blank model pick or a failed baseline request. `suggestedToolCalling` + // is still true here — that default is for a first-time setup, not for a re-detect over a + // user who deliberately turned tools off for a vLLM server without --enable-auto-tool-choice. + val r = report() + assertTrue(r.suggestedToolCalling) + assertFalse(r.toolCallingConclusive) + assertEquals(LocalStructuredOutput.OFF, r.suggestedStructuredOutput) + assertFalse(r.structuredOutputConclusive) + } + + @Test + fun `a refusal is conclusive`() { + val r = report( + tools = LocalCapabilityProbe.Support.NO, + schema = LocalCapabilityProbe.Support.NO, + obj = LocalCapabilityProbe.Support.NO, + ) + assertTrue(r.toolCallingConclusive) + assertTrue(r.structuredOutputConclusive) + } + + @Test + fun `a strict-schema yes is conclusive even though JSON mode goes untested`() { + // The weaker mode is deliberately skipped once the stronger one is accepted. + val r = report(schema = LocalCapabilityProbe.Support.YES) + assertTrue(r.structuredOutputConclusive) + assertEquals(LocalStructuredOutput.JSON_SCHEMA, r.suggestedStructuredOutput) + } + + @Test + fun `a context window the server never reported suggests nothing`() { + // 0 means "not detected", which must not clear a Max tokens value the user typed. + assertEquals(0, report().suggestedMaxTokens) + } + + @Test + fun `the summary names the engine, model and both capabilities`() { + val s = report( + tools = LocalCapabilityProbe.Support.YES, + schema = LocalCapabilityProbe.Support.YES, + ).summary + assertTrue(s, s.contains("vLLM 0.27.1")) + assertTrue(s, s.contains("qwen3.8-27b-int8-w8a16-mtp")) + assertTrue(s, s.contains("tools yes")) + assertTrue(s, s.contains("strict schema")) + } + + @Test + fun `the summary says unknown rather than implying a negative`() { + val s = report(tools = LocalCapabilityProbe.Support.UNKNOWN).summary + assertTrue(s, s.contains("tools unknown")) + } +} diff --git a/app/src/test/java/com/pulseloop/coach/LocalContextBudgetTest.kt b/app/src/test/java/com/pulseloop/coach/LocalContextBudgetTest.kt new file mode 100644 index 0000000..a75d8d8 --- /dev/null +++ b/app/src/test/java/com/pulseloop/coach/LocalContextBudgetTest.kt @@ -0,0 +1,70 @@ +package com.pulseloop.coach.local + +import org.junit.Assert.* +import org.junit.Test + +/** + * The context-window → Max-tokens derivation. A context window is prompt + completion, so it can + * never be copied into `max_tokens` directly: the server checks `max_tokens` against what's LEFT + * after the prompt and rejects the request when the two together overflow. + */ +class LocalContextBudgetTest { + + private fun report(ctx: Int?) = LocalCapabilityProbe.Report( + engine = LocalCapabilityProbe.Engine.VLLM, + models = listOf("m"), suggestedModel = "m", contextWindow = ctx, + ) + + @Test + fun `a huge context is capped, not copied`() { + // The real server in docs/local-llm-coach.md reports 262144. + val r = report(262_144) + assertEquals(LocalCapabilityProbe.MAX_SUGGESTED_TOKENS, r.suggestedMaxTokens) + assertNotEquals(262_144, r.suggestedMaxTokens) + assertFalse(r.contextTooSmall) + } + + @Test + fun `a mid-size context reserves room for the prompt`() { + // 16384 - 6144 reserve = 10240 headroom, under the cap so it's used as-is. + assertEquals(10_240, report(16_384).suggestedMaxTokens) + } + + @Test + fun `prompt plus suggested budget always fits the context`() { + for (ctx in listOf(8_192, 16_384, 32_768, 131_072, 262_144)) { + val r = report(ctx) + assertTrue( + "ctx=$ctx budget=${r.suggestedMaxTokens}", + LocalCapabilityProbe.PROMPT_RESERVE_TOKENS + r.suggestedMaxTokens <= ctx, + ) + } + } + + @Test + fun `Ollama's 2048 default is flagged rather than budgeted`() { + // Smaller than the coach's own prompt — the fix is on the server, not in the app. + val r = report(2_048) + assertEquals(0, r.suggestedMaxTokens) + assertTrue(r.contextTooSmall) + } + + @Test + fun `an unreported context leaves the field blank and warns about nothing`() { + val r = report(null) + assertEquals(0, r.suggestedMaxTokens) + assertFalse(r.contextTooSmall) + } + + @Test + fun `the summary reports the context window`() { + assertTrue(report(262_144).summary.contains("262k ctx")) + assertFalse(report(null).summary.contains("ctx")) + } + + @Test + fun `token formatting stays literal for small values`() { + assertEquals("2048", LocalCapabilityProbe.formatTokens(2_048)) + assertEquals("262k", LocalCapabilityProbe.formatTokens(262_144)) + } +} diff --git a/app/src/test/java/com/pulseloop/coach/LocalEndpointTest.kt b/app/src/test/java/com/pulseloop/coach/LocalEndpointTest.kt new file mode 100644 index 0000000..17e72e4 --- /dev/null +++ b/app/src/test/java/com/pulseloop/coach/LocalEndpointTest.kt @@ -0,0 +1,92 @@ +package com.pulseloop.coach + +import com.pulseloop.coach.local.LocalEndpoint +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNull +import org.junit.Test + +/** Covers `docs/local-llm-coach.md` §3 — URL normalization and the plaintext-host rule. */ +class LocalEndpointTest { + + @Test + fun `normalizes a bare host and port to http`() { + assertEquals("http://192.168.1.50:11434", LocalEndpoint.normalize("192.168.1.50:11434")) + assertEquals("http://localhost:1234", LocalEndpoint.normalize("localhost:1234")) + } + + @Test + fun `strips a trailing slash, v1, and a pasted full endpoint`() { + val expected = "http://localhost:11434" + for (input in listOf( + "http://localhost:11434", + "http://localhost:11434/", + "http://localhost:11434/v1", + "http://localhost:11434/v1/", + "http://localhost:11434/v1/chat/completions", + )) { + assertEquals(input, expected, LocalEndpoint.normalize(input)) + } + } + + @Test + fun `keeps a reverse-proxy path prefix`() { + assertEquals("https://box.example.com/llm", + LocalEndpoint.normalize("https://box.example.com/llm/v1/")) + } + + @Test + fun `builds the chat and models urls`() { + assertEquals("http://localhost:8080/v1/chat/completions", + LocalEndpoint.chatCompletionsUrl("localhost:8080/v1")) + assertEquals("http://localhost:8080/v1/models", + LocalEndpoint.modelsUrl("localhost:8080")) + } + + @Test + fun `accepts cleartext only for private hosts`() { + for (host in listOf( + "http://localhost:11434", "http://127.0.0.1:8080", "http://10.0.2.2:1234", + "http://192.168.1.50:11434", "http://10.1.2.3:8000", "http://172.16.0.9:30000", + "http://100.64.1.2:11434", "http://mac-studio.local:1234", "http://[::1]:8080", + // Name forms that only resolve on a local network. Rejecting these told the user the + // server had to be on their LAN, which is exactly where it was. + "http://nas:11434", "http://ollama.lan:8080", "http://box.tail1234.ts.net:11434", + "http://pi.home:8000", "http://llm.internal:1234", "http://srv.home.arpa:11434", + )) { + assertNull(host, LocalEndpoint.validate(host)) + } + for (host in listOf("http://example.com:11434", "http://8.8.8.8:8080", "http://172.32.0.1:80")) { + assertEquals(host, LocalEndpoint.Problem.PUBLIC_CLEARTEXT, LocalEndpoint.validate(host)) + } + } + + @Test + fun `https is unrestricted and other schemes are rejected`() { + assertNull(LocalEndpoint.validate("https://llm.example.com")) + assertEquals(LocalEndpoint.Problem.UNSUPPORTED_SCHEME, LocalEndpoint.validate("ftp://box/llm")) + } + + @Test + fun `blank and malformed are distinguished`() { + assertEquals(LocalEndpoint.Problem.BLANK, LocalEndpoint.validate(" ")) + assertEquals(LocalEndpoint.Problem.MALFORMED, LocalEndpoint.validate("http://")) + } + + @Test + fun `a public dotted hostname is still rejected over cleartext`() { + // The single-label allowance must not leak into ordinary registered domains. + assert(!LocalEndpoint.isPrivateHost("example.com")) + assert(!LocalEndpoint.isPrivateHost("llm.example.com")) + assert(!LocalEndpoint.isPrivateHost("ollama.io")) + assert(!LocalEndpoint.isPrivateHost("notlocal.localdomain")) + } + + @Test + fun `172 private range boundaries`() { + // 172.16-172.31 are private; 172.15 and 172.32 are not. + assert(LocalEndpoint.isPrivateHost("172.16.0.1")) + assert(LocalEndpoint.isPrivateHost("172.31.255.254")) + assert(!LocalEndpoint.isPrivateHost("172.15.0.1")) + assert(!LocalEndpoint.isPrivateHost("172.32.0.1")) + } +} diff --git a/app/src/test/java/com/pulseloop/coach/LocalModelCatalogTest.kt b/app/src/test/java/com/pulseloop/coach/LocalModelCatalogTest.kt new file mode 100644 index 0000000..92bfbd3 --- /dev/null +++ b/app/src/test/java/com/pulseloop/coach/LocalModelCatalogTest.kt @@ -0,0 +1,61 @@ +package com.pulseloop.coach.local + +import com.pulseloop.coach.openai.ResponsesError +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNull +import org.junit.Assert.assertThrows +import org.junit.Test + +/** `/v1/models` parsing across the envelope shapes the engines actually return. */ +class LocalModelCatalogTest { + + @Test + fun `parses the openai list envelope, sorted and deduplicated`() { + val body = """ + {"object":"list","data":[ + {"id":"qwen3:8b","object":"model"}, + {"id":"llama3.2:3b","object":"model"}, + {"id":"qwen3:8b","object":"model"}]} + """.trimIndent() + assertEquals(listOf("llama3.2:3b", "qwen3:8b"), LocalModelCatalog.parse(body)) + } + + @Test + fun `parses a bare array, as some thin proxies return`() { + assertEquals(listOf("a", "b"), LocalModelCatalog.parse("""["b","a"]""")) + assertEquals(listOf("a"), LocalModelCatalog.parse("""[{"id":"a"}]""")) + } + + @Test + fun `an empty list is a valid answer, not an error`() { + assertEquals(emptyList(), LocalModelCatalog.parse("""{"object":"list","data":[]}""")) + } + + @Test + fun `context window is read under each engine's own field name`() { + // vLLM: max_model_len. Confirmed live against a 0.27.1 server. + assertEquals(262144, LocalModelCatalog.parseEntries( + """{"data":[{"id":"q","max_model_len":262144}]}""").first().contextWindow) + // llama.cpp: n_ctx as served, n_ctx_train as the model ceiling. + assertEquals(8192, LocalModelCatalog.parseEntries( + """{"data":[{"id":"q","n_ctx":8192,"n_ctx_train":32768}]}""").first().contextWindow) + // LM Studio: what's actually loaded wins over what the model could support. + assertEquals(4096, LocalModelCatalog.parseEntries( + """{"data":[{"id":"q","max_context_length":32768,"loaded_context_length":4096}]}""") + .first().contextWindow) + } + + @Test + fun `a missing or zero context window is null, not zero`() { + assertNull(LocalModelCatalog.parseEntries("""{"data":[{"id":"q"}]}""").first().contextWindow) + assertNull(LocalModelCatalog.parseEntries( + """{"data":[{"id":"q","max_model_len":0}]}""").first().contextWindow) + } + + @Test + fun `a response with no data array is rejected`() { + assertThrows(ResponsesError.Decoding::class.java) { + LocalModelCatalog.parse("""{"models":["a"]}""") + } + } +} diff --git a/app/src/test/java/com/pulseloop/coach/LocalOpenAICompatClientTest.kt b/app/src/test/java/com/pulseloop/coach/LocalOpenAICompatClientTest.kt new file mode 100644 index 0000000..09375ac --- /dev/null +++ b/app/src/test/java/com/pulseloop/coach/LocalOpenAICompatClientTest.kt @@ -0,0 +1,318 @@ +package com.pulseloop.coach.local + +import com.pulseloop.coach.config.LocalStructuredOutput +import com.pulseloop.coach.openai.FunctionCallOutput +import com.pulseloop.coach.openai.ResponsesError +import kotlinx.serialization.json.* +import org.junit.Assert.* +import org.junit.Test + +/** + * Request assembly + response ingestion for the self-hosted provider — no network. + * The cases here are the ones `docs/local-llm-coach.md` says a local backend gets wrong: the + * `developer` role (SGLang 400s / templates can't render it), a system turn that isn't first, + * capability toggles, and tool `arguments` returned as an object rather than a JSON string. + */ +class LocalOpenAICompatClientTest { + + private fun client( + tools: Boolean = true, + structured: LocalStructuredOutput = LocalStructuredOutput.OFF, + maxTokens: Int? = null, + model: String = "qwen3:8b", + ) = LocalOpenAICompatClient( + baseUrl = "http://192.168.1.50:11434", + model = model, + apiKey = null, + toolCallingEnabled = tools, + structuredOutput = structured, + maxOutputTokens = maxTokens, + ) + + private fun msg(role: String, content: String) = JsonObject(mapOf( + "role" to JsonPrimitive(role), + "content" to JsonPrimitive(content), + )) + + private fun request( + input: List, + tools: List = emptyList(), + previousResponseId: String? = null, + ) = JsonObject(buildMap { + put("model", JsonPrimitive("qwen3:8b")) + put("input", JsonArray(input)) + put("tools", JsonArray(tools)) + previousResponseId?.let { put("previous_response_id", JsonPrimitive(it)) } + }) + + private val functionTool = JsonObject(mapOf( + "type" to JsonPrimitive("function"), + "name" to JsonPrimitive("get_hr"), + "description" to JsonPrimitive("desc"), + "parameters" to JsonObject(mapOf("type" to JsonPrimitive("object"))), + "strict" to JsonPrimitive(true), + )) + + private fun messages(body: JsonObject) = + (body["messages"] as JsonArray).map { it.jsonObject } + + private fun role(m: JsonObject) = m["role"]!!.jsonPrimitive.content + private fun content(m: JsonObject) = m["content"]!!.jsonPrimitive.content + + // ── The developer-role fold ────────────────────────────────────────── + + @Test + fun `no message ever carries the developer role`() { + val body = client().buildRequestBody(request(listOf( + msg("system", "SYS"), msg("developer", "DEV"), msg("user", "hi"), + ))) + assertTrue(messages(body).none { role(it) == "developer" }) + } + + @Test + fun `system and developer turns merge into one leading system message`() { + val body = client().buildRequestBody(request(listOf( + msg("system", "SYS"), msg("developer", "DEV"), msg("user", "hi"), + ))) + val m = messages(body) + // Exactly one system message, and it is first — several local chat templates raise + // otherwise. + assertEquals(1, m.count { role(it) == "system" }) + assertEquals("system", role(m[0])) + assertTrue(content(m[0]).contains("SYS")) + assertTrue(content(m[0]).contains("DEV")) + // The coach_response spec joins the system block rather than trailing the conversation. + assertTrue(content(m[0]).contains("coach_response")) + assertEquals("user", role(m[1])) + assertEquals("hi", content(m[1])) + } + + @Test + fun `a system turn arriving mid-conversation is folded back into the system block`() { + val c = client() + c.buildRequestBody(request(listOf(msg("system", "SYS"), msg("user", "hi")))) + val body = c.buildRequestBody(request( + input = listOf(msg("developer", "LATE CONTEXT")), + previousResponseId = "resp_1", + )) + val m = messages(body) + assertEquals(1, m.count { role(it) == "system" }) + assertTrue(content(m[0]).contains("LATE CONTEXT")) + } + + // ── Capability toggles ─────────────────────────────────────────────── + + @Test + fun `tools are converted to the nested chat shape without strict`() { + val body = client().buildRequestBody( + request(listOf(msg("user", "hi")), tools = listOf(functionTool))) + val tools = (body["tools"] as JsonArray).map { it.jsonObject } + assertEquals(1, tools.size) + assertEquals("function", tools[0]["type"]!!.jsonPrimitive.content) + val fn = tools[0]["function"]!!.jsonObject + assertEquals("get_hr", fn["name"]!!.jsonPrimitive.content) + // `strict` is an OpenAI structured-outputs extension; local engines don't act on it. + assertNull(fn["strict"]) + } + + @Test + fun `tool calling off omits tools entirely`() { + val body = client(tools = false).buildRequestBody( + request(listOf(msg("user", "hi")), tools = listOf(functionTool))) + assertNull(body["tools"]) + } + + @Test + fun `web search is dropped - no local engine hosts one`() { + val webSearch = JsonObject(mapOf("type" to JsonPrimitive("web_search"))) + val body = client().buildRequestBody( + request(listOf(msg("user", "hi")), tools = listOf(webSearch))) + assertNull(body["tools"]) + } + + @Test + fun `structured output off sends no response_format`() { + val body = client().buildRequestBody(request(listOf(msg("user", "hi")))) + assertNull(body["response_format"]) + } + + @Test + fun `json_object and json_schema send the expected response_format`() { + val obj = client(structured = LocalStructuredOutput.JSON_OBJECT) + .buildRequestBody(request(listOf(msg("user", "hi"))))["response_format"]!!.jsonObject + assertEquals("json_object", obj["type"]!!.jsonPrimitive.content) + + val schema = client(structured = LocalStructuredOutput.JSON_SCHEMA) + .buildRequestBody(request(listOf(msg("user", "hi"))))["response_format"]!!.jsonObject + assertEquals("json_schema", schema["type"]!!.jsonPrimitive.content) + val js = schema["json_schema"]!!.jsonObject + assertEquals("coach_response", js["name"]!!.jsonPrimitive.content) + assertTrue(js["strict"]!!.jsonPrimitive.boolean) + assertNotNull(js["schema"]) + } + + @Test + fun `max_tokens is omitted unless positive`() { + assertNull(client().buildRequestBody(request(listOf(msg("user", "hi"))))["max_tokens"]) + assertEquals(2048, client(maxTokens = 2048) + .buildRequestBody(request(listOf(msg("user", "hi"))))["max_tokens"]!!.jsonPrimitive.int) + } + + @Test + fun `an empty model name is still sent - llama-cpp ignores the field`() { + val body = client(model = "").buildRequestBody(request(listOf(msg("user", "hi")))) + assertEquals("", body["model"]!!.jsonPrimitive.content) + } + + // ── Nothing a local backend would reject ───────────────────────────── + + @Test + fun `no reasoning, cache_control or provider block is ever sent`() { + val body = client().buildRequestBody( + request(listOf(msg("user", "hi")), tools = listOf(functionTool))) + assertNull(body["reasoning"]) + assertNull(body["reasoning_effort"]) + assertNull(body["provider"]) + assertFalse(body.toString().contains("cache_control")) + } + + // ── Response ingestion ─────────────────────────────────────────────── + + private fun parse(json: String) = + Json.parseToJsonElement(json).jsonObject + + @Test + fun `parses content and strips think blocks`() { + val r = client().ingestResponse(parse(""" + {"id":"chatcmpl-1","choices":[{"message":{"role":"assistant", + "content":"hmm{\"title\":\"ok\"}"}}], + "usage":{"prompt_tokens":10,"completion_tokens":4}} + """.trimIndent())) + assertEquals("chatcmpl-1", r.id) + assertEquals("{\"title\":\"ok\"}", r.outputText) + assertEquals(10, r.usage?.inputTokens) + assertEquals(4, r.usage?.outputTokens) + } + + @Test + fun `an unmatched leading close-think is stripped`() { + // R1-style distills on llama.cpp/Ollama get the OPENING tag from the chat template, so the + // completion starts mid-thought and content carries only the closing tag. Left in, the + // whole chain of thought reached the parser and burned the repair budget. + val r = client().ingestResponse(parse(""" + {"id":"c","choices":[{"message":{"role":"assistant", + "content":"the user wants a plan. let me think.{\"title\":\"ok\"}"}}]} + """.trimIndent())) + assertEquals("{\"title\":\"ok\"}", r.outputText) + } + + @Test + fun `an unterminated trailing open-think still drops its remainder`() { + val r = client().ingestResponse(parse(""" + {"id":"c","choices":[{"message":{"role":"assistant", + "content":"{\"title\":\"ok\"}and then I would"}}]} + """.trimIndent())) + assertEquals("{\"title\":\"ok\"}", r.outputText) + } + + @Test + fun `text with no think tags at all is untouched`() { + val r = client().ingestResponse(parse(""" + {"id":"c","choices":[{"message":{"role":"assistant","content":"{\"title\":\"ok\"}"}}]} + """.trimIndent())) + assertEquals("{\"title\":\"ok\"}", r.outputText) + } + + @Test + fun `tool call arguments survive both the string and object encodings`() { + val asString = client().ingestResponse(parse(""" + {"id":"a","choices":[{"message":{"role":"assistant","content":null,"tool_calls":[ + {"id":"call_1","type":"function","function":{"name":"get_hr","arguments":"{\"days\":7}"}}]}}]} + """.trimIndent())) + assertEquals("{\"days\":7}", + (asString.output.first() as FunctionCallOutput).arguments) + + // Several local tool-call parsers emit `arguments` as an object instead of a string. + val asObject = client().ingestResponse(parse(""" + {"id":"b","choices":[{"message":{"role":"assistant","content":null,"tool_calls":[ + {"id":"call_2","type":"function","function":{"name":"get_hr","arguments":{"days":7}}}]}}]} + """.trimIndent())) + assertEquals("{\"days\":7}", + (asObject.output.first() as FunctionCallOutput).arguments) + } + + @Test + fun `a body-level error is surfaced as a decoding failure`() { + for (body in listOf( + """{"error":{"message":"model not found"}}""", + """{"error":"model not found"}""", + )) { + val e = assertThrows(ResponsesError.Decoding::class.java) { + client().ingestResponse(parse(body)) + } + assertTrue(e.msg.contains("model not found")) + } + } + + @Test + fun `a non-OpenAI response says so instead of throwing a bare parse error`() { + val e = assertThrows(ResponsesError.Decoding::class.java) { + client().ingestResponse(parse("""{"response":"hello"}""")) + } + assertTrue(e.msg.contains("OpenAI-compatible")) + } + + @Test + fun `an empty choice throws EmptyOutput`() { + assertThrows(ResponsesError.EmptyOutput::class.java) { + client().ingestResponse(parse( + """{"id":"c","choices":[{"message":{"role":"assistant","content":""}}]}""")) + } + } + + @Test + fun `a reasoning model truncated mid-thought reports the token limit, not empty output`() { + // vLLM 0.27 with a reasoning parser: content null, reasoning present, finish_reason length. + val e = assertThrows(ResponsesError.Decoding::class.java) { + client().ingestResponse(parse(""" + {"id":"e","choices":[{"finish_reason":"length","message":{ + "role":"assistant","content":null,"reasoning":"The user asks"}}]} + """.trimIndent())) + } + assertTrue(e.msg, e.msg.contains("token limit")) + assertTrue(e.msg, e.msg.contains("reasoning")) + } + + @Test + fun `usage is null rather than zero when the server omits the block`() { + val r = client().ingestResponse(parse( + """{"id":"d","choices":[{"message":{"role":"assistant","content":"hi"}}]}""")) + assertNull(r.usage) + } + + // ── Continuation turns ─────────────────────────────────────────────── + + @Test + fun `a continuation replays the assistant tool_calls before the tool results`() { + val c = client() + c.buildRequestBody(request(listOf(msg("user", "hi")), tools = listOf(functionTool))) + c.ingestResponse(parse(""" + {"id":"resp_9","choices":[{"message":{"role":"assistant","content":null,"tool_calls":[ + {"id":"call_1","type":"function","function":{"name":"get_hr","arguments":"{}"}}]}}]} + """.trimIndent())) + val body = c.buildRequestBody(request( + input = listOf(JsonObject(mapOf( + "type" to JsonPrimitive("function_call_output"), + "call_id" to JsonPrimitive("call_1"), + "output" to JsonPrimitive("""{"bpm":62}"""), + ))), + tools = listOf(functionTool), + previousResponseId = "resp_9", + )) + val m = messages(body) + val assistantIdx = m.indexOfFirst { role(it) == "assistant" } + val toolIdx = m.indexOfFirst { role(it) == "tool" } + assertTrue(assistantIdx in 0 until toolIdx) + assertEquals("call_1", m[toolIdx]["tool_call_id"]!!.jsonPrimitive.content) + } +} diff --git a/docs/local-llm-coach.md b/docs/local-llm-coach.md new file mode 100644 index 0000000..ffa01b3 --- /dev/null +++ b/docs/local-llm-coach.md @@ -0,0 +1,237 @@ +# Local / self-hosted LLM support for the AI Coach + +Branch: `feat/local-llm-coach`. Adds a `LOCAL_OPENAI_COMPAT` coach provider that points at any +OpenAI-**Chat-Completions**-compatible server the user runs themselves — Ollama, llama.cpp +(`llama-server`), vLLM, SGLang, LM Studio, and anything else speaking the same wire format — +with the API key **optional**. + +## 1. What the engines actually implement + +Every popular local engine converged on the same de-facto standard: OpenAI's **Chat Completions** +(`POST {base}/v1/chat/completions`) plus `GET {base}/v1/models`. None of them implement the +OpenAI **Responses** API in the form this app speaks natively (Ollama and llama.cpp expose a +`/v1/responses` shim, but it is non-stateful and not universal), so the adapter targets Chat +Completions — exactly like `MiniMaxClient` and `OpenRouterClient` already do. + +| Engine | Default base | Auth | `tools` | `response_format` | Notes | +|---|---|---|---|---|---| +| **Ollama** | `http://localhost:11434` | none — key field "required but ignored", dummy `ollama` | yes | yes (JSON mode / schema) | `tool_choice`, `n`, `user`, `logit_bias`, image **URLs** unsupported (base64 images only) | +| **llama.cpp** `llama-server` | `http://127.0.0.1:8080` | none unless `--api-key` | yes, best with `--jinja` | `json_object` **and** `json_schema`; can't combine with `grammar` | `model` field ignored unless `--alias`/router mode | +| **vLLM** | `http://localhost:8000` | none unless `--api-key` / `VLLM_API_KEY` | only with `--enable-auto-tool-choice --tool-call-parser

` | yes (xgrammar/guided decoding) | pydantic `extra="allow"` → unknown top-level fields are **warned, not rejected** | +| **SGLang** | `http://localhost:30000` | none unless `--api-key` | yes (`tools`, `tool_choice`, `parallel_tool_calls`) | yes, plus `regex` / `ebnf` | message roles are a strict `Literal` — see §2 | +| **LM Studio** | `http://localhost:1234` | none | yes | `json_schema` only (**no** `json_object`) | also has `/v1/responses` | + +Consequence: **assume Chat Completions, assume nothing else.** Everything beyond +`model` / `messages` / `tools` / `response_format` / `max_tokens` has to be opt-in. + +## 2. The `role: developer` trap + +OpenAI's Responses API (what `CoachOrchestrator` builds) puts the per-turn context in a +`developer` message. Chat Completions predates that role, and the local engines disagree: + +- **SGLang** validates roles against a pydantic `Literal`. Current `main` has + `_GenericMessageRole = Literal["system","assistant","tool","function","developer","latest_reminder"]` + with a `_normalize_role` validator that **raises** (→ HTTP 400) for anything else. `developer` + was added later; released versions in the wild reject it outright. +- **vLLM** *rejects* it. Verified against a live vLLM **0.27.1** server: + `{"role":"developer"}` returns **HTTP 422** — `Failed to deserialize the JSON body into the + target type: messages[0]: unknown role: developer`. (Older vLLM parsed requests with pydantic + models set to `extra="allow"` and did accept the role, passing it to the Jinja chat template, + which usually has no `developer` branch either. Don't rely on the old behaviour.) +- **Ollama / llama.cpp / LM Studio** document only `system` / `user` / `assistant` / `tool`. + +Even where the server accepts the role, the *chat template* usually can't render it. So the adapter +**always folds `developer` → `system`**, unconditionally, for every local backend. This is lossless +(the content is instructions either way) and is already what `MiniMaxClient.chatRole` does. Two of +the five engines are confirmed to hard-fail without it, so this is load-bearing, not defensive. + +Second, related hazard: many local chat templates require the system message to be **first and +singular** and raise on a system turn after a user turn. `MiniMaxClient` appends +`CoachResponseSchema.promptInstruction` as a *trailing* system message; the local adapter instead +**merges all system messages into one leading system message**, so strict templates render. + +## 3. Cleartext HTTP + +A LAN box at `http://192.168.1.50:11434` has no TLS. Android blocks cleartext by default +(`usesCleartextTraffic` is false from API 28) and the app currently ships **no** +`network_security_config.xml`, so every local request would fail with `CleartextNotPermitted` +before it left the device. + +Network Security Config can't express a CIDR allowlist, so the fix is three-layer: + +1. `res/xml/network_security_config.xml` permits cleartext (cloud providers stay HTTPS because + their endpoints are hardcoded `https://` constants). +2. `LocalEndpoint.validate` refuses a plaintext `http://` URL whose host is **not** loopback, + RFC1918/CGNAT private, link-local, or a local-only name — the CIDR check the manifest can't do, + enforced where it can be. `https://` hosts are unrestricted (a self-hosted box with a cert). + Local-only names cover `*.local`, `*.lan`, `*.home`, `*.internal`, `*.home.arpa`, Tailscale's + `*.ts.net`, and single-label hosts (`http://nas:11434`) — a box addressed by the name its + router or mDNS hands out is an ordinary setup, and rejecting it told the user their server had + to be on their local network, which is where it already was. +3. The local provider sends with **redirects disabled** (`ResponsesHttp.clientFor`, + `followRedirects = false`). Layer 2 vets the URL the user typed, not the one a request ends up + on; with cleartext permitted app-wide, a `307` from the validated LAN host to a public `http://` + one would resend the health-context POST body in the clear. Cloud providers keep redirects — + their hosts are `https://` constants. + +## 4. Timeouts + +`ResponsesHttp` hardcodes a 60 s read timeout. A 30B model on CPU can spend minutes on one +tool-loop round, so the local provider needs a user-configurable read timeout (default 180 s). +`ResponsesHttp.post` gains an optional `readTimeoutSeconds` that derives a per-call client from the +shared one (`newBuilder()` keeps the connection pool and dispatcher). + +The existing retry policy already fits: a stopped local server raises `ConnectException`, which +`isProvablyUnsent` classifies as retryable, and a read timeout is (correctly) not retried. + +## 5. Capability toggles, because local ≠ uniform + +Three switches in Settings, because the same request body is fatal on one setup and required on +another: + +- **Tool calling** (default on). vLLM 400s on `tools` without `--enable-auto-tool-choice`; small + models hallucinate calls. Off ⇒ the adapter drops `tools` entirely and the coach answers from + the prompt context alone. +- **Structured output**: `off` (default) / `json_object` / `json_schema`. Off relies on the + injected `promptInstruction` plus the orchestrator's JSON-repair loop — the same path MiniMax + uses, and the only one that works everywhere. `json_schema` sends + `{type:"json_schema", json_schema:{name, strict, schema}}` from `CoachResponseSchema.schema`. + LM Studio has no `json_object`; some llama.cpp builds error when `json_schema` meets `grammar`. +- **Max output tokens** (blank = omit). Local defaults vary from unlimited to a few hundred. + Auto-detect fills this in from the server's reported **context window** — see §5b, and note it is + a derivation, never a copy. + +`reasoning` / `reasoning_effort` are **not** sent: only Ollama documents them, and vLLM/SGLang +would warn or 400. Anthropic `cache_control` and OpenRouter's `provider` block are likewise absent. + +## 5a. Self-discovery — why the toggles are probed, not looked up + +Asking the user to know whether their vLLM was started with `--enable-auto-tool-choice` is a bad +deal, and no metadata endpoint answers it: `/v1/models` describes the *model*, while the two +fields most likely to fail a turn (`tools`, `response_format`) are gated by *launch flags*. So +`LocalCapabilityProbe` sends the fields and reads the answer. + +One press of **Detect server & configure** runs: + +1. `GET /v1/models` — reachability, the model list, and the sole-model shortcut. This is the only + step whose failure is fatal; nothing after it can be trusted if the server isn't there. +2. Engine identity, best-effort, from each engine's own info route (first hit wins): + `GET /version` (vLLM), `/api/version` (Ollama), `/props` (llama.cpp), `/get_server_info` + (SGLang), `/api/v0/models` (LM Studio). Deliberately *not* `owned_by` from `/v1/models` — + vLLM says `vllm`, but Ollama says `library` and LM Studio says `organization_owner`, and any + proxy rewrites all three. Cosmetic only: it drives the summary line, never the request body. +3. A **baseline** chat request carrying no optional fields at all. +4. A chat request carrying one throwaway tool. +5. A chat request carrying a minimal `response_format: json_schema`; only if that's refused is + `json_object` tried. + +Step 3 is what makes steps 4 and 5 readable. Without it, every rejection *of the request as a +whole* — a model id `/v1/models` lists but can't actually load (LM Studio with JIT loading off, a +model pulled between the two calls), a chat route that wants auth when the listing didn't, a broken +chat template — comes back as "tools: not supported" and persists `toolCalling = false`, costing +the coach all access to the user's data while blaming the wrong thing. If the baseline is refused +or inconclusive, steps 4 and 5 are skipped and both settings are left exactly as they were. + +Classification rule: **4xx means the server refused the field** (vLLM answers `400` for a disabled +tool parser and `422` for a field its deserializer doesn't know, so the status itself carries no +extra meaning) → `NO`. A 5xx or a transport failure says nothing about the capability → `UNKNOWN`, +and the setting is **left at its default rather than switched off**, with a note explaining why. +Tool calling in particular only ever turns off on an explicit refusal — an inconclusive probe must +not silently strip the coach of its ability to read the user's data. + +**A suggestion only overwrites a stored setting when the probe reached a verdict.** Detect is also +how you refresh the model list, so it gets pressed on setups that already work, and the safe +defaults above (tools ON, structured OFF) are right for a first run and wrong for a re-detect: +a user who turned tools off by hand for a vLLM server without `--enable-auto-tool-choice` would +otherwise have them switched back on by a press meant to do something else, and every turn would +`400`. Same for Max tokens — a server that reports no context window yields `0`, which means "not +detected", not "clear what the user typed". + +Probes 3–5 use `max_tokens: 8` and a two-character prompt, and a minimal schema rather than the +coach's own (a large schema risks a rejection *about the schema* being read as "unsupported"). The +timeout is 120 s because on Ollama/LM Studio the first probe also pays for paging the model in. + +The probe never picks a model when several are served and none matches the current setting — +guessing would silently move a working setup onto a different model. + +### 5b. Max tokens is derived from the context window, never copied from it + +Every engine reports the model's context window, under its own name: + +| Engine | Route | Field | +|---|---|---| +| vLLM | `/v1/models` | `max_model_len` (262144 on the reference server) | +| llama.cpp | `/v1/models`, `/props` | `n_ctx` as served, `n_ctx_train` as the model ceiling | +| LM Studio | `/api/v0/models` | `loaded_context_length`, `max_context_length` | +| Ollama | `POST /api/show` | `model_info[".context_length"]` | +| SGLang | `/get_model_info` | context length | + +A context window is **prompt + completion**, so writing it straight into `max_tokens` is wrong in +a way that fails closed: the server checks `max_tokens` against what is *left* after the prompt and +rejects a request where the two overflow. The derivation instead reserves room for the prompt: + +``` +headroom = context − PROMPT_RESERVE_TOKENS (6144) +suggested = min(headroom, MAX_SUGGESTED_TOKENS (32768)) +headroom < 512 ⇒ leave Max tokens blank and warn +``` + +6144 is the measured coach prompt (3.1–3.3k input tokens for a plain turn on-device) doubled, so a +turn replaying history and feeding back tool results still fits. The 32768 cap keeps a 262k context +from becoming a licence for a runaway generation — a `coach_response` needs far less. + +**The warning is the more valuable half.** Ollama ships a default `num_ctx` of **2048**, smaller +than the coach's own prompt: without detection the prompt is silently truncated and the model gets +blamed. Detecting context lets Settings say so, and point at the server-side fix (`num_ctx`, +llama.cpp `-c`, vLLM `--max-model-len`) rather than at a setting in the app. + +### Measured on a real server (vLLM 0.27.1, Qwen3.8-27B-INT8) + +| Probe | Result | +|---|---| +| `GET /v1/models` | `qwen3.8-27b-int8-w8a16-mtp`, `max_model_len` 262144 | +| `GET /version` | `{"version":"0.27.1"}` → engine identified | +| `tools` | HTTP 200 → supported | +| `response_format: json_schema` | HTTP 200 → supported | +| `max_model_len` | 262144 → Max tokens suggested as 32768 (capped) | +| `role: developer` | **HTTP 422, `unknown role: developer`** | + +Also observed: with a reasoning parser enabled, vLLM returns the chain of thought in +`message.reasoning` (older builds: `reasoning_content`) and leaves `content` null until reasoning +finishes. The adapter reads neither field, so this is inert — but it means a `max_tokens` low +enough to truncate mid-reasoning yields no content at all, which the client reports as an +out-of-tokens error rather than a bare "no output". + +## 6. Changes in this repo + +| File | Change | +|---|---| +| `coach/config/CoachProviderSettings.kt` | new `LOCAL_OPENAI_COMPAT` mode + `local*` fields | +| `coach/config/CoachProviderSettingsStore.kt` | persist base URL, model, optional key, toggles | +| `coach/local/LocalEndpoint.kt` | **new** — URL normalise/validate, private-host rule | +| `coach/local/LocalOpenAICompatClient.kt` | **new** — the Chat Completions adapter | +| `coach/local/LocalModelCatalog.kt` | **new** — `GET /v1/models` for the model dropdown | +| `coach/local/LocalCapabilityProbe.kt` | **new** — engine identity + probed capabilities (§5a) | +| `coach/openai/OpenAIResponsesClient.kt` | `ResponsesHttp`: per-call read timeout + `get` | +| `coach/config/CoachClientResolver.kt` | local branch; readiness sentinel is the **base URL**, not a key | +| `coach/usage/CoachModelPricing.kt` | local models cost $0 | +| `ui/screens/SettingsSubScreens.kt` | local provider section | +| `AndroidManifest.xml`, `res/xml/network_security_config.xml` | cleartext for LAN servers | + +### Readiness gate + +`CoachClientResolver.Resolution.key` is the sentinel that `PulseLoopApp` ANDs into +`CoachFeatureFlags.coachEnabled`. For local, the key is legitimately absent, so the resolver +returns the **base URL** as the sentinel — blank URL ⇒ coach stays off, key or no key. The client +mirrors that: it throws `ResponsesError.MissingAPIKey` only when the *base URL* is missing, never +for an empty key. + +## Sources + +- [Ollama — OpenAI compatibility](https://docs.ollama.com/api/openai-compatibility) +- [llama.cpp — server README](https://github.com/ggml-org/llama.cpp/blob/master/tools/server/README.md) +- [vLLM — Tool Calling](https://docs.vllm.ai/en/stable/features/tool_calling/) +- [vLLM — `entrypoints/openai/protocol.py`](https://github.com/vllm-project/vllm/blob/v0.11.0/vllm/entrypoints/openai/protocol.py) (`OpenAIBaseModel`, `extra="allow"`) +- [SGLang — `entrypoints/openai/protocol.py`](https://github.com/sgl-project/sglang/blob/main/python/sglang/srt/entrypoints/openai/protocol.py) (`_GenericMessageRole`, `_normalize_role`) +- [SGLang — OpenAI APIs: Completions](https://docs.sglang.io/docs/basic_usage/openai_api_completions) +- [LM Studio — OpenAI compatibility endpoints](https://lmstudio.ai/docs/developer/openai-compat)