diff --git a/Cargo.lock b/Cargo.lock index baf71ae..a92e7dc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -49,6 +49,61 @@ version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" +[[package]] +name = "axum" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" +dependencies = [ + "async-trait", + "axum-core", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "serde_json", + "serde_path_to_error", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "axum-core" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "rustversion", + "sync_wrapper", + "tower-layer", + "tower-service", + "tracing", +] + [[package]] name = "base64" version = "0.22.1" @@ -454,6 +509,12 @@ version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + [[package]] name = "hyper" version = "1.10.0" @@ -468,6 +529,7 @@ dependencies = [ "http", "http-body", "httparse", + "httpdate", "itoa", "pin-project-lite", "smallvec", @@ -754,6 +816,12 @@ version = "0.4.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "616ec5685824bcc94416c6d4a7a446eea774a31efd7062c8480ba6fd06d7a6e5" +[[package]] +name = "matchit" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" + [[package]] name = "memchr" version = "2.8.1" @@ -837,6 +905,7 @@ name = "openai-endpoint" version = "0.1.0" dependencies = [ "anyhow", + "axum", "mesh-llm-plugin", "reqwest", "serde_json", @@ -1170,6 +1239,7 @@ dependencies = [ "bytes", "encoding_rs", "futures-core", + "futures-util", "h2", "http", "http-body", @@ -1191,12 +1261,14 @@ dependencies = [ "sync_wrapper", "tokio", "tokio-native-tls", + "tokio-util", "tower", "tower-http", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", + "wasm-streams", "web-sys", ] @@ -1431,6 +1503,17 @@ dependencies = [ "zmij", ] +[[package]] +name = "serde_path_to_error" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" +dependencies = [ + "itoa", + "serde", + "serde_core", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -1668,6 +1751,7 @@ dependencies = [ "tokio", "tower-layer", "tower-service", + "tracing", ] [[package]] @@ -1706,6 +1790,7 @@ version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" dependencies = [ + "log", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -1889,6 +1974,19 @@ dependencies = [ "wasmparser", ] +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "wasmparser" version = "0.244.0" diff --git a/Cargo.toml b/Cargo.toml index 2f4b3bb..27e7203 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,9 +8,10 @@ repository = "https://github.com/Mesh-LLM/openai-endpoint" [dependencies] anyhow = "1" +axum = "0.7" mesh-llm-plugin = { git = "https://github.com/Mesh-LLM/mesh-llm.git", branch = "codex/plugin-cli-commands" } +reqwest = { version = "0.12", features = ["json", "stream"] } tokio = { version = "1", features = ["full"] } [dev-dependencies] -reqwest = { version = "0.12", features = ["json"] } serde_json = "1" diff --git a/src/lib.rs b/src/lib.rs index 53bdfc9..8915ecc 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,13 +1,31 @@ -use anyhow::Result; +use anyhow::{Context, Result}; +use axum::{ + Router, + body::Body, + extract::{Request, State}, + http::{HeaderMap, StatusCode}, + response::{IntoResponse, Response}, + routing::any, +}; use mesh_llm_plugin::{ PluginMetadata, PluginRuntime, PluginStartupPolicy, capability, plugin_server_info, }; +use std::net::SocketAddr; +use std::sync::Arc; +use tokio::net::TcpListener; pub const VERSION: &str = env!("CARGO_PKG_VERSION"); const DEFAULT_BASE_URL: &str = "http://localhost:8000/v1"; const PLUGIN_ID: &str = "openai-endpoint"; +/// Caller request bodies are buffered in full before forwarding (needed to +/// inject the bearer header and recompute framing). Cap it rather than +/// buffering an unbounded body — the listener is loopback-only, so exposure +/// is limited, but an unbounded cap is still a needless memory-exhaustion +/// footgun for a fixed-cost safeguard. Comfortably above any real chat +/// completion payload (large prompts, embedded images/audio as base64). +const MAX_REQUEST_BODY_BYTES: usize = 64 * 1024 * 1024; -fn base_url() -> String { +fn upstream_base_url() -> String { std::env::var("MESH_LLM_PLUGIN_URL") .ok() .map(|value| value.trim().to_string()) @@ -15,10 +33,221 @@ fn base_url() -> String { .unwrap_or_else(|| DEFAULT_BASE_URL.to_string()) } -fn build_plugin(name: String) -> mesh_llm_plugin::SimplePlugin { - let base_url = base_url(); - let health_url = base_url.clone(); +/// Reads the bearer key from a file path passed as `--api-key-file `. +/// +/// A file, not an env var or a bare CLI value, because mesh-llm's plugin +/// config has no generic env-passthrough channel to the child process (only +/// `command`/`args` reach it), and a bare arg would show up in `ps`/ +/// `/proc//cmdline`. Absent entirely, the plugin runs unauthenticated, +/// matching its original behavior against an open upstream. +fn api_key_from_args() -> Result> { + let args: Vec = std::env::args().collect(); + let Some(index) = args.iter().position(|arg| arg == "--api-key-file") else { + return Ok(None); + }; + let path = args + .get(index + 1) + .context("--api-key-file requires a path argument")?; + let contents = + std::fs::read_to_string(path).with_context(|| format!("reading api key file at {path}"))?; + let key = contents.trim().to_string(); + if key.is_empty() { + anyhow::bail!("api key file at {path} is empty"); + } + Ok(Some(key)) +} + +struct ProxyState { + client: reqwest::Client, + upstream_base_url: String, + api_key: Option, +} + +/// Hop-by-hop / credential headers that must never ride through unchanged: +/// `host` is rebuilt from `upstream_base_url`, `authorization` from any +/// original caller is discarded so only this proxy's own key ever reaches +/// the upstream, body-framing headers are recomputed by reqwest/axum for +/// the (possibly re-encoded) forwarded body, and the rest are the +/// RFC 7230 §6.1 hop-by-hop set (connection-scoped, never meaningful two +/// hops away). +fn is_hop_by_hop_request_header(name: &str) -> bool { + matches!( + name, + "host" + | "authorization" + | "content-length" + | "transfer-encoding" + | "connection" + | "proxy-authorization" + | "proxy-authenticate" + | "keep-alive" + | "te" + | "trailer" + | "upgrade" + ) +} + +fn is_hop_by_hop_response_header(name: &str) -> bool { + matches!( + name, + "content-length" + | "transfer-encoding" + | "connection" + | "proxy-authenticate" + | "keep-alive" + | "te" + | "trailer" + | "upgrade" + ) +} + +/// Path prefix this proxy always advertises as its own base (see +/// `spawn_auth_proxy`'s `format!("http://{addr}{ADVERTISED_PATH_PREFIX}")`). +/// Callers build requests by appending endpoint suffixes (`/models`, +/// `/chat/completions`) to whatever base_url we registered, so every incoming +/// request path starts with this prefix — it must be stripped before +/// re-appending the suffix to the real `upstream_base_url` (which carries +/// its own, possibly different, base path), or the two prefixes concatenate +/// into a broken URL like `.../v1/v1/models`. +const ADVERTISED_PATH_PREFIX: &str = "/v1"; + +async fn proxy_handler(State(state): State>, request: Request) -> Response { + let (parts, body) = request.into_parts(); + let path_and_query = parts + .uri + .path_and_query() + .map(|value| value.as_str()) + .unwrap_or("/"); + let suffix = path_and_query + .strip_prefix(ADVERTISED_PATH_PREFIX) + .unwrap_or(path_and_query); + let upstream_url = format!( + "{}{}", + state.upstream_base_url.trim_end_matches('/'), + suffix + ); + + let body_bytes = match axum::body::to_bytes(body, MAX_REQUEST_BODY_BYTES).await { + Ok(bytes) => bytes, + Err(error) => { + // The failure is in reading the *caller's* body, before any + // upstream call is made — a client error, not a gateway one. + return ( + StatusCode::BAD_REQUEST, + format!("reading request body: {error}"), + ) + .into_response(); + } + }; + + let mut upstream_request = state.client.request(parts.method, &upstream_url); + for (name, value) in parts.headers.iter() { + if is_hop_by_hop_request_header(name.as_str()) { + continue; + } + upstream_request = upstream_request.header(name, value); + } + if let Some(key) = &state.api_key { + upstream_request = upstream_request.bearer_auth(key); + } + + let upstream_response = match upstream_request.body(body_bytes).send().await { + Ok(response) => response, + Err(error) => { + // Never put the raw reqwest error in the response: its Display + // output commonly includes the real upstream host/scheme, which + // is exactly what this proxy exists to keep out of anything the + // mesh (or a caller two hops away) can see. Log locally instead. + eprintln!("openai-endpoint proxy: upstream request failed: {error:#}"); + return (StatusCode::BAD_GATEWAY, "upstream request failed").into_response(); + } + }; + let status = upstream_response.status(); + let mut response_headers = HeaderMap::new(); + for (name, value) in upstream_response.headers().iter() { + if is_hop_by_hop_response_header(name.as_str()) { + continue; + } + // append, not insert: a repeated header (e.g. set-cookie) collapses + // to its last value under insert, silently dropping the rest. + response_headers.append(name.clone(), value.clone()); + } + + // Buffer the full body rather than streaming it: `content-length` is + // stripped above (it's stale once headers are rewritten in general), + // and a streamed body with no content-length forces + // `Transfer-Encoding: chunked`. mesh-llm-host-runtime's non-streaming + // JSON response path reads exactly `content_length` bytes and expects + // that to be the whole body — fed chunked framing instead, it parses the + // chunk-size/CRLF markers and the terminating `0\r\n\r\n` as part of the + // JSON payload, which fails with a "trailing characters" parse error. A + // known-size body lets the server set `content-length` correctly again. + // + // This does mean a true `stream: true` SSE completion is buffered in + // full before any of it reaches the caller rather than forwarded + // token-by-token — streaming pass-through is a known follow-up worth + // its own change once the host runtime's streaming path is verified + // against a buffered-vs-chunked plugin response. + let body_bytes = match upstream_response.bytes().await { + Ok(bytes) => bytes, + Err(error) => { + // Same rationale as the request-send error above: log locally, + // keep the real upstream out of the response. + eprintln!("openai-endpoint proxy: reading upstream response body failed: {error:#}"); + return (StatusCode::BAD_GATEWAY, "reading upstream response body").into_response(); + } + }; + + let mut response = Response::new(Body::from(body_bytes)); + *response.status_mut() = status; + *response.headers_mut() = response_headers; + response +} + +/// Binds a loopback-only reverse proxy that injects `Authorization: Bearer +/// ` (when configured) into every request before forwarding to the real +/// upstream. Returns the local address the plugin advertises to mesh-llm +/// instead of the real, possibly-credentialed upstream URL — so the bearer +/// key never appears in this process's manifest, in mesh gossip, or in any +/// other pool member's view of "where this model is served." +async fn spawn_auth_proxy( + upstream_base_url: String, + api_key: Option, +) -> Result { + let state = Arc::new(ProxyState { + // Unbounded by default: a stalled upstream (wedged process, dropped + // connection with no RST) would otherwise hold the proxying task + // open indefinitely. `read_timeout` is a sliding per-read window + // (resets on every successful chunk), not a total-duration cap, so + // it won't cut off a slow-but-progressing streaming completion — + // only one that's gone genuinely silent. + client: reqwest::Client::builder() + .connect_timeout(std::time::Duration::from_secs(10)) + .read_timeout(std::time::Duration::from_secs(300)) + .build() + .context("building auth-proxy upstream client")?, + upstream_base_url, + api_key, + }); + let app = Router::new().fallback(any(proxy_handler)).with_state(state); + let listener = TcpListener::bind("127.0.0.1:0") + .await + .context("binding local auth-proxy listener")?; + let addr = listener.local_addr().context("reading local proxy addr")?; + tokio::spawn(async move { + if let Err(error) = axum::serve(listener, app).await { + eprintln!("openai-endpoint auth proxy stopped: {error:#}"); + } + }); + Ok(addr) +} + +fn build_plugin( + name: String, + advertised_base_url: String, + upstream_base_url: String, +) -> mesh_llm_plugin::SimplePlugin { mesh_llm_plugin::plugin! { metadata: PluginMetadata::new( name, @@ -27,10 +256,15 @@ fn build_plugin(name: String) -> mesh_llm_plugin::SimplePlugin { "mesh-openai-endpoint", VERSION, "OpenAI-Compatible Endpoint Plugin", - "Routes inference to an external OpenAI-compatible server (vLLM, TGI, Ollama, etc.).", + "Routes inference to an external OpenAI-compatible server (vLLM, TGI, \ + Ollama, LM Studio, etc.), optionally authenticating to it with a \ + bearer API key injected by a local proxy.", Some( - "Set MESH_LLM_PLUGIN_URL to point at any server \ - that speaks the OpenAI /v1/chat/completions API.", + "Set MESH_LLM_PLUGIN_URL to point at any server that speaks the \ + OpenAI /v1/chat/completions API. Pass --api-key-file to a \ + file containing a bearer token to authenticate to it; the token \ + is injected by a local loopback proxy and never appears in this \ + plugin's advertised endpoint address.", ), ), ), @@ -40,18 +274,32 @@ fn build_plugin(name: String) -> mesh_llm_plugin::SimplePlugin { capability("endpoint:inference/openai_compatible"), ], inference: [ - mesh_llm_plugin::inference::openai_http(PLUGIN_ID, base_url.clone()) + mesh_llm_plugin::inference::openai_http(PLUGIN_ID, advertised_base_url.clone()) .managed_by_plugin(false), ], health: move |_context| { - let health_url = health_url.clone(); - Box::pin(async move { Ok(format!("base_url={health_url}")) }) + let upstream_base_url = upstream_base_url.clone(); + Box::pin(async move { Ok(format!("upstream={upstream_base_url}")) }) }, } } +/// The base URL this plugin advertises to mesh-llm for a proxy bound at +/// `proxy_addr` — always loopback, since `proxy_addr` always is (see +/// `spawn_auth_proxy`, which only ever binds `127.0.0.1`). Extracted so +/// tests can exercise the actual derivation `run_plugin` uses, rather than +/// only the (unrelated) claim that `build_plugin` echoes back whatever +/// string it's given. +fn advertised_base_url_for_proxy(proxy_addr: SocketAddr) -> String { + format!("http://{proxy_addr}{ADVERTISED_PATH_PREFIX}") +} + async fn run_plugin(name: String) -> Result<()> { - PluginRuntime::run(build_plugin(name)).await + let upstream_base_url = upstream_base_url(); + let api_key = api_key_from_args()?; + let proxy_addr = spawn_auth_proxy(upstream_base_url.clone(), api_key).await?; + let advertised_base_url = advertised_base_url_for_proxy(proxy_addr); + PluginRuntime::run(build_plugin(name, advertised_base_url, upstream_base_url)).await } pub fn run_main() -> i32 { @@ -73,15 +321,23 @@ pub fn run_main() -> i32 { #[cfg(test)] mod tests { use super::*; - use anyhow::{Context, bail}; + use anyhow::bail; use mesh_llm_plugin::Plugin; use serde_json::{Value, json}; use std::time::Duration; + fn loopback_manifest(advertised_base_url: &str) -> mesh_llm_plugin::proto::PluginManifest { + let plugin = build_plugin( + PLUGIN_ID.to_string(), + advertised_base_url.to_string(), + "http://real-upstream.example:1234/v1".to_string(), + ); + plugin.manifest().expect("manifest") + } + #[test] fn manifest_declares_external_openai_endpoint() { - let plugin = build_plugin(PLUGIN_ID.to_string()); - let manifest = plugin.manifest().expect("manifest"); + let manifest = loopback_manifest("http://127.0.0.1:59123/v1"); assert!( manifest @@ -94,21 +350,164 @@ mod tests { assert!(!manifest.endpoints[0].managed_by_plugin); } + /// Permanent regression test for the property this fork exists to buy: + /// whatever the real (possibly credentialed) upstream URL is, the + /// manifest this plugin advertises to the mesh pool must always be a + /// loopback address — never the real upstream, and never anything that + /// could leak the API key's host/scheme to other pool members. + #[test] + fn advertised_endpoint_is_always_loopback_never_real_upstream() { + for real_upstream in [ + "https://100.114.85.122:1234/v1", + "https://lmstudio.tail637714.ts.net/v1", + "http://localhost:8000/v1", + ] { + let plugin = build_plugin( + PLUGIN_ID.to_string(), + "http://127.0.0.1:59123/v1".to_string(), + real_upstream.to_string(), + ); + let manifest = plugin.manifest().expect("manifest"); + let address = manifest.endpoints[0] + .address + .as_deref() + .expect("endpoint address"); + assert!( + address.starts_with("http://127.0.0.1:"), + "advertised endpoint {address} must be loopback, not derived from \ + real upstream {real_upstream}" + ); + assert_ne!(address, real_upstream); + } + } + + #[tokio::test] + async fn proxy_injects_bearer_key_and_strips_caller_authorization() -> Result<()> { + let upstream = axum_test_upstream().await?; + // Real servers (e.g. LM Studio) are commonly configured with a base + // URL that itself ends in "/v1" — exactly like this plugin's own + // advertised base. Naively concatenating the two produces + // "/v1/v1/models". Use a real "/v1" upstream base and an EXACT route + // (no fallback) so a reintroduced double-prefix bug 404s instead of + // silently passing. + let proxy_addr = spawn_auth_proxy( + format!("http://{}/v1", upstream.addr), + Some("secret-lmstudio-key".to_string()), + ) + .await?; + + let client = reqwest::Client::new(); + let response = client + .get(format!("http://{proxy_addr}/v1/models")) + .header("authorization", "Bearer caller-supplied-should-be-dropped") + .send() + .await?; + assert_eq!( + response.status(), + StatusCode::OK, + "must reach the upstream's exact /v1/models route, not /v1/v1/models" + ); + let seen_auth = response.text().await?; + assert_eq!(seen_auth, "Bearer secret-lmstudio-key"); + Ok(()) + } + + /// Unlike `advertised_endpoint_is_always_loopback_never_real_upstream` + /// (which only proves `build_plugin` echoes back whatever + /// `advertised_base_url` it's handed), this exercises the actual + /// derivation `run_plugin` uses — a real bound proxy address run + /// through `advertised_base_url_for_proxy` — so a change to that + /// derivation that stopped producing a loopback URL would be caught + /// here. + #[tokio::test] + async fn run_plugin_derives_a_loopback_advertised_url_from_the_real_proxy_addr() -> Result<()> + { + let proxy_addr = spawn_auth_proxy("http://127.0.0.1:1".to_string(), None).await?; + let advertised = advertised_base_url_for_proxy(proxy_addr); + assert_eq!(advertised, format!("http://{proxy_addr}/v1")); + assert!( + advertised.starts_with("http://127.0.0.1:"), + "advertised URL {advertised} must be loopback" + ); + Ok(()) + } + + #[tokio::test] + async fn proxy_omits_authorization_when_no_api_key_configured() -> Result<()> { + let upstream = axum_test_upstream().await?; + let proxy_addr = spawn_auth_proxy(format!("http://{}/v1", upstream.addr), None).await?; + + let client = reqwest::Client::new(); + let response = client + .get(format!("http://{proxy_addr}/v1/models")) + .send() + .await?; + assert_eq!(response.status(), StatusCode::OK); + let seen_auth = response.text().await?; + assert_eq!( + seen_auth, "", + "no api_key configured means no Authorization header should reach upstream" + ); + Ok(()) + } + + #[tokio::test] + async fn proxy_returns_bad_gateway_when_upstream_unreachable() -> Result<()> { + // Port 1 is reserved/unroutable, so this fails immediately without a + // real dependency on "nothing listens there" being stable elsewhere. + let proxy_addr = spawn_auth_proxy("http://127.0.0.1:1".to_string(), None).await?; + let client = reqwest::Client::new(); + let response = client + .get(format!("http://{proxy_addr}/v1/models")) + .send() + .await?; + assert_eq!(response.status(), StatusCode::BAD_GATEWAY); + Ok(()) + } + + struct TestUpstream { + addr: SocketAddr, + } + + /// Minimal upstream double that echoes back the `Authorization` header + /// it received, so tests can assert on exactly what the proxy sent. + /// Routes ONLY `/v1/models` — deliberately no wildcard fallback, so a + /// wrong forwarded path (e.g. a reintroduced "/v1/v1/models" double + /// prefix) 404s instead of silently matching anyway. + async fn axum_test_upstream() -> Result { + async fn echo_auth(headers: HeaderMap) -> String { + headers + .get("authorization") + .and_then(|value| value.to_str().ok()) + .unwrap_or_default() + .to_string() + } + let app = Router::new().route("/v1/models", axum::routing::get(echo_auth)); + let listener = TcpListener::bind("127.0.0.1:0").await?; + let addr = listener.local_addr()?; + tokio::spawn(async move { + let _ = axum::serve(listener, app).await; + }); + Ok(TestUpstream { addr }) + } + #[tokio::test] async fn e2e_llama_server_answers_openai_requests() -> Result<()> { if std::env::var_os("OPENAI_ENDPOINT_E2E").is_none() { return Ok(()); } - let base_url = std::env::var("MESH_LLM_PLUGIN_URL").unwrap_or_else(|_| base_url()); - let plugin = build_plugin(PLUGIN_ID.to_string()); - let manifest = plugin.manifest().context("plugin manifest")?; + let upstream_base_url = + std::env::var("MESH_LLM_PLUGIN_URL").unwrap_or_else(|_| upstream_base_url()); + let proxy_addr = spawn_auth_proxy(upstream_base_url.clone(), api_key_from_args()?).await?; + let base_url = format!("http://{proxy_addr}/v1"); + + let manifest = loopback_manifest(&base_url); let endpoint = manifest .endpoints .iter() .find(|endpoint| endpoint.endpoint_id == PLUGIN_ID) .context("openai endpoint manifest entry")?; - assert_eq!(endpoint.address.as_deref(), Some(base_url.as_str())); assert!(!endpoint.managed_by_plugin);