From 1f6fe2f2367ea4fe7b9fb81fb53d3e4f68b97a8e Mon Sep 17 00:00:00 2001 From: mattshax Date: Sat, 12 Sep 2026 18:08:44 +0000 Subject: [PATCH] feat(libraries): mount several indexes at once, read only beside the knowledge base The Studio built and owned its index, which suited a knowledge base and nothing else. It now mounts libraries: the knowledge base is the first, writable, and every request that does not name a library means it, so nothing that existed before has changed. Any other library is read only, a site's root-built GUFI index or one someone handed over, reached through a local tree with an optional source root. The registry lives in libraries.ts. Libraries come from the deployment (STUDIO_LIBRARIES, pinned, also a field on the deploy form) or from Settings, where an administrator adds one by path and the Studio probes the tree first: a GUFI index, the Studio's full-text tables, vectors, and whether the source is readable. Nothing is asked of GUFI and nothing is written into the index. Addressing is a library parameter on every corpus and index route, defaulting to the primary; the index functions take an index root and the corpus functions a source root, with per-root caches. A read-only library refuses upload, move, rename, delete, labels, and re-index with a 403 that says so, and the interface does not offer them. A library with no files on this host can be searched and described but not opened: the file routes answer 409 and the viewer says why. The chat tools read from the library the conversation names. On the client the Library rail header becomes the switcher when more than one library is mounted; the current library rides on every API call from one place, and on deep links as &lib=, so a shared link and a reload land in the right library. STUDIO_SECTIONS chooses which sections appear, so a site can run an index viewer with no assistant. Verified against a real second GUFI index mounted on the dev server: the probe reported each library's capabilities, listing and filename search and file reads followed the named library, the primary was untouched, and mutations on the read-only library were refused. Not yet: enrichment sidecars for read-only libraries, the Query page's scope, searching across libraries, and remote libraries through GUFI's client. Each builds on this rather than changing it. --- .env.example | 2 + README.md | 10 + deploy/workflow-cfd-studio.yaml | 23 +++ deploy/workflow.yaml | 25 +++ docs/ARCHITECTURE.md | 2 + docs/LIBRARIES.md | 82 ++++++++ server/src/chat/routes.ts | 4 +- server/src/chat/tools.ts | 29 ++- server/src/gufi.ts | 58 +++--- server/src/kb.ts | 19 +- server/src/libraries.ts | 240 ++++++++++++++++++++++++ server/src/main.ts | 2 + server/src/routes.ts | 74 ++++++-- server/src/settings.ts | 4 + server/src/tags.ts | 2 + server/src/uploads.ts | 4 + server/test/libraries.test.mjs | 141 ++++++++++++++ web/src/App.tsx | 25 ++- web/src/api.ts | 61 +++++- web/src/components/LibrariesSection.tsx | 76 ++++++++ web/src/components/LibrarySwitch.tsx | 34 ++++ web/src/components/Viewer.tsx | 6 +- web/src/config.ts | 7 + web/src/nav.ts | 13 +- web/src/styles.css | 7 + web/src/views/LibraryView.tsx | 20 +- web/src/views/SettingsView.tsx | 5 +- web/test/libraries.test.tsx | 34 ++++ web/test/nav.test.tsx | 10 + 29 files changed, 933 insertions(+), 86 deletions(-) create mode 100644 docs/LIBRARIES.md create mode 100644 server/src/libraries.ts create mode 100644 server/test/libraries.test.mjs create mode 100644 web/src/components/LibrariesSection.tsx create mode 100644 web/src/components/LibrarySwitch.tsx create mode 100644 web/test/libraries.test.tsx diff --git a/.env.example b/.env.example index 27042ad..b953d93 100644 --- a/.env.example +++ b/.env.example @@ -4,6 +4,8 @@ # --- corpus --- KB_ROOT=/data/knowledge-base # the directory this studio works over KB_LABEL='Knowledge Base' # label shown at the top of the library tree +# STUDIO_LIBRARIES='[{"id":"scratch","label":"Scratch","indexRoot":"/gufi/scratch"}]' # read-only indexes to mount beside the knowledge base (docs/LIBRARIES.md) +# STUDIO_SECTIONS='library,search,overview' # which sections to show; unset shows all # --- branding --- APP_NAME='Studio' # product name in the header and browser tab diff --git a/README.md b/README.md index 867cddc..050efac 100644 --- a/README.md +++ b/README.md @@ -63,6 +63,16 @@ request. Start to finish, including what works before an index exists: [`docs/MACOS.md`](docs/MACOS.md). +## Libraries + +The Studio can mount several indexes at once: the knowledge base it owns, +plus read-only ones such as a site's root-built GUFI index or an index +someone handed over. Users pick one in the Library rail; administrators +list them in the deploy form, set `STUDIO_LIBRARIES`, or add one in +Settings, where the tree is probed first. `STUDIO_SECTIONS` chooses which +parts of the app appear, so a site can run an index viewer with no +assistant. Details: [`docs/LIBRARIES.md`](docs/LIBRARIES.md). + ## Container build `deploy/app.def` packages the server, web build, GUFI, and the diff --git a/deploy/workflow-cfd-studio.yaml b/deploy/workflow-cfd-studio.yaml index 84badaa..b4b5fe7 100644 --- a/deploy/workflow-cfd-studio.yaml +++ b/deploy/workflow-cfd-studio.yaml @@ -336,6 +336,21 @@ permissions: type: string optional: true tooltip: One short paragraph on what this Studio is for; injected into the assistant's system prompt and editable later in Settings. + libraries: + label: Additional Libraries + type: string + optional: true + tooltip: | + Other indexes this Studio mounts read only, as a JSON list. Each entry + needs an id and an indexRoot (a GUFI tree); a label and a sourceRoot + (where the files are, when they are on this host) are optional. + sections: + label: Visible Sections + type: string + optional: true + tooltip: | + Comma-separated list of the sections to show, from chat, library, + search, query, overview, history, agents. Leave empty for all of them. build_gufi: label: Build GUFI Index Engine type: boolean @@ -901,6 +916,12 @@ jobs: if [ -n "${{ inputs.kb_settings.starter_dirs }}" ]; then echo "export KB_STARTER_DIRS='${{ inputs.kb_settings.starter_dirs }}'" >> ${SCRIPT} fi + if [ -n "${{ inputs.kb_settings.libraries }}" ]; then + printf 'export STUDIO_LIBRARIES=%q\n' '${{ inputs.kb_settings.libraries }}' >> ${SCRIPT} + fi + if [ -n "${{ inputs.kb_settings.sections }}" ]; then + printf 'export STUDIO_SECTIONS=%q\n' '${{ inputs.kb_settings.sections }}' >> ${SCRIPT} + fi if [ -n "${{ inputs.app_settings.icon_path }}" ]; then echo "export APP_ICON='${{ inputs.app_settings.icon_path }}'" >> ${SCRIPT} fi @@ -972,6 +993,8 @@ jobs: --env INDEX_BASE="${INDEX_BASE}" --env KB_LABEL="${KB_LABEL}" \ --env APP_NAME="${APP_NAME}" --env INDEX_ON_START="${INDEX_ON_START}" \ ${KB_STARTER_DIRS:+--env KB_STARTER_DIRS="${KB_STARTER_DIRS}"} \ + ${STUDIO_LIBRARIES:+--env STUDIO_LIBRARIES="${STUDIO_LIBRARIES}"} \ + ${STUDIO_SECTIONS:+--env STUDIO_SECTIONS="${STUDIO_SECTIONS}"} \ ${BANNER_TEXT:+--env BANNER_TEXT="${BANNER_TEXT}"} \ ${BANNER_COLOR:+--env BANNER_COLOR="${BANNER_COLOR}"} \ ${APP_ICON:+--env APP_ICON="${APP_ICON}"} \ diff --git a/deploy/workflow.yaml b/deploy/workflow.yaml index 257ce71..44def86 100644 --- a/deploy/workflow.yaml +++ b/deploy/workflow.yaml @@ -331,6 +331,23 @@ permissions: type: string optional: true tooltip: One short paragraph on what this Studio is for; injected into the assistant's system prompt and editable later in Settings. + libraries: + label: Additional Libraries + type: string + optional: true + tooltip: | + Other indexes this Studio mounts read only, as a JSON list. Each entry + needs an id and an indexRoot (a GUFI tree); a label and a sourceRoot + (where the files are, when they are on this host) are optional. + Example: [{"id":"scratch","label":"Scratch","indexRoot":"/gufi/scratch"}] + sections: + label: Visible Sections + type: string + optional: true + tooltip: | + Comma-separated list of the sections to show, from chat, library, + search, query, overview, history, agents. Leave empty for all of them. + "library,search,overview" makes the Studio an index viewer with no assistant. build_gufi: label: Build GUFI Index Engine type: boolean @@ -930,6 +947,12 @@ jobs: if [ -n "${{ inputs.kb_settings.starter_dirs }}" ]; then echo "export KB_STARTER_DIRS='${{ inputs.kb_settings.starter_dirs }}'" >> ${SCRIPT} fi + if [ -n "${{ inputs.kb_settings.libraries }}" ]; then + printf 'export STUDIO_LIBRARIES=%q\n' '${{ inputs.kb_settings.libraries }}' >> ${SCRIPT} + fi + if [ -n "${{ inputs.kb_settings.sections }}" ]; then + printf 'export STUDIO_SECTIONS=%q\n' '${{ inputs.kb_settings.sections }}' >> ${SCRIPT} + fi if [ -n "${{ inputs.app_settings.icon_path }}" ]; then echo "export APP_ICON='${{ inputs.app_settings.icon_path }}'" >> ${SCRIPT} fi @@ -1019,6 +1042,8 @@ jobs: --env INDEX_BASE="${INDEX_BASE}" --env KB_LABEL="${KB_LABEL}" \ --env APP_NAME="${APP_NAME}" --env INDEX_ON_START="${INDEX_ON_START}" \ ${KB_STARTER_DIRS:+--env KB_STARTER_DIRS="${KB_STARTER_DIRS}"} \ + ${STUDIO_LIBRARIES:+--env STUDIO_LIBRARIES="${STUDIO_LIBRARIES}"} \ + ${STUDIO_SECTIONS:+--env STUDIO_SECTIONS="${STUDIO_SECTIONS}"} \ ${BANNER_TEXT:+--env BANNER_TEXT="${BANNER_TEXT}"} \ ${BANNER_COLOR:+--env BANNER_COLOR="${BANNER_COLOR}"} \ ${APP_ICON:+--env APP_ICON="${APP_ICON}"} \ diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 678208a..6209b3d 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -8,6 +8,8 @@ GUFI represents a filesystem as a parallel tree of SQLite databases. `gufi_dir2i The index lives at `$INDEX_BASE/gufi//`, where `INDEX_BASE` defaults to `index/` beside the code and is usually pointed at durable storage instead. That extra level is the one `gufi_dir2index` adds for the source directory, and both the full rebuild (`indexer/reindex.sh`) and the server's incremental passes write into it; flattening it leaves a rebuilt index the server never reads. Alongside the tree, `INDEX_BASE` holds the extract cache, rendered PDF pages, settings, the credential vault, saved queries, conversations, and the embedding model, so a deployment can rebuild its working directory without losing state. The build excludes `.git`, `node_modules`, `.venv`, `__pycache__`, `dist`, `build`, caches, screenshots, and dot-directories via a `--skip-file`. +That index is the primary library. The server can mount others read only, each a GUFI tree with an optional source root, addressed by a `library` parameter that defaults to the primary; `server/src/libraries.ts` holds the registry and the probe, and `docs/LIBRARIES.md` describes the behavior. + Two properties of this design determine the rest of the system: - Queries fan out per directory. `gufi_query` walks the index breadth-first with a thread pool and runs your SQL against every `db.db` independently, so a corpus-wide query is hundreds of small local queries. diff --git a/docs/LIBRARIES.md b/docs/LIBRARIES.md new file mode 100644 index 0000000..a80b3b8 --- /dev/null +++ b/docs/LIBRARIES.md @@ -0,0 +1,82 @@ +# Libraries + +A library is an index the Studio has mounted. The knowledge base the server +builds and owns is the first library; it is the only one the Studio writes +to, and every request that does not name a library means it, so nothing +that existed before libraries has changed. Any other library is read only: +a site's index built by root on a schedule, or an index someone handed +over on a drive. + +## What a library carries + +| Field | Decides | +|---|---| +| `id`, `label` | what the user picks in the Library switcher, and which library a path belongs to | +| `indexRoot` | the GUFI tree: a directory with a `db.db` at its root | +| `sourceRoot`, optional | whether files can be opened, or only searched and described | +| writable | whether the Studio may add files, index, and enrich; the primary only | + +When a library is added the Studio probes it once and records what it +found: whether it is a GUFI index, whether the Studio's full-text tables +are present, whether vector tables are present, and whether the source is +readable. Nothing is asked of GUFI and nothing is written into the index. + +## What works on a read-only library + +Browsing, filename and metadata search, statistics, and opening files +when a source root is set. Full-text and semantic search need the +Studio's enrichment tables, which today are written only into the primary +index; on any other library those modes are skipped and search still +answers from names and metadata. Upload, move, rename, delete, labels on +files, and re-indexing are refused with a 403 that says the library is +read only, and the interface does not offer them. + +A library without a source root can be searched and described but not +opened: the file routes answer 409, and the viewer says the file is not +reachable from this host. That is the normal shape for an index of a +filesystem the Studio's machine cannot see. + +## Adding libraries + +*By the deployment:* the ACTIVATE deploy form has an Additional Libraries +field under Knowledge Base, a JSON list, and a Visible Sections field. The +same two settings are the environment variables `STUDIO_LIBRARIES` and +`STUDIO_SECTIONS` when running standalone: + +``` +STUDIO_LIBRARIES='[{"id":"scratch","label":"Scratch","indexRoot":"/gufi/scratch","sourceRoot":"/lustre/scratch"}]' +STUDIO_SECTIONS='library,search,overview' +``` + +Libraries set this way are pinned: they appear for every user and cannot +be removed from Settings. + +*By an administrator:* Settings > Libraries lists what is mounted and what +each supports, and adds one by path. The Studio probes the tree and +refuses a path that is not a GUFI index, with the reason. Added libraries +are kept in `libraries.json` under the index base. + +## Sections + +`STUDIO_SECTIONS` chooses which parts of the app appear, from `chat`, +`library`, `search`, `query`, `overview`, `history`, and `agents`. Leave it +unset for all of them. A site that wants an index viewer and nothing else +sets `library,search,overview`, and the assistant and the agents are not +rendered. Settings and Help always remain reachable. + +## Addressing + +Every request that touches the corpus or the index accepts a `library` +query or body parameter. The client adds it from the current selection, +so views never build it themselves; the primary is sent as nothing. Deep +links carry it as `&lib=` after the path, so a shared `#open=` link +and a reload land in the right library. + +## Not yet + +Enrichment for read-only libraries, in sidecar databases beside the index +on GUFI's external attach mechanism, so full-text and semantic search work +there too. The Query page, which still scopes to the primary. Searching +across several libraries in one query. A remote library reached through +GUFI's client rather than a local tree. Each is a follow-on to this +mechanism rather than a change to it. diff --git a/server/src/chat/routes.ts b/server/src/chat/routes.ts index b255404..703e2cb 100644 --- a/server/src/chat/routes.ts +++ b/server/src/chat/routes.ts @@ -1078,7 +1078,7 @@ ${ctx}` : ctx const q = String((lastUserQuestion as WireMessage | undefined)?.content ?? '').trim() if (q && q.length > 12) { try { - const seeded = await executeTool('search_kb', JSON.stringify({ query: q.slice(0, 300) }), { labelScope, userKey: pwToolKey, model: String(body.model ?? '') || null, conversationId: body.conversationId ?? null, userId: req.user?.id ?? null }) + const seeded = await executeTool('search_kb', JSON.stringify({ query: q.slice(0, 300) }), { labelScope, library: String((body as { library?: string }).library ?? '') || null, userKey: pwToolKey, model: String(body.model ?? '') || null, conversationId: body.conversationId ?? null, userId: req.user?.id ?? null }) const callId = `seed-${Date.now()}` messages.push({ role: 'assistant', content: null, tool_calls: [{ index: 0, id: callId, type: 'function', function: { name: 'search_kb', arguments: JSON.stringify({ query: q.slice(0, 300) }) } }] } as WireMessage) messages.push({ role: 'tool', tool_call_id: callId, content: seeded.result } as WireMessage) @@ -1138,7 +1138,7 @@ ${ctx}` : ctx continue } try { - const out = await executeTool(tc.function.name, tc.function.arguments, { labelScope, userKey: pwToolKey, model: String(body.model ?? '') || null, conversationId: body.conversationId ?? null, userId: req.user?.id ?? null }) + const out = await executeTool(tc.function.name, tc.function.arguments, { labelScope, library: String((body as { library?: string }).library ?? '') || null, userKey: pwToolKey, model: String(body.model ?? '') || null, conversationId: body.conversationId ?? null, userId: req.user?.id ?? null }) toolCache.set(key, out.result) outcomes[i] = out } catch (e) { diff --git a/server/src/chat/tools.ts b/server/src/chat/tools.ts index 92b6164..f6bcbeb 100644 --- a/server/src/chat/tools.ts +++ b/server/src/chat/tools.ts @@ -7,6 +7,7 @@ import path from 'node:path' import { gufiAvailable, isMissingCli, KB_ROOT, MAX_PREVIEW_BYTES, NO_CLI_MESSAGE, PROJECT_ROOT, PW_CLI } from '../config.js' import { listDir, readFileContent, KbError } from '../kb.js' import { blendHits, searchFts, searchNames, searchVector } from '../gufi.js' +import { getLibrary } from '../libraries.js' import { annotateHits } from '../tags.js' import { effectiveSettings } from '../settings.js' import { composeWorkflows } from '../workflowCompose.js' @@ -937,10 +938,22 @@ export interface ToolOutcome { // Per-invocation context (caller's own platform key for pw CLI executions) // travels via AsyncLocalStorage so nested helpers stay signature-stable and // concurrent tool calls from different users cannot cross-contaminate. -const toolContext = new AsyncLocalStorage<{ userKey: string | null; conversationId?: string | null; userId?: string | null }>() +const toolContext = new AsyncLocalStorage<{ userKey: string | null; conversationId?: string | null; userId?: string | null; library?: string | null }>() -export async function executeTool(name: string, argsJson: string, ctx?: { labelScope?: string[]; userKey?: string | null; model?: string | null; conversationId?: string | null; userId?: string | null }): Promise { - return toolContext.run({ userKey: ctx?.userKey ?? null, conversationId: ctx?.conversationId ?? null, userId: ctx?.userId ?? null }, () => executeToolImpl(name, argsJson, ctx)) +export async function executeTool(name: string, argsJson: string, ctx?: { labelScope?: string[]; userKey?: string | null; model?: string | null; library?: string | null; conversationId?: string | null; userId?: string | null }): Promise { + return toolContext.run({ userKey: ctx?.userKey ?? null, conversationId: ctx?.conversationId ?? null, userId: ctx?.userId ?? null, library: ctx?.library ?? null }, () => executeToolImpl(name, argsJson, ctx)) +} + +// The library a tool call reads from: the primary unless the conversation +// named one. A library without files on this host can be searched but its +// files cannot be opened, and the tool says so instead of failing oddly. +function toolLibrary() { + return getLibrary(toolContext.getStore()?.library ?? null) +} +function toolSourceRoot(): string { + const lib = toolLibrary() + if (!lib.sourceRoot) throw new Error(`${lib.label} has no files on this host; it can be searched but not read`) + return lib.sourceRoot } async function executeToolImpl(name: string, argsJson: string, ctx?: { labelScope?: string[]; userKey?: string | null; model?: string | null }): Promise { @@ -953,9 +966,9 @@ async function executeToolImpl(name: string, argsJson: string, ctx?: { labelScop const limit = Math.min(Number(args.limit) || 10, 25) if (gufiAvailable()) { const [fts, names, vec] = await Promise.all([ - searchFts(query, limit), - searchNames(query, 5), - searchVector(query, Math.min(limit, 8)).catch(() => []), + searchFts(query, limit, toolLibrary().indexRoot).catch(() => []), + searchNames(query, 5, toolLibrary().indexRoot), + searchVector(query, Math.min(limit, 8), toolLibrary().indexRoot).catch(() => []), ]) // A conversation-level label scope is enforced here regardless of // what the model asked for; model-requested tags narrow further @@ -979,7 +992,7 @@ async function executeToolImpl(name: string, argsJson: string, ctx?: { labelScop case 'read_kb_file': { const rel = String(args.path ?? '') const offset = Math.max(0, Number(args.offset) || 0) - const fc = await readFileContent(rel) + const fc = await readFileContent(rel, toolSourceRoot()) if (fc.content == null) { return { result: `Binary file (${fc.kind}, ${fc.size} bytes); no text available.`, summary: 'binary' } } @@ -989,7 +1002,7 @@ async function executeToolImpl(name: string, argsJson: string, ctx?: { labelScop return { result: slice + more, summary: `${slice.length} chars (${fc.source})` } } case 'list_kb_dir': { - const entries = await listDir(String(args.path ?? '')) + const entries = await listDir(String(args.path ?? ''), toolSourceRoot()) const result = entries.map(e => `${e.type === 'dir' ? 'd' : '-'} ${e.path}${e.type === 'dir' ? '/' : ` (${e.size}b)`}`).join('\n') return { result: result || '(empty)', summary: `${entries.length} entries` } } diff --git a/server/src/gufi.ts b/server/src/gufi.ts index be275e1..66fecd8 100644 --- a/server/src/gufi.ts +++ b/server/src/gufi.ts @@ -177,14 +177,15 @@ export function blendHits(fts: SearchHit[], names: SearchHit[], vec: SearchHit[] .slice(0, limit) } -export async function queryPerDir(sql: string, opts: { threads?: number; subdir?: string } = {}): Promise { - const target = opts.subdir ? path.join(GUFI_INDEX, opts.subdir) : GUFI_INDEX +export async function queryPerDir(sql: string, opts: { threads?: number; subdir?: string; indexRoot?: string } = {}): Promise { + const root = opts.indexRoot ?? GUFI_INDEX + const target = opts.subdir ? path.join(root, opts.subdir) : root const { stdout } = await run('gufi_query', ['-n', String(opts.threads ?? 8), '-d', DELIM, '-E', sql, target]) return parseRows(stdout, 1) } /** Full-text search over enriched fts5 `words` tables, joined to entries by inode. */ -export async function searchFts(queryText: string, limit = 20): Promise { +export async function searchFts(queryText: string, limit = 20, indexRoot?: string): Promise { const expr = ftsExpr(queryText) if (!expr) return [] const sql = @@ -192,9 +193,9 @@ export async function searchFts(queryText: string, limit = 20): Promise ({ - path: toKbRel(r[0]), + path: toKbRel(r[0], indexRoot), name: r[1], size: Number(r[2]) || 0, mtime: Number(r[3]) || 0, @@ -205,15 +206,15 @@ export async function searchFts(queryText: string, limit = 20): Promise { +export async function searchNames(queryText: string, limit = 20, indexRoot?: string): Promise { const where = nameWhere(parseSearchQuery(queryText)) if (where === '0') return [] const sql = `SELECT rpath(sname, sroll)||'/'||name, name, size, mtime ` + `FROM vrpentries WHERE ${where} LIMIT ${limit};` - const rows = await queryPerDir(sql) + const rows = await queryPerDir(sql, { indexRoot }) return rows.slice(0, limit).map(r => ({ - path: toKbRel(r[0]), + path: toKbRel(r[0], indexRoot), name: r[1], size: Number(r[2]) || 0, mtime: Number(r[3]) || 0, @@ -224,8 +225,8 @@ export async function searchNames(queryText: string, limit = 20): Promise() +const vecDbCache = new Map() +export function invalidateDbList(): void { dbListCache.clear(); vecDbCache.clear() } +function listIndexDbs(indexRoot: string = GUFI_INDEX): string[] { + const hit = dbListCache.get(indexRoot) + if (hit && Date.now() - hit.at < 60_000) return hit.dbs const dbs: string[] = [] const walk = (dir: string) => { let entries: fs.Dirent[] @@ -246,8 +249,8 @@ function listIndexDbs(): string[] { if (entries.some(e => e.name === 'db.db')) dbs.push(path.join(dir, 'db.db')) for (const e of entries) if (e.isDirectory()) walk(path.join(dir, e.name)) } - walk(GUFI_INDEX) - dbListCache = { dbs, at: Date.now() } + walk(indexRoot) + dbListCache.set(indexRoot, { dbs, at: Date.now() }) return dbs } @@ -268,9 +271,10 @@ export function vectorsAvailable(): boolean { * semantic search for the entire corpus. Reading sqlite_master is always * valid, so this pass is safe to run over everything. */ -async function listVectorDbs(): Promise { - if (vecDbCache && Date.now() - vecDbCache.at < 60_000) return vecDbCache.dbs - const all = listIndexDbs() +async function listVectorDbs(indexRoot: string = GUFI_INDEX): Promise { + const hit = vecDbCache.get(indexRoot) + if (hit && Date.now() - hit.at < 60_000) return hit.dbs + const all = listIndexDbs(indexRoot) if (!all.length) return [] const script = all.flatMap(db => [ `ATTACH ${q(db)} AS c;`, @@ -287,18 +291,18 @@ async function listVectorDbs(): Promise { .filter(l => l.startsWith('HASVEC')) .map(l => l.split(DELIM)[1]) .filter(Boolean) - vecDbCache = { dbs, at: Date.now() } + vecDbCache.set(indexRoot, { dbs, at: Date.now() }) return dbs } -export async function searchVector(queryText: string, limit = 10): Promise { +export async function searchVector(queryText: string, limit = 10, indexRoot: string = GUFI_INDEX): Promise { if (!vectorsAvailable()) return [] - const idx = path.resolve(GUFI_INDEX) + const idx = path.resolve(indexRoot) const script: string[] = [ `INSERT INTO temp.lembed_models(name, model) SELECT 'minilm384', lembed_model_from_file(${q(EMBED_MODEL)});`, `CREATE TABLE temp.qv AS SELECT lembed('minilm384', ${q(queryText.slice(0, 500))}) AS qe;`, ] - const vecDbs = await listVectorDbs() + const vecDbs = await listVectorDbs(indexRoot) if (!vecDbs.length) return [] for (const db of vecDbs) { const relDir = path.relative(idx, path.dirname(db)) @@ -347,16 +351,16 @@ export interface CorpusStats { byExt?: { ext: string; count: number; bytes: number }[] } -export async function corpusStats(): Promise { +export async function corpusStats(indexRoot?: string): Promise { if (!gufiAvailable()) return { available: false } const sumSql = `SELECT totfiles, totsize FROM vrsummary;` - const rows = await queryPerDir(sumSql) + const rows = await queryPerDir(sumSql, { indexRoot }) let files = 0, totalBytes = 0 for (const r of rows) { files += Number(r[0]) || 0; totalBytes += Number(r[1]) || 0 } const extSql = `SELECT lower(CASE WHEN name LIKE '%.%' THEN replace(name, rtrim(name, replace(name, '.', '')), '') ELSE '' END), ` + `count(*), sum(size) FROM vrpentries GROUP BY 1;` - const extRows = await queryPerDir(extSql) + const extRows = await queryPerDir(extSql, { indexRoot }) const agg = new Map() for (const r of extRows) { const ext = r[0] || '(none)' diff --git a/server/src/kb.ts b/server/src/kb.ts index 1fd79e2..ad2da4c 100644 --- a/server/src/kb.ts +++ b/server/src/kb.ts @@ -19,11 +19,12 @@ export class KbError extends Error { constructor(public status: number, message: string) { super(message) } } -/** Resolve a client-supplied relative path safely inside KB_ROOT. */ -export function resolveKb(rel: string): string { +/** Resolve a client-supplied relative path safely inside a corpus root, the + * knowledge base unless a library's own source root is given. */ +export function resolveKb(rel: string, base: string = KB_ROOT): string { const cleaned = rel.replace(/^\/+/, '') - const abs = path.resolve(KB_ROOT, cleaned) - if (abs !== KB_ROOT && !abs.startsWith(KB_ROOT + path.sep)) { + const abs = path.resolve(base, cleaned) + if (abs !== base && !abs.startsWith(base + path.sep)) { throw new KbError(400, 'path escapes the knowledge base root') } return abs @@ -47,8 +48,8 @@ export function excluded(name: string, isDir: boolean): boolean { return EXCLUDE_FILE_SUFFIXES.some(s => name.toLowerCase().endsWith(s)) } -export async function listDir(rel: string): Promise { - const abs = resolveKb(rel) +export async function listDir(rel: string, base: string = KB_ROOT): Promise { + const abs = resolveKb(rel, base) let dirents try { dirents = await fs.readdir(abs, { withFileTypes: true }) @@ -71,7 +72,7 @@ export async function listDir(rel: string): Promise { try { st = await fs.stat(p) } catch { continue } out.push({ name: d.name, - path: path.relative(KB_ROOT, p), + path: path.relative(base, p), type: d.isDirectory() ? 'dir' : 'file', size: st.size, mtime: Math.floor(st.mtimeMs / 1000), @@ -112,8 +113,8 @@ export interface FileContent { format?: 'markdown' | 'text' } -export async function readFileContent(rel: string): Promise { - const abs = resolveKb(rel) +export async function readFileContent(rel: string, root: string = KB_ROOT): Promise { + const abs = resolveKb(rel, root) const st = await fs.stat(abs) if (st.isDirectory()) throw new KbError(400, 'path is a directory') const kind = classify(abs, false) diff --git a/server/src/libraries.ts b/server/src/libraries.ts new file mode 100644 index 0000000..ea57bcf --- /dev/null +++ b/server/src/libraries.ts @@ -0,0 +1,240 @@ +/** + * Libraries: the indexes the Studio can mount at once. + * + * The knowledge base the server builds and owns is the first library, + * writable, and everything that exists today keeps working against it + * without naming it. Any other library is read only: a root-built site + * index, or an index someone handed over, reached through a local tree. + * Each carries what it supports, which the Studio learns by probing the + * tree once when the library is added; nothing is asked of GUFI and + * nothing is written into the index. + * + * Where they come from, in order: the primary from KB_ROOT, pinned ones + * from STUDIO_LIBRARIES (a JSON array set by the deployment), and ones an + * administrator added in Settings, kept in INDEX_BASE/libraries.json. + */ +import fs from 'node:fs' +import path from 'node:path' +import { execFile } from 'node:child_process' +import { GUFI_BIN, GUFI_INDEX, INDEX_BASE, KB_ROOT, gufiAvailable } from './config.js' +import { KbError } from './kb.js' + +export interface LibraryCaps { + /** A GUFI tree: a db.db at the root. */ + index: boolean + /** The Studio's full-text tables are present in at least one directory. */ + fullText: boolean + /** Vector tables are present in at least one directory. */ + vectors: boolean +} + +export interface Library { + id: string + label: string + indexRoot: string + /** Where the files are, when they are reachable from this host. */ + sourceRoot: string | null + /** Only the primary may add files, index, and enrich. */ + writable: boolean + primary: boolean + /** Pinned by the deployment; cannot be removed from Settings. */ + pinned: boolean + caps: LibraryCaps +} + +export interface LibraryDef { + id: string + label?: string + indexRoot: string + sourceRoot?: string | null +} + +/** The shape the client sees. Paths stay on the server. */ +export interface PublicLibrary { + id: string + label: string + primary: boolean + writable: boolean + pinned: boolean + source: boolean + caps: LibraryCaps +} + +export const PRIMARY_ID = 'kb' +const FILE = path.join(INDEX_BASE, 'libraries.json') +const ID_RE = /^[a-z0-9][a-z0-9_-]{0,39}$/ + +let added: LibraryDef[] | null = null +let probeCache = new Map() + +function loadAdded(): LibraryDef[] { + if (added) return added + try { + const raw = JSON.parse(fs.readFileSync(FILE, 'utf8')) as unknown + added = Array.isArray(raw) ? raw.filter(isDef) : [] + } catch { added = [] } + return added +} + +function saveAdded(defs: LibraryDef[]): void { + added = defs + fs.mkdirSync(path.dirname(FILE), { recursive: true }) + fs.writeFileSync(FILE, JSON.stringify(defs, null, 2)) +} + +function isDef(x: unknown): x is LibraryDef { + if (!x || typeof x !== 'object') return false + const d = x as Record + return typeof d.id === 'string' && ID_RE.test(d.id) && typeof d.indexRoot === 'string' && d.indexRoot.length > 0 +} + +function pinnedDefs(): LibraryDef[] { + const raw = process.env.STUDIO_LIBRARIES + if (!raw) return [] + try { + const arr = JSON.parse(raw) as unknown + return Array.isArray(arr) ? arr.filter(isDef) : [] + } catch { + return [] + } +} + +function sqlite(script: string, timeoutMs = 10_000): Promise { + return new Promise((resolve, reject) => { + const child = execFile(path.join(GUFI_BIN, 'gufi_sqlite3'), [], { timeout: timeoutMs, maxBuffer: 1 << 20 }, + (err, so) => (err && !so ? reject(err) : resolve(so))) + child.stdin?.end(script) + }) +} + +/** Up to `max` db.db files under a tree, breadth first, without walking everything. */ +function sampleDbs(root: string, max = 12): string[] { + const out: string[] = [] + const queue = [root] + let visited = 0 + while (queue.length && out.length < max && visited < 400) { + const dir = queue.shift()! + visited++ + let entries: fs.Dirent[] + try { entries = fs.readdirSync(dir, { withFileTypes: true }) } catch { continue } + if (entries.some(e => e.name === 'db.db')) out.push(path.join(dir, 'db.db')) + for (const e of entries) if (e.isDirectory()) queue.push(path.join(dir, e.name)) + } + return out +} + +/** + * Learn what a tree supports. A GUFI index has a db.db at its root; the + * Studio's own tables are looked for in a sample of directory databases, + * which is enough to say whether search will answer without walking the + * whole tree of a large index. + */ +export async function probeLibrary(indexRoot: string): Promise { + const abs = path.resolve(indexRoot) + const cached = probeCache.get(abs) + if (cached && Date.now() - cached.at < 300_000) return cached.caps + const caps: LibraryCaps = { index: false, fullText: false, vectors: false } + if (fs.existsSync(path.join(abs, 'db.db'))) { + caps.index = true + if (gufiAvailable()) { + for (const db of sampleDbs(abs)) { + try { + const out = await sqlite(`ATTACH '${db.replace(/'/g, "''")}' AS d;\nSELECT name FROM d.sqlite_master WHERE name IN ('words','gvec');\n`) + if (/\bwords\b/.test(out)) caps.fullText = true + if (/\bgvec\b/.test(out)) caps.vectors = true + if (caps.fullText && caps.vectors) break + } catch { /* an unreadable db says nothing */ } + } + } + } + probeCache.set(abs, { caps, at: Date.now() }) + return caps +} + +function primaryLibrary(label: string): Library { + return { + id: PRIMARY_ID, + label, + indexRoot: GUFI_INDEX, + sourceRoot: KB_ROOT, + writable: true, + primary: true, + pinned: true, + caps: probeCache.get(path.resolve(GUFI_INDEX))?.caps ?? { index: gufiAvailable(), fullText: gufiAvailable(), vectors: false }, + } +} + +function fromDef(d: LibraryDef, pinned: boolean): Library { + const abs = path.resolve(d.indexRoot) + return { + id: d.id, + label: d.label || path.basename(abs) || d.id, + indexRoot: abs, + sourceRoot: d.sourceRoot ? path.resolve(d.sourceRoot) : null, + writable: false, + primary: false, + pinned, + caps: probeCache.get(abs)?.caps ?? { index: fs.existsSync(path.join(abs, 'db.db')), fullText: false, vectors: false }, + } +} + +/** Every mounted library, primary first. `label` names the primary. */ +export function listLibraries(primaryLabel = path.basename(KB_ROOT)): Library[] { + const seen = new Set([PRIMARY_ID]) + const out: Library[] = [primaryLibrary(primaryLabel)] + for (const [defs, pinned] of [[pinnedDefs(), true], [loadAdded(), false]] as const) { + for (const d of defs) { + if (seen.has(d.id)) continue + seen.add(d.id) + out.push(fromDef(d, pinned)) + } + } + return out +} + +/** Refresh the capabilities of every library; called at startup and after a change. */ +export async function probeAll(primaryLabel?: string): Promise { + const libs = listLibraries(primaryLabel) + for (const l of libs) l.caps = await probeLibrary(l.indexRoot) + return libs +} + +export function getLibrary(id: string | undefined | null, primaryLabel?: string): Library { + const want = (id || PRIMARY_ID).trim() + const lib = listLibraries(primaryLabel).find(l => l.id === want) + if (!lib) throw new KbError(404, `no library named ${want}`) + return lib +} + +/** The guard every mutation route runs before touching files or the index. */ +export function requireWritable(lib: Library): void { + if (!lib.writable) throw new KbError(403, `${lib.label} is read only`) +} + +export async function addLibrary(def: LibraryDef): Promise { + if (!isDef(def)) throw new KbError(400, 'a library needs an id (letters, digits, dash, underscore) and an index root') + if (def.id === PRIMARY_ID || listLibraries().some(l => l.id === def.id)) throw new KbError(409, `a library named ${def.id} already exists`) + const abs = path.resolve(def.indexRoot) + if (!fs.existsSync(abs)) throw new KbError(400, `no such directory: ${def.indexRoot}`) + if (def.sourceRoot && !fs.existsSync(path.resolve(def.sourceRoot))) throw new KbError(400, `no such directory: ${def.sourceRoot}`) + const caps = await probeLibrary(abs) + if (!caps.index) throw new KbError(400, `${def.indexRoot} does not look like a GUFI index (no db.db at its root)`) + saveAdded([...loadAdded(), { id: def.id, label: def.label?.slice(0, 60) || undefined, indexRoot: abs, sourceRoot: def.sourceRoot ? path.resolve(def.sourceRoot) : null }]) + return getLibrary(def.id) +} + +export function removeLibrary(id: string): void { + const lib = getLibrary(id) + if (lib.primary || lib.pinned) throw new KbError(403, `${lib.label} is set by the deployment and cannot be removed here`) + saveAdded(loadAdded().filter(d => d.id !== id)) +} + +export function publicLibrary(l: Library): PublicLibrary { + return { id: l.id, label: l.label, primary: l.primary, writable: l.writable, pinned: l.pinned, source: !!l.sourceRoot, caps: l.caps } +} + +/** Tests replace the stored set and the probe memory between cases. */ +export function resetLibrariesForTests(): void { + added = null + probeCache = new Map() +} diff --git a/server/src/main.ts b/server/src/main.ts index 869c88f..8578db1 100644 --- a/server/src/main.ts +++ b/server/src/main.ts @@ -28,6 +28,7 @@ import { maybeAutoStart, ragEndpointRoutes } from './ragEndpoint.js' import { gatewayConfigured } from './chat/gateway.js' import { startSweepTimer } from './indexing.js' import { seedKnowledgeBase } from './seed.js' +import { probeAll } from './libraries.js' const app = Fastify({ logger: { level: 'info' } }) await app.register(fastifyMultipart) @@ -95,6 +96,7 @@ await app.register(fleetRoutes) // A brand-new deployment opens on a corpus with some shape rather than an // empty tree; only ever runs when the knowledge base has nothing in it. await seedKnowledgeBase(msg => app.log.info(msg)) +for (const l of await probeAll()) app.log.info(`library ${l.id}: index=${l.caps.index} fullText=${l.caps.fullText} vectors=${l.caps.vectors} files=${l.sourceRoot ? 'yes' : 'no'}${l.writable ? ' writable' : ''}`) startSweepTimer(msg => app.log.info(msg)) // Deploy-time full reindexes rebuild the index without overlay labels // (xattr-less filesystems); restore them once the server is up. diff --git a/server/src/routes.ts b/server/src/routes.ts index e0d6c96..0a18eba 100644 --- a/server/src/routes.ts +++ b/server/src/routes.ts @@ -1,3 +1,4 @@ +import { getLibrary, requireWritable, listLibraries, publicLibrary, addLibrary, removeLibrary, probeAll } from './libraries.js' import type { FastifyInstance } from 'fastify' import { execFile } from 'node:child_process' import path from 'node:path' @@ -113,6 +114,20 @@ export function sanitizedErrorHandler(app: FastifyInstance) { } export async function kbRoutes(app: FastifyInstance): Promise { + // Which library a request means. The primary when unnamed, so every + // existing client keeps its behavior; a name resolves to a mounted + // library or a 404. + const libOf = (req: { query?: unknown; body?: unknown }) => { + const q = (req.query ?? {}) as { library?: string } + const b = (req.body ?? {}) as { library?: string } + return getLibrary(q.library ?? b.library, effectiveSettings().kbLabel) + } + // Files are only reachable when the library has a source root on this host. + const sourceOf = (req: { query?: unknown; body?: unknown }) => { + const lib = libOf(req) + if (!lib.sourceRoot) throw new KbError(409, `${lib.label} has no files on this host; it can be searched and described but not opened`) + return lib.sourceRoot + } app.get('/healthz', async () => ({ ok: true, gufi: gufiAvailable() })) @@ -168,6 +183,8 @@ export async function kbRoutes(app: FastifyInstance): Promise { // Feature previews a deployment has switched on; the client shows // their controls only when the flag and its configuration are both present. features: { voice: { enabled: !!eff.voiceEnabled && !!eff.voiceUrl, url: eff.voiceUrl || '' } }, + libraries: listLibraries(eff.kbLabel).map(publicLibrary), + sections: eff.sections.length ? eff.sections : undefined, user, } }) @@ -226,12 +243,13 @@ export async function kbRoutes(app: FastifyInstance): Promise { app.get('/api/kb/tree', async req => { const { path: rel = '' } = req.query as { path?: string } - const entries = await listDir(rel) + const root = sourceOf(req) + const entries = await listDir(rel, root) // Own labels straight from the filesystem xattrs (one getfattr per // listing), so a just-applied label shows without waiting on the index. - const tagged = await readTagsBatch(entries.map(e => resolveKb(e.path))).catch(() => new Map()) + const tagged = await readTagsBatch(entries.map(e => resolveKb(e.path, root))).catch(() => new Map()) for (const e of entries) { - const t = tagged.get(resolveKb(e.path)) + const t = tagged.get(resolveKb(e.path, root)) if (t?.length) (e as typeof e & { tags?: string[] }).tags = t } return { path: rel, entries } @@ -239,12 +257,12 @@ export async function kbRoutes(app: FastifyInstance): Promise { app.get('/api/kb/file', async req => { const { path: rel = '' } = req.query as { path?: string } - return readFileContent(rel) + return readFileContent(rel, sourceOf(req)) }) app.get('/api/kb/download', async (req, reply) => { const { path: rel = '' } = req.query as { path?: string } - const abs = resolveKb(rel) + const abs = resolveKb(rel, sourceOf(req)) reply.header('Content-Type', mimeFor(abs)) reply.header('Content-Disposition', `attachment; filename="${path.basename(abs)}"`) return reply.send(createReadStream(abs)) @@ -253,7 +271,7 @@ export async function kbRoutes(app: FastifyInstance): Promise { // Inline serving of the actual file (images in , PDFs in