diff --git a/.env.example b/.env.example index 27042ad..b953d93 100644 --- a/.env.example +++ b/.env.example @@ -4,6 +4,8 @@ # --- corpus --- KB_ROOT=/data/knowledge-base # the directory this studio works over KB_LABEL='Knowledge Base' # label shown at the top of the library tree +# STUDIO_LIBRARIES='[{"id":"scratch","label":"Scratch","indexRoot":"/gufi/scratch"}]' # read-only indexes to mount beside the knowledge base (docs/LIBRARIES.md) +# STUDIO_SECTIONS='library,search,overview' # which sections to show; unset shows all # --- branding --- APP_NAME='Studio' # product name in the header and browser tab diff --git a/README.md b/README.md index 867cddc..050efac 100644 --- a/README.md +++ b/README.md @@ -63,6 +63,16 @@ request. Start to finish, including what works before an index exists: [`docs/MACOS.md`](docs/MACOS.md). +## Libraries + +The Studio can mount several indexes at once: the knowledge base it owns, +plus read-only ones such as a site's root-built GUFI index or an index +someone handed over. Users pick one in the Library rail; administrators +list them in the deploy form, set `STUDIO_LIBRARIES`, or add one in +Settings, where the tree is probed first. `STUDIO_SECTIONS` chooses which +parts of the app appear, so a site can run an index viewer with no +assistant. Details: [`docs/LIBRARIES.md`](docs/LIBRARIES.md). + ## Container build `deploy/app.def` packages the server, web build, GUFI, and the diff --git a/deploy/workflow-cfd-studio.yaml b/deploy/workflow-cfd-studio.yaml index 84badaa..b4b5fe7 100644 --- a/deploy/workflow-cfd-studio.yaml +++ b/deploy/workflow-cfd-studio.yaml @@ -336,6 +336,21 @@ permissions: type: string optional: true tooltip: One short paragraph on what this Studio is for; injected into the assistant's system prompt and editable later in Settings. + libraries: + label: Additional Libraries + type: string + optional: true + tooltip: | + Other indexes this Studio mounts read only, as a JSON list. Each entry + needs an id and an indexRoot (a GUFI tree); a label and a sourceRoot + (where the files are, when they are on this host) are optional. + sections: + label: Visible Sections + type: string + optional: true + tooltip: | + Comma-separated list of the sections to show, from chat, library, + search, query, overview, history, agents. Leave empty for all of them. build_gufi: label: Build GUFI Index Engine type: boolean @@ -901,6 +916,12 @@ jobs: if [ -n "${{ inputs.kb_settings.starter_dirs }}" ]; then echo "export KB_STARTER_DIRS='${{ inputs.kb_settings.starter_dirs }}'" >> ${SCRIPT} fi + if [ -n "${{ inputs.kb_settings.libraries }}" ]; then + printf 'export STUDIO_LIBRARIES=%q\n' '${{ inputs.kb_settings.libraries }}' >> ${SCRIPT} + fi + if [ -n "${{ inputs.kb_settings.sections }}" ]; then + printf 'export STUDIO_SECTIONS=%q\n' '${{ inputs.kb_settings.sections }}' >> ${SCRIPT} + fi if [ -n "${{ inputs.app_settings.icon_path }}" ]; then echo "export APP_ICON='${{ inputs.app_settings.icon_path }}'" >> ${SCRIPT} fi @@ -972,6 +993,8 @@ jobs: --env INDEX_BASE="${INDEX_BASE}" --env KB_LABEL="${KB_LABEL}" \ --env APP_NAME="${APP_NAME}" --env INDEX_ON_START="${INDEX_ON_START}" \ ${KB_STARTER_DIRS:+--env KB_STARTER_DIRS="${KB_STARTER_DIRS}"} \ + ${STUDIO_LIBRARIES:+--env STUDIO_LIBRARIES="${STUDIO_LIBRARIES}"} \ + ${STUDIO_SECTIONS:+--env STUDIO_SECTIONS="${STUDIO_SECTIONS}"} \ ${BANNER_TEXT:+--env BANNER_TEXT="${BANNER_TEXT}"} \ ${BANNER_COLOR:+--env BANNER_COLOR="${BANNER_COLOR}"} \ ${APP_ICON:+--env APP_ICON="${APP_ICON}"} \ diff --git a/deploy/workflow.yaml b/deploy/workflow.yaml index 257ce71..44def86 100644 --- a/deploy/workflow.yaml +++ b/deploy/workflow.yaml @@ -331,6 +331,23 @@ permissions: type: string optional: true tooltip: One short paragraph on what this Studio is for; injected into the assistant's system prompt and editable later in Settings. + libraries: + label: Additional Libraries + type: string + optional: true + tooltip: | + Other indexes this Studio mounts read only, as a JSON list. Each entry + needs an id and an indexRoot (a GUFI tree); a label and a sourceRoot + (where the files are, when they are on this host) are optional. + Example: [{"id":"scratch","label":"Scratch","indexRoot":"/gufi/scratch"}] + sections: + label: Visible Sections + type: string + optional: true + tooltip: | + Comma-separated list of the sections to show, from chat, library, + search, query, overview, history, agents. Leave empty for all of them. + "library,search,overview" makes the Studio an index viewer with no assistant. build_gufi: label: Build GUFI Index Engine type: boolean @@ -930,6 +947,12 @@ jobs: if [ -n "${{ inputs.kb_settings.starter_dirs }}" ]; then echo "export KB_STARTER_DIRS='${{ inputs.kb_settings.starter_dirs }}'" >> ${SCRIPT} fi + if [ -n "${{ inputs.kb_settings.libraries }}" ]; then + printf 'export STUDIO_LIBRARIES=%q\n' '${{ inputs.kb_settings.libraries }}' >> ${SCRIPT} + fi + if [ -n "${{ inputs.kb_settings.sections }}" ]; then + printf 'export STUDIO_SECTIONS=%q\n' '${{ inputs.kb_settings.sections }}' >> ${SCRIPT} + fi if [ -n "${{ inputs.app_settings.icon_path }}" ]; then echo "export APP_ICON='${{ inputs.app_settings.icon_path }}'" >> ${SCRIPT} fi @@ -1019,6 +1042,8 @@ jobs: --env INDEX_BASE="${INDEX_BASE}" --env KB_LABEL="${KB_LABEL}" \ --env APP_NAME="${APP_NAME}" --env INDEX_ON_START="${INDEX_ON_START}" \ ${KB_STARTER_DIRS:+--env KB_STARTER_DIRS="${KB_STARTER_DIRS}"} \ + ${STUDIO_LIBRARIES:+--env STUDIO_LIBRARIES="${STUDIO_LIBRARIES}"} \ + ${STUDIO_SECTIONS:+--env STUDIO_SECTIONS="${STUDIO_SECTIONS}"} \ ${BANNER_TEXT:+--env BANNER_TEXT="${BANNER_TEXT}"} \ ${BANNER_COLOR:+--env BANNER_COLOR="${BANNER_COLOR}"} \ ${APP_ICON:+--env APP_ICON="${APP_ICON}"} \ diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 678208a..6209b3d 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -8,6 +8,8 @@ GUFI represents a filesystem as a parallel tree of SQLite databases. `gufi_dir2i The index lives at `$INDEX_BASE/gufi//`, where `INDEX_BASE` defaults to `index/` beside the code and is usually pointed at durable storage instead. That extra level is the one `gufi_dir2index` adds for the source directory, and both the full rebuild (`indexer/reindex.sh`) and the server's incremental passes write into it; flattening it leaves a rebuilt index the server never reads. Alongside the tree, `INDEX_BASE` holds the extract cache, rendered PDF pages, settings, the credential vault, saved queries, conversations, and the embedding model, so a deployment can rebuild its working directory without losing state. The build excludes `.git`, `node_modules`, `.venv`, `__pycache__`, `dist`, `build`, caches, screenshots, and dot-directories via a `--skip-file`. +That index is the primary library. The server can mount others read only, each a GUFI tree with an optional source root, addressed by a `library` parameter that defaults to the primary; `server/src/libraries.ts` holds the registry and the probe, and `docs/LIBRARIES.md` describes the behavior. + Two properties of this design determine the rest of the system: - Queries fan out per directory. `gufi_query` walks the index breadth-first with a thread pool and runs your SQL against every `db.db` independently, so a corpus-wide query is hundreds of small local queries. diff --git a/docs/LIBRARIES.md b/docs/LIBRARIES.md new file mode 100644 index 0000000..a80b3b8 --- /dev/null +++ b/docs/LIBRARIES.md @@ -0,0 +1,82 @@ +# Libraries + +A library is an index the Studio has mounted. The knowledge base the server +builds and owns is the first library; it is the only one the Studio writes +to, and every request that does not name a library means it, so nothing +that existed before libraries has changed. Any other library is read only: +a site's index built by root on a schedule, or an index someone handed +over on a drive. + +## What a library carries + +| Field | Decides | +|---|---| +| `id`, `label` | what the user picks in the Library switcher, and which library a path belongs to | +| `indexRoot` | the GUFI tree: a directory with a `db.db` at its root | +| `sourceRoot`, optional | whether files can be opened, or only searched and described | +| writable | whether the Studio may add files, index, and enrich; the primary only | + +When a library is added the Studio probes it once and records what it +found: whether it is a GUFI index, whether the Studio's full-text tables +are present, whether vector tables are present, and whether the source is +readable. Nothing is asked of GUFI and nothing is written into the index. + +## What works on a read-only library + +Browsing, filename and metadata search, statistics, and opening files +when a source root is set. Full-text and semantic search need the +Studio's enrichment tables, which today are written only into the primary +index; on any other library those modes are skipped and search still +answers from names and metadata. Upload, move, rename, delete, labels on +files, and re-indexing are refused with a 403 that says the library is +read only, and the interface does not offer them. + +A library without a source root can be searched and described but not +opened: the file routes answer 409, and the viewer says the file is not +reachable from this host. That is the normal shape for an index of a +filesystem the Studio's machine cannot see. + +## Adding libraries + +*By the deployment:* the ACTIVATE deploy form has an Additional Libraries +field under Knowledge Base, a JSON list, and a Visible Sections field. The +same two settings are the environment variables `STUDIO_LIBRARIES` and +`STUDIO_SECTIONS` when running standalone: + +``` +STUDIO_LIBRARIES='[{"id":"scratch","label":"Scratch","indexRoot":"/gufi/scratch","sourceRoot":"/lustre/scratch"}]' +STUDIO_SECTIONS='library,search,overview' +``` + +Libraries set this way are pinned: they appear for every user and cannot +be removed from Settings. + +*By an administrator:* Settings > Libraries lists what is mounted and what +each supports, and adds one by path. The Studio probes the tree and +refuses a path that is not a GUFI index, with the reason. Added libraries +are kept in `libraries.json` under the index base. + +## Sections + +`STUDIO_SECTIONS` chooses which parts of the app appear, from `chat`, +`library`, `search`, `query`, `overview`, `history`, and `agents`. Leave it +unset for all of them. A site that wants an index viewer and nothing else +sets `library,search,overview`, and the assistant and the agents are not +rendered. Settings and Help always remain reachable. + +## Addressing + +Every request that touches the corpus or the index accepts a `library` +query or body parameter. The client adds it from the current selection, +so views never build it themselves; the primary is sent as nothing. Deep +links carry it as `&lib=` after the path, so a shared `#open=` link +and a reload land in the right library. + +## Not yet + +Enrichment for read-only libraries, in sidecar databases beside the index +on GUFI's external attach mechanism, so full-text and semantic search work +there too. The Query page, which still scopes to the primary. Searching +across several libraries in one query. A remote library reached through +GUFI's client rather than a local tree. Each is a follow-on to this +mechanism rather than a change to it. diff --git a/server/src/chat/routes.ts b/server/src/chat/routes.ts index b255404..703e2cb 100644 --- a/server/src/chat/routes.ts +++ b/server/src/chat/routes.ts @@ -1078,7 +1078,7 @@ ${ctx}` : ctx const q = String((lastUserQuestion as WireMessage | undefined)?.content ?? '').trim() if (q && q.length > 12) { try { - const seeded = await executeTool('search_kb', JSON.stringify({ query: q.slice(0, 300) }), { labelScope, userKey: pwToolKey, model: String(body.model ?? '') || null, conversationId: body.conversationId ?? null, userId: req.user?.id ?? null }) + const seeded = await executeTool('search_kb', JSON.stringify({ query: q.slice(0, 300) }), { labelScope, library: String((body as { library?: string }).library ?? '') || null, userKey: pwToolKey, model: String(body.model ?? '') || null, conversationId: body.conversationId ?? null, userId: req.user?.id ?? null }) const callId = `seed-${Date.now()}` messages.push({ role: 'assistant', content: null, tool_calls: [{ index: 0, id: callId, type: 'function', function: { name: 'search_kb', arguments: JSON.stringify({ query: q.slice(0, 300) }) } }] } as WireMessage) messages.push({ role: 'tool', tool_call_id: callId, content: seeded.result } as WireMessage) @@ -1138,7 +1138,7 @@ ${ctx}` : ctx continue } try { - const out = await executeTool(tc.function.name, tc.function.arguments, { labelScope, userKey: pwToolKey, model: String(body.model ?? '') || null, conversationId: body.conversationId ?? null, userId: req.user?.id ?? null }) + const out = await executeTool(tc.function.name, tc.function.arguments, { labelScope, library: String((body as { library?: string }).library ?? '') || null, userKey: pwToolKey, model: String(body.model ?? '') || null, conversationId: body.conversationId ?? null, userId: req.user?.id ?? null }) toolCache.set(key, out.result) outcomes[i] = out } catch (e) { diff --git a/server/src/chat/tools.ts b/server/src/chat/tools.ts index 92b6164..f6bcbeb 100644 --- a/server/src/chat/tools.ts +++ b/server/src/chat/tools.ts @@ -7,6 +7,7 @@ import path from 'node:path' import { gufiAvailable, isMissingCli, KB_ROOT, MAX_PREVIEW_BYTES, NO_CLI_MESSAGE, PROJECT_ROOT, PW_CLI } from '../config.js' import { listDir, readFileContent, KbError } from '../kb.js' import { blendHits, searchFts, searchNames, searchVector } from '../gufi.js' +import { getLibrary } from '../libraries.js' import { annotateHits } from '../tags.js' import { effectiveSettings } from '../settings.js' import { composeWorkflows } from '../workflowCompose.js' @@ -937,10 +938,22 @@ export interface ToolOutcome { // Per-invocation context (caller's own platform key for pw CLI executions) // travels via AsyncLocalStorage so nested helpers stay signature-stable and // concurrent tool calls from different users cannot cross-contaminate. -const toolContext = new AsyncLocalStorage<{ userKey: string | null; conversationId?: string | null; userId?: string | null }>() +const toolContext = new AsyncLocalStorage<{ userKey: string | null; conversationId?: string | null; userId?: string | null; library?: string | null }>() -export async function executeTool(name: string, argsJson: string, ctx?: { labelScope?: string[]; userKey?: string | null; model?: string | null; conversationId?: string | null; userId?: string | null }): Promise { - return toolContext.run({ userKey: ctx?.userKey ?? null, conversationId: ctx?.conversationId ?? null, userId: ctx?.userId ?? null }, () => executeToolImpl(name, argsJson, ctx)) +export async function executeTool(name: string, argsJson: string, ctx?: { labelScope?: string[]; userKey?: string | null; model?: string | null; library?: string | null; conversationId?: string | null; userId?: string | null }): Promise { + return toolContext.run({ userKey: ctx?.userKey ?? null, conversationId: ctx?.conversationId ?? null, userId: ctx?.userId ?? null, library: ctx?.library ?? null }, () => executeToolImpl(name, argsJson, ctx)) +} + +// The library a tool call reads from: the primary unless the conversation +// named one. A library without files on this host can be searched but its +// files cannot be opened, and the tool says so instead of failing oddly. +function toolLibrary() { + return getLibrary(toolContext.getStore()?.library ?? null) +} +function toolSourceRoot(): string { + const lib = toolLibrary() + if (!lib.sourceRoot) throw new Error(`${lib.label} has no files on this host; it can be searched but not read`) + return lib.sourceRoot } async function executeToolImpl(name: string, argsJson: string, ctx?: { labelScope?: string[]; userKey?: string | null; model?: string | null }): Promise { @@ -953,9 +966,9 @@ async function executeToolImpl(name: string, argsJson: string, ctx?: { labelScop const limit = Math.min(Number(args.limit) || 10, 25) if (gufiAvailable()) { const [fts, names, vec] = await Promise.all([ - searchFts(query, limit), - searchNames(query, 5), - searchVector(query, Math.min(limit, 8)).catch(() => []), + searchFts(query, limit, toolLibrary().indexRoot).catch(() => []), + searchNames(query, 5, toolLibrary().indexRoot), + searchVector(query, Math.min(limit, 8), toolLibrary().indexRoot).catch(() => []), ]) // A conversation-level label scope is enforced here regardless of // what the model asked for; model-requested tags narrow further @@ -979,7 +992,7 @@ async function executeToolImpl(name: string, argsJson: string, ctx?: { labelScop case 'read_kb_file': { const rel = String(args.path ?? '') const offset = Math.max(0, Number(args.offset) || 0) - const fc = await readFileContent(rel) + const fc = await readFileContent(rel, toolSourceRoot()) if (fc.content == null) { return { result: `Binary file (${fc.kind}, ${fc.size} bytes); no text available.`, summary: 'binary' } } @@ -989,7 +1002,7 @@ async function executeToolImpl(name: string, argsJson: string, ctx?: { labelScop return { result: slice + more, summary: `${slice.length} chars (${fc.source})` } } case 'list_kb_dir': { - const entries = await listDir(String(args.path ?? '')) + const entries = await listDir(String(args.path ?? ''), toolSourceRoot()) const result = entries.map(e => `${e.type === 'dir' ? 'd' : '-'} ${e.path}${e.type === 'dir' ? '/' : ` (${e.size}b)`}`).join('\n') return { result: result || '(empty)', summary: `${entries.length} entries` } } diff --git a/server/src/gufi.ts b/server/src/gufi.ts index be275e1..66fecd8 100644 --- a/server/src/gufi.ts +++ b/server/src/gufi.ts @@ -177,14 +177,15 @@ export function blendHits(fts: SearchHit[], names: SearchHit[], vec: SearchHit[] .slice(0, limit) } -export async function queryPerDir(sql: string, opts: { threads?: number; subdir?: string } = {}): Promise { - const target = opts.subdir ? path.join(GUFI_INDEX, opts.subdir) : GUFI_INDEX +export async function queryPerDir(sql: string, opts: { threads?: number; subdir?: string; indexRoot?: string } = {}): Promise { + const root = opts.indexRoot ?? GUFI_INDEX + const target = opts.subdir ? path.join(root, opts.subdir) : root const { stdout } = await run('gufi_query', ['-n', String(opts.threads ?? 8), '-d', DELIM, '-E', sql, target]) return parseRows(stdout, 1) } /** Full-text search over enriched fts5 `words` tables, joined to entries by inode. */ -export async function searchFts(queryText: string, limit = 20): Promise { +export async function searchFts(queryText: string, limit = 20, indexRoot?: string): Promise { const expr = ftsExpr(queryText) if (!expr) return [] const sql = @@ -192,9 +193,9 @@ export async function searchFts(queryText: string, limit = 20): Promise ({ - path: toKbRel(r[0]), + path: toKbRel(r[0], indexRoot), name: r[1], size: Number(r[2]) || 0, mtime: Number(r[3]) || 0, @@ -205,15 +206,15 @@ export async function searchFts(queryText: string, limit = 20): Promise { +export async function searchNames(queryText: string, limit = 20, indexRoot?: string): Promise { const where = nameWhere(parseSearchQuery(queryText)) if (where === '0') return [] const sql = `SELECT rpath(sname, sroll)||'/'||name, name, size, mtime ` + `FROM vrpentries WHERE ${where} LIMIT ${limit};` - const rows = await queryPerDir(sql) + const rows = await queryPerDir(sql, { indexRoot }) return rows.slice(0, limit).map(r => ({ - path: toKbRel(r[0]), + path: toKbRel(r[0], indexRoot), name: r[1], size: Number(r[2]) || 0, mtime: Number(r[3]) || 0, @@ -224,8 +225,8 @@ export async function searchNames(queryText: string, limit = 20): Promise() +const vecDbCache = new Map() +export function invalidateDbList(): void { dbListCache.clear(); vecDbCache.clear() } +function listIndexDbs(indexRoot: string = GUFI_INDEX): string[] { + const hit = dbListCache.get(indexRoot) + if (hit && Date.now() - hit.at < 60_000) return hit.dbs const dbs: string[] = [] const walk = (dir: string) => { let entries: fs.Dirent[] @@ -246,8 +249,8 @@ function listIndexDbs(): string[] { if (entries.some(e => e.name === 'db.db')) dbs.push(path.join(dir, 'db.db')) for (const e of entries) if (e.isDirectory()) walk(path.join(dir, e.name)) } - walk(GUFI_INDEX) - dbListCache = { dbs, at: Date.now() } + walk(indexRoot) + dbListCache.set(indexRoot, { dbs, at: Date.now() }) return dbs } @@ -268,9 +271,10 @@ export function vectorsAvailable(): boolean { * semantic search for the entire corpus. Reading sqlite_master is always * valid, so this pass is safe to run over everything. */ -async function listVectorDbs(): Promise { - if (vecDbCache && Date.now() - vecDbCache.at < 60_000) return vecDbCache.dbs - const all = listIndexDbs() +async function listVectorDbs(indexRoot: string = GUFI_INDEX): Promise { + const hit = vecDbCache.get(indexRoot) + if (hit && Date.now() - hit.at < 60_000) return hit.dbs + const all = listIndexDbs(indexRoot) if (!all.length) return [] const script = all.flatMap(db => [ `ATTACH ${q(db)} AS c;`, @@ -287,18 +291,18 @@ async function listVectorDbs(): Promise { .filter(l => l.startsWith('HASVEC')) .map(l => l.split(DELIM)[1]) .filter(Boolean) - vecDbCache = { dbs, at: Date.now() } + vecDbCache.set(indexRoot, { dbs, at: Date.now() }) return dbs } -export async function searchVector(queryText: string, limit = 10): Promise { +export async function searchVector(queryText: string, limit = 10, indexRoot: string = GUFI_INDEX): Promise { if (!vectorsAvailable()) return [] - const idx = path.resolve(GUFI_INDEX) + const idx = path.resolve(indexRoot) const script: string[] = [ `INSERT INTO temp.lembed_models(name, model) SELECT 'minilm384', lembed_model_from_file(${q(EMBED_MODEL)});`, `CREATE TABLE temp.qv AS SELECT lembed('minilm384', ${q(queryText.slice(0, 500))}) AS qe;`, ] - const vecDbs = await listVectorDbs() + const vecDbs = await listVectorDbs(indexRoot) if (!vecDbs.length) return [] for (const db of vecDbs) { const relDir = path.relative(idx, path.dirname(db)) @@ -347,16 +351,16 @@ export interface CorpusStats { byExt?: { ext: string; count: number; bytes: number }[] } -export async function corpusStats(): Promise { +export async function corpusStats(indexRoot?: string): Promise { if (!gufiAvailable()) return { available: false } const sumSql = `SELECT totfiles, totsize FROM vrsummary;` - const rows = await queryPerDir(sumSql) + const rows = await queryPerDir(sumSql, { indexRoot }) let files = 0, totalBytes = 0 for (const r of rows) { files += Number(r[0]) || 0; totalBytes += Number(r[1]) || 0 } const extSql = `SELECT lower(CASE WHEN name LIKE '%.%' THEN replace(name, rtrim(name, replace(name, '.', '')), '') ELSE '' END), ` + `count(*), sum(size) FROM vrpentries GROUP BY 1;` - const extRows = await queryPerDir(extSql) + const extRows = await queryPerDir(extSql, { indexRoot }) const agg = new Map() for (const r of extRows) { const ext = r[0] || '(none)' diff --git a/server/src/kb.ts b/server/src/kb.ts index 1fd79e2..ad2da4c 100644 --- a/server/src/kb.ts +++ b/server/src/kb.ts @@ -19,11 +19,12 @@ export class KbError extends Error { constructor(public status: number, message: string) { super(message) } } -/** Resolve a client-supplied relative path safely inside KB_ROOT. */ -export function resolveKb(rel: string): string { +/** Resolve a client-supplied relative path safely inside a corpus root, the + * knowledge base unless a library's own source root is given. */ +export function resolveKb(rel: string, base: string = KB_ROOT): string { const cleaned = rel.replace(/^\/+/, '') - const abs = path.resolve(KB_ROOT, cleaned) - if (abs !== KB_ROOT && !abs.startsWith(KB_ROOT + path.sep)) { + const abs = path.resolve(base, cleaned) + if (abs !== base && !abs.startsWith(base + path.sep)) { throw new KbError(400, 'path escapes the knowledge base root') } return abs @@ -47,8 +48,8 @@ export function excluded(name: string, isDir: boolean): boolean { return EXCLUDE_FILE_SUFFIXES.some(s => name.toLowerCase().endsWith(s)) } -export async function listDir(rel: string): Promise { - const abs = resolveKb(rel) +export async function listDir(rel: string, base: string = KB_ROOT): Promise { + const abs = resolveKb(rel, base) let dirents try { dirents = await fs.readdir(abs, { withFileTypes: true }) @@ -71,7 +72,7 @@ export async function listDir(rel: string): Promise { try { st = await fs.stat(p) } catch { continue } out.push({ name: d.name, - path: path.relative(KB_ROOT, p), + path: path.relative(base, p), type: d.isDirectory() ? 'dir' : 'file', size: st.size, mtime: Math.floor(st.mtimeMs / 1000), @@ -112,8 +113,8 @@ export interface FileContent { format?: 'markdown' | 'text' } -export async function readFileContent(rel: string): Promise { - const abs = resolveKb(rel) +export async function readFileContent(rel: string, root: string = KB_ROOT): Promise { + const abs = resolveKb(rel, root) const st = await fs.stat(abs) if (st.isDirectory()) throw new KbError(400, 'path is a directory') const kind = classify(abs, false) diff --git a/server/src/libraries.ts b/server/src/libraries.ts new file mode 100644 index 0000000..ea57bcf --- /dev/null +++ b/server/src/libraries.ts @@ -0,0 +1,240 @@ +/** + * Libraries: the indexes the Studio can mount at once. + * + * The knowledge base the server builds and owns is the first library, + * writable, and everything that exists today keeps working against it + * without naming it. Any other library is read only: a root-built site + * index, or an index someone handed over, reached through a local tree. + * Each carries what it supports, which the Studio learns by probing the + * tree once when the library is added; nothing is asked of GUFI and + * nothing is written into the index. + * + * Where they come from, in order: the primary from KB_ROOT, pinned ones + * from STUDIO_LIBRARIES (a JSON array set by the deployment), and ones an + * administrator added in Settings, kept in INDEX_BASE/libraries.json. + */ +import fs from 'node:fs' +import path from 'node:path' +import { execFile } from 'node:child_process' +import { GUFI_BIN, GUFI_INDEX, INDEX_BASE, KB_ROOT, gufiAvailable } from './config.js' +import { KbError } from './kb.js' + +export interface LibraryCaps { + /** A GUFI tree: a db.db at the root. */ + index: boolean + /** The Studio's full-text tables are present in at least one directory. */ + fullText: boolean + /** Vector tables are present in at least one directory. */ + vectors: boolean +} + +export interface Library { + id: string + label: string + indexRoot: string + /** Where the files are, when they are reachable from this host. */ + sourceRoot: string | null + /** Only the primary may add files, index, and enrich. */ + writable: boolean + primary: boolean + /** Pinned by the deployment; cannot be removed from Settings. */ + pinned: boolean + caps: LibraryCaps +} + +export interface LibraryDef { + id: string + label?: string + indexRoot: string + sourceRoot?: string | null +} + +/** The shape the client sees. Paths stay on the server. */ +export interface PublicLibrary { + id: string + label: string + primary: boolean + writable: boolean + pinned: boolean + source: boolean + caps: LibraryCaps +} + +export const PRIMARY_ID = 'kb' +const FILE = path.join(INDEX_BASE, 'libraries.json') +const ID_RE = /^[a-z0-9][a-z0-9_-]{0,39}$/ + +let added: LibraryDef[] | null = null +let probeCache = new Map() + +function loadAdded(): LibraryDef[] { + if (added) return added + try { + const raw = JSON.parse(fs.readFileSync(FILE, 'utf8')) as unknown + added = Array.isArray(raw) ? raw.filter(isDef) : [] + } catch { added = [] } + return added +} + +function saveAdded(defs: LibraryDef[]): void { + added = defs + fs.mkdirSync(path.dirname(FILE), { recursive: true }) + fs.writeFileSync(FILE, JSON.stringify(defs, null, 2)) +} + +function isDef(x: unknown): x is LibraryDef { + if (!x || typeof x !== 'object') return false + const d = x as Record + return typeof d.id === 'string' && ID_RE.test(d.id) && typeof d.indexRoot === 'string' && d.indexRoot.length > 0 +} + +function pinnedDefs(): LibraryDef[] { + const raw = process.env.STUDIO_LIBRARIES + if (!raw) return [] + try { + const arr = JSON.parse(raw) as unknown + return Array.isArray(arr) ? arr.filter(isDef) : [] + } catch { + return [] + } +} + +function sqlite(script: string, timeoutMs = 10_000): Promise { + return new Promise((resolve, reject) => { + const child = execFile(path.join(GUFI_BIN, 'gufi_sqlite3'), [], { timeout: timeoutMs, maxBuffer: 1 << 20 }, + (err, so) => (err && !so ? reject(err) : resolve(so))) + child.stdin?.end(script) + }) +} + +/** Up to `max` db.db files under a tree, breadth first, without walking everything. */ +function sampleDbs(root: string, max = 12): string[] { + const out: string[] = [] + const queue = [root] + let visited = 0 + while (queue.length && out.length < max && visited < 400) { + const dir = queue.shift()! + visited++ + let entries: fs.Dirent[] + try { entries = fs.readdirSync(dir, { withFileTypes: true }) } catch { continue } + if (entries.some(e => e.name === 'db.db')) out.push(path.join(dir, 'db.db')) + for (const e of entries) if (e.isDirectory()) queue.push(path.join(dir, e.name)) + } + return out +} + +/** + * Learn what a tree supports. A GUFI index has a db.db at its root; the + * Studio's own tables are looked for in a sample of directory databases, + * which is enough to say whether search will answer without walking the + * whole tree of a large index. + */ +export async function probeLibrary(indexRoot: string): Promise { + const abs = path.resolve(indexRoot) + const cached = probeCache.get(abs) + if (cached && Date.now() - cached.at < 300_000) return cached.caps + const caps: LibraryCaps = { index: false, fullText: false, vectors: false } + if (fs.existsSync(path.join(abs, 'db.db'))) { + caps.index = true + if (gufiAvailable()) { + for (const db of sampleDbs(abs)) { + try { + const out = await sqlite(`ATTACH '${db.replace(/'/g, "''")}' AS d;\nSELECT name FROM d.sqlite_master WHERE name IN ('words','gvec');\n`) + if (/\bwords\b/.test(out)) caps.fullText = true + if (/\bgvec\b/.test(out)) caps.vectors = true + if (caps.fullText && caps.vectors) break + } catch { /* an unreadable db says nothing */ } + } + } + } + probeCache.set(abs, { caps, at: Date.now() }) + return caps +} + +function primaryLibrary(label: string): Library { + return { + id: PRIMARY_ID, + label, + indexRoot: GUFI_INDEX, + sourceRoot: KB_ROOT, + writable: true, + primary: true, + pinned: true, + caps: probeCache.get(path.resolve(GUFI_INDEX))?.caps ?? { index: gufiAvailable(), fullText: gufiAvailable(), vectors: false }, + } +} + +function fromDef(d: LibraryDef, pinned: boolean): Library { + const abs = path.resolve(d.indexRoot) + return { + id: d.id, + label: d.label || path.basename(abs) || d.id, + indexRoot: abs, + sourceRoot: d.sourceRoot ? path.resolve(d.sourceRoot) : null, + writable: false, + primary: false, + pinned, + caps: probeCache.get(abs)?.caps ?? { index: fs.existsSync(path.join(abs, 'db.db')), fullText: false, vectors: false }, + } +} + +/** Every mounted library, primary first. `label` names the primary. */ +export function listLibraries(primaryLabel = path.basename(KB_ROOT)): Library[] { + const seen = new Set([PRIMARY_ID]) + const out: Library[] = [primaryLibrary(primaryLabel)] + for (const [defs, pinned] of [[pinnedDefs(), true], [loadAdded(), false]] as const) { + for (const d of defs) { + if (seen.has(d.id)) continue + seen.add(d.id) + out.push(fromDef(d, pinned)) + } + } + return out +} + +/** Refresh the capabilities of every library; called at startup and after a change. */ +export async function probeAll(primaryLabel?: string): Promise { + const libs = listLibraries(primaryLabel) + for (const l of libs) l.caps = await probeLibrary(l.indexRoot) + return libs +} + +export function getLibrary(id: string | undefined | null, primaryLabel?: string): Library { + const want = (id || PRIMARY_ID).trim() + const lib = listLibraries(primaryLabel).find(l => l.id === want) + if (!lib) throw new KbError(404, `no library named ${want}`) + return lib +} + +/** The guard every mutation route runs before touching files or the index. */ +export function requireWritable(lib: Library): void { + if (!lib.writable) throw new KbError(403, `${lib.label} is read only`) +} + +export async function addLibrary(def: LibraryDef): Promise { + if (!isDef(def)) throw new KbError(400, 'a library needs an id (letters, digits, dash, underscore) and an index root') + if (def.id === PRIMARY_ID || listLibraries().some(l => l.id === def.id)) throw new KbError(409, `a library named ${def.id} already exists`) + const abs = path.resolve(def.indexRoot) + if (!fs.existsSync(abs)) throw new KbError(400, `no such directory: ${def.indexRoot}`) + if (def.sourceRoot && !fs.existsSync(path.resolve(def.sourceRoot))) throw new KbError(400, `no such directory: ${def.sourceRoot}`) + const caps = await probeLibrary(abs) + if (!caps.index) throw new KbError(400, `${def.indexRoot} does not look like a GUFI index (no db.db at its root)`) + saveAdded([...loadAdded(), { id: def.id, label: def.label?.slice(0, 60) || undefined, indexRoot: abs, sourceRoot: def.sourceRoot ? path.resolve(def.sourceRoot) : null }]) + return getLibrary(def.id) +} + +export function removeLibrary(id: string): void { + const lib = getLibrary(id) + if (lib.primary || lib.pinned) throw new KbError(403, `${lib.label} is set by the deployment and cannot be removed here`) + saveAdded(loadAdded().filter(d => d.id !== id)) +} + +export function publicLibrary(l: Library): PublicLibrary { + return { id: l.id, label: l.label, primary: l.primary, writable: l.writable, pinned: l.pinned, source: !!l.sourceRoot, caps: l.caps } +} + +/** Tests replace the stored set and the probe memory between cases. */ +export function resetLibrariesForTests(): void { + added = null + probeCache = new Map() +} diff --git a/server/src/main.ts b/server/src/main.ts index 869c88f..8578db1 100644 --- a/server/src/main.ts +++ b/server/src/main.ts @@ -28,6 +28,7 @@ import { maybeAutoStart, ragEndpointRoutes } from './ragEndpoint.js' import { gatewayConfigured } from './chat/gateway.js' import { startSweepTimer } from './indexing.js' import { seedKnowledgeBase } from './seed.js' +import { probeAll } from './libraries.js' const app = Fastify({ logger: { level: 'info' } }) await app.register(fastifyMultipart) @@ -95,6 +96,7 @@ await app.register(fleetRoutes) // A brand-new deployment opens on a corpus with some shape rather than an // empty tree; only ever runs when the knowledge base has nothing in it. await seedKnowledgeBase(msg => app.log.info(msg)) +for (const l of await probeAll()) app.log.info(`library ${l.id}: index=${l.caps.index} fullText=${l.caps.fullText} vectors=${l.caps.vectors} files=${l.sourceRoot ? 'yes' : 'no'}${l.writable ? ' writable' : ''}`) startSweepTimer(msg => app.log.info(msg)) // Deploy-time full reindexes rebuild the index without overlay labels // (xattr-less filesystems); restore them once the server is up. diff --git a/server/src/routes.ts b/server/src/routes.ts index e0d6c96..0a18eba 100644 --- a/server/src/routes.ts +++ b/server/src/routes.ts @@ -1,3 +1,4 @@ +import { getLibrary, requireWritable, listLibraries, publicLibrary, addLibrary, removeLibrary, probeAll } from './libraries.js' import type { FastifyInstance } from 'fastify' import { execFile } from 'node:child_process' import path from 'node:path' @@ -113,6 +114,20 @@ export function sanitizedErrorHandler(app: FastifyInstance) { } export async function kbRoutes(app: FastifyInstance): Promise { + // Which library a request means. The primary when unnamed, so every + // existing client keeps its behavior; a name resolves to a mounted + // library or a 404. + const libOf = (req: { query?: unknown; body?: unknown }) => { + const q = (req.query ?? {}) as { library?: string } + const b = (req.body ?? {}) as { library?: string } + return getLibrary(q.library ?? b.library, effectiveSettings().kbLabel) + } + // Files are only reachable when the library has a source root on this host. + const sourceOf = (req: { query?: unknown; body?: unknown }) => { + const lib = libOf(req) + if (!lib.sourceRoot) throw new KbError(409, `${lib.label} has no files on this host; it can be searched and described but not opened`) + return lib.sourceRoot + } app.get('/healthz', async () => ({ ok: true, gufi: gufiAvailable() })) @@ -168,6 +183,8 @@ export async function kbRoutes(app: FastifyInstance): Promise { // Feature previews a deployment has switched on; the client shows // their controls only when the flag and its configuration are both present. features: { voice: { enabled: !!eff.voiceEnabled && !!eff.voiceUrl, url: eff.voiceUrl || '' } }, + libraries: listLibraries(eff.kbLabel).map(publicLibrary), + sections: eff.sections.length ? eff.sections : undefined, user, } }) @@ -226,12 +243,13 @@ export async function kbRoutes(app: FastifyInstance): Promise { app.get('/api/kb/tree', async req => { const { path: rel = '' } = req.query as { path?: string } - const entries = await listDir(rel) + const root = sourceOf(req) + const entries = await listDir(rel, root) // Own labels straight from the filesystem xattrs (one getfattr per // listing), so a just-applied label shows without waiting on the index. - const tagged = await readTagsBatch(entries.map(e => resolveKb(e.path))).catch(() => new Map()) + const tagged = await readTagsBatch(entries.map(e => resolveKb(e.path, root))).catch(() => new Map()) for (const e of entries) { - const t = tagged.get(resolveKb(e.path)) + const t = tagged.get(resolveKb(e.path, root)) if (t?.length) (e as typeof e & { tags?: string[] }).tags = t } return { path: rel, entries } @@ -239,12 +257,12 @@ export async function kbRoutes(app: FastifyInstance): Promise { app.get('/api/kb/file', async req => { const { path: rel = '' } = req.query as { path?: string } - return readFileContent(rel) + return readFileContent(rel, sourceOf(req)) }) app.get('/api/kb/download', async (req, reply) => { const { path: rel = '' } = req.query as { path?: string } - const abs = resolveKb(rel) + const abs = resolveKb(rel, sourceOf(req)) reply.header('Content-Type', mimeFor(abs)) reply.header('Content-Disposition', `attachment; filename="${path.basename(abs)}"`) return reply.send(createReadStream(abs)) @@ -253,7 +271,7 @@ export async function kbRoutes(app: FastifyInstance): Promise { // Inline serving of the actual file (images in , PDFs in