Libraries: mount several indexes at once, read only beside the knowledge base

.env.example

@@ -4,6 +4,8 @@
 # --- corpus ---
 KB_ROOT=/data/knowledge-base          # the directory this studio works over
 KB_LABEL='Knowledge Base'             # label shown at the top of the library tree
+# STUDIO_LIBRARIES='[{"id":"scratch","label":"Scratch","indexRoot":"/gufi/scratch"}]'  # read-only indexes to mount beside the knowledge base (docs/LIBRARIES.md)
+# STUDIO_SECTIONS='library,search,overview'   # which sections to show; unset shows all
 
 # --- branding ---
 APP_NAME='Studio'                     # product name in the header and browser tab

README.md

@@ -63,6 +63,16 @@ request.
 Start to finish, including what works before an index exists:
 [`docs/MACOS.md`](docs/MACOS.md).
 
+## Libraries
+
+The Studio can mount several indexes at once: the knowledge base it owns,
+plus read-only ones such as a site's root-built GUFI index or an index
+someone handed over. Users pick one in the Library rail; administrators
+list them in the deploy form, set `STUDIO_LIBRARIES`, or add one in
+Settings, where the tree is probed first. `STUDIO_SECTIONS` chooses which
+parts of the app appear, so a site can run an index viewer with no
+assistant. Details: [`docs/LIBRARIES.md`](docs/LIBRARIES.md).
+
 ## Container build
 
 `deploy/app.def` packages the server, web build, GUFI, and the

deploy/workflow-cfd-studio.yaml

@@ -336,6 +336,21 @@ permissions:
             type: string
             optional: true
             tooltip: One short paragraph on what this Studio is for; injected into the assistant's system prompt and editable later in Settings.
+          libraries:
+            label: Additional Libraries
+            type: string
+            optional: true
+            tooltip: |
+              Other indexes this Studio mounts read only, as a JSON list. Each entry
+              needs an id and an indexRoot (a GUFI tree); a label and a sourceRoot
+              (where the files are, when they are on this host) are optional.
+          sections:
+            label: Visible Sections
+            type: string
+            optional: true
+            tooltip: |
+              Comma-separated list of the sections to show, from chat, library,
+              search, query, overview, history, agents. Leave empty for all of them.
           build_gufi:
             label: Build GUFI Index Engine
             type: boolean
@@ -901,6 +916,12 @@ jobs:
           if [ -n "${{ inputs.kb_settings.starter_dirs }}" ]; then
             echo "export KB_STARTER_DIRS='${{ inputs.kb_settings.starter_dirs }}'" >> ${SCRIPT}
           fi
+          if [ -n "${{ inputs.kb_settings.libraries }}" ]; then
+            printf 'export STUDIO_LIBRARIES=%q\n' '${{ inputs.kb_settings.libraries }}' >> ${SCRIPT}
+          fi
+          if [ -n "${{ inputs.kb_settings.sections }}" ]; then
+            printf 'export STUDIO_SECTIONS=%q\n' '${{ inputs.kb_settings.sections }}' >> ${SCRIPT}
+          fi
           if [ -n "${{ inputs.app_settings.icon_path }}" ]; then
             echo "export APP_ICON='${{ inputs.app_settings.icon_path }}'" >> ${SCRIPT}
           fi
@@ -972,6 +993,8 @@ jobs:
             --env INDEX_BASE="${INDEX_BASE}" --env KB_LABEL="${KB_LABEL}" \
             --env APP_NAME="${APP_NAME}" --env INDEX_ON_START="${INDEX_ON_START}" \
             ${KB_STARTER_DIRS:+--env KB_STARTER_DIRS="${KB_STARTER_DIRS}"} \
+            ${STUDIO_LIBRARIES:+--env STUDIO_LIBRARIES="${STUDIO_LIBRARIES}"} \
+            ${STUDIO_SECTIONS:+--env STUDIO_SECTIONS="${STUDIO_SECTIONS}"} \
             ${BANNER_TEXT:+--env BANNER_TEXT="${BANNER_TEXT}"} \
             ${BANNER_COLOR:+--env BANNER_COLOR="${BANNER_COLOR}"} \
             ${APP_ICON:+--env APP_ICON="${APP_ICON}"} \

deploy/workflow.yaml

@@ -331,6 +331,23 @@ permissions:
             type: string
             optional: true
             tooltip: One short paragraph on what this Studio is for; injected into the assistant's system prompt and editable later in Settings.
+          libraries:
+            label: Additional Libraries
+            type: string
+            optional: true
+            tooltip: |
+              Other indexes this Studio mounts read only, as a JSON list. Each entry
+              needs an id and an indexRoot (a GUFI tree); a label and a sourceRoot
+              (where the files are, when they are on this host) are optional.
+              Example: [{"id":"scratch","label":"Scratch","indexRoot":"/gufi/scratch"}]
+          sections:
+            label: Visible Sections
+            type: string
+            optional: true
+            tooltip: |
+              Comma-separated list of the sections to show, from chat, library,
+              search, query, overview, history, agents. Leave empty for all of them.
+              "library,search,overview" makes the Studio an index viewer with no assistant.
           build_gufi:
             label: Build GUFI Index Engine
             type: boolean
@@ -930,6 +947,12 @@ jobs:
           if [ -n "${{ inputs.kb_settings.starter_dirs }}" ]; then
             echo "export KB_STARTER_DIRS='${{ inputs.kb_settings.starter_dirs }}'" >> ${SCRIPT}
           fi
+          if [ -n "${{ inputs.kb_settings.libraries }}" ]; then
+            printf 'export STUDIO_LIBRARIES=%q\n' '${{ inputs.kb_settings.libraries }}' >> ${SCRIPT}
+          fi
+          if [ -n "${{ inputs.kb_settings.sections }}" ]; then
+            printf 'export STUDIO_SECTIONS=%q\n' '${{ inputs.kb_settings.sections }}' >> ${SCRIPT}
+          fi
           if [ -n "${{ inputs.app_settings.icon_path }}" ]; then
             echo "export APP_ICON='${{ inputs.app_settings.icon_path }}'" >> ${SCRIPT}
           fi
@@ -1019,6 +1042,8 @@ jobs:
             --env INDEX_BASE="${INDEX_BASE}" --env KB_LABEL="${KB_LABEL}" \
             --env APP_NAME="${APP_NAME}" --env INDEX_ON_START="${INDEX_ON_START}" \
             ${KB_STARTER_DIRS:+--env KB_STARTER_DIRS="${KB_STARTER_DIRS}"} \
+            ${STUDIO_LIBRARIES:+--env STUDIO_LIBRARIES="${STUDIO_LIBRARIES}"} \
+            ${STUDIO_SECTIONS:+--env STUDIO_SECTIONS="${STUDIO_SECTIONS}"} \
             ${BANNER_TEXT:+--env BANNER_TEXT="${BANNER_TEXT}"} \
             ${BANNER_COLOR:+--env BANNER_COLOR="${BANNER_COLOR}"} \
             ${APP_ICON:+--env APP_ICON="${APP_ICON}"} \

docs/ARCHITECTURE.md

@@ -8,6 +8,8 @@ GUFI represents a filesystem as a parallel tree of SQLite databases. `gufi_dir2i
 
 The index lives at `$INDEX_BASE/gufi/<basename of KB_ROOT>/`, where `INDEX_BASE` defaults to `index/` beside the code and is usually pointed at durable storage instead. That extra level is the one `gufi_dir2index` adds for the source directory, and both the full rebuild (`indexer/reindex.sh`) and the server's incremental passes write into it; flattening it leaves a rebuilt index the server never reads. Alongside the tree, `INDEX_BASE` holds the extract cache, rendered PDF pages, settings, the credential vault, saved queries, conversations, and the embedding model, so a deployment can rebuild its working directory without losing state. The build excludes `.git`, `node_modules`, `.venv`, `__pycache__`, `dist`, `build`, caches, screenshots, and dot-directories via a `--skip-file`.
 
+That index is the primary library. The server can mount others read only, each a GUFI tree with an optional source root, addressed by a `library` parameter that defaults to the primary; `server/src/libraries.ts` holds the registry and the probe, and `docs/LIBRARIES.md` describes the behavior.
+
 Two properties of this design determine the rest of the system:
 
 - Queries fan out per directory. `gufi_query` walks the index breadth-first with a thread pool and runs your SQL against every `db.db` independently, so a corpus-wide query is hundreds of small local queries.

docs/LIBRARIES.md

@@ -0,0 +1,82 @@
+# Libraries
+
+A library is an index the Studio has mounted. The knowledge base the server
+builds and owns is the first library; it is the only one the Studio writes
+to, and every request that does not name a library means it, so nothing
+that existed before libraries has changed. Any other library is read only:
+a site's index built by root on a schedule, or an index someone handed
+over on a drive.
+
+## What a library carries
+
+| Field | Decides |
+|---|---|
+| `id`, `label` | what the user picks in the Library switcher, and which library a path belongs to |
+| `indexRoot` | the GUFI tree: a directory with a `db.db` at its root |
+| `sourceRoot`, optional | whether files can be opened, or only searched and described |
+| writable | whether the Studio may add files, index, and enrich; the primary only |
+
+When a library is added the Studio probes it once and records what it
+found: whether it is a GUFI index, whether the Studio's full-text tables
+are present, whether vector tables are present, and whether the source is
+readable. Nothing is asked of GUFI and nothing is written into the index.
+
+## What works on a read-only library
+
+Browsing, filename and metadata search, statistics, and opening files
+when a source root is set. Full-text and semantic search need the
+Studio's enrichment tables, which today are written only into the primary
+index; on any other library those modes are skipped and search still
+answers from names and metadata. Upload, move, rename, delete, labels on
+files, and re-indexing are refused with a 403 that says the library is
+read only, and the interface does not offer them.
+
+A library without a source root can be searched and described but not
+opened: the file routes answer 409, and the viewer says the file is not
+reachable from this host. That is the normal shape for an index of a
+filesystem the Studio's machine cannot see.
+
+## Adding libraries
+
+*By the deployment:* the ACTIVATE deploy form has an Additional Libraries
+field under Knowledge Base, a JSON list, and a Visible Sections field. The
+same two settings are the environment variables `STUDIO_LIBRARIES` and
+`STUDIO_SECTIONS` when running standalone:
+
+```
+STUDIO_LIBRARIES='[{"id":"scratch","label":"Scratch","indexRoot":"/gufi/scratch","sourceRoot":"/lustre/scratch"}]'
+STUDIO_SECTIONS='library,search,overview'
+```
+
+Libraries set this way are pinned: they appear for every user and cannot
+be removed from Settings.
+
+*By an administrator:* Settings > Libraries lists what is mounted and what
+each supports, and adds one by path. The Studio probes the tree and
+refuses a path that is not a GUFI index, with the reason. Added libraries
+are kept in `libraries.json` under the index base.
+
+## Sections
+
+`STUDIO_SECTIONS` chooses which parts of the app appear, from `chat`,
+`library`, `search`, `query`, `overview`, `history`, and `agents`. Leave it
+unset for all of them. A site that wants an index viewer and nothing else
+sets `library,search,overview`, and the assistant and the agents are not
+rendered. Settings and Help always remain reachable.
+
+## Addressing
+
+Every request that touches the corpus or the index accepts a `library`
+query or body parameter. The client adds it from the current selection,
+so views never build it themselves; the primary is sent as nothing. Deep
+links carry it as `&lib=<id>` after the path, so a shared `#open=` link
+and a reload land in the right library.
+
+## Not yet
+
+Enrichment for read-only libraries, in sidecar databases beside the index
+on GUFI's external attach mechanism, so full-text and semantic search work
+there too. The Query page, which still scopes to the primary. Searching
+across several libraries in one query. A remote library reached through
+GUFI's client rather than a local tree. Each is a follow-on to this
+mechanism rather than a change to it.

server/src/chat/routes.ts

@@ -1078,7 +1078,7 @@ ${ctx}` : ctx
         const q = String((lastUserQuestion as WireMessage | undefined)?.content ?? '').trim()
         if (q && q.length > 12) {
           try {
-            const seeded = await executeTool('search_kb', JSON.stringify({ query: q.slice(0, 300) }), { labelScope, userKey: pwToolKey, model: String(body.model ?? '') || null, conversationId: body.conversationId ?? null, userId: req.user?.id ?? null })
+            const seeded = await executeTool('search_kb', JSON.stringify({ query: q.slice(0, 300) }), { labelScope, library: String((body as { library?: string }).library ?? '') || null, userKey: pwToolKey, model: String(body.model ?? '') || null, conversationId: body.conversationId ?? null, userId: req.user?.id ?? null })
             const callId = `seed-${Date.now()}`
             messages.push({ role: 'assistant', content: null, tool_calls: [{ index: 0, id: callId, type: 'function', function: { name: 'search_kb', arguments: JSON.stringify({ query: q.slice(0, 300) }) } }] } as WireMessage)
             messages.push({ role: 'tool', tool_call_id: callId, content: seeded.result } as WireMessage)
@@ -1138,7 +1138,7 @@ ${ctx}` : ctx
                 continue
               }
               try {
-                const out = await executeTool(tc.function.name, tc.function.arguments, { labelScope, userKey: pwToolKey, model: String(body.model ?? '') || null, conversationId: body.conversationId ?? null, userId: req.user?.id ?? null })
+                const out = await executeTool(tc.function.name, tc.function.arguments, { labelScope, library: String((body as { library?: string }).library ?? '') || null, userKey: pwToolKey, model: String(body.model ?? '') || null, conversationId: body.conversationId ?? null, userId: req.user?.id ?? null })
                 toolCache.set(key, out.result)
                 outcomes[i] = out
               } catch (e) {

server/src/chat/tools.ts

@@ -7,6 +7,7 @@ import path from 'node:path'
 import { gufiAvailable, isMissingCli, KB_ROOT, MAX_PREVIEW_BYTES, NO_CLI_MESSAGE, PROJECT_ROOT, PW_CLI } from '../config.js'
 import { listDir, readFileContent, KbError } from '../kb.js'
 import { blendHits, searchFts, searchNames, searchVector } from '../gufi.js'
+import { getLibrary } from '../libraries.js'
 import { annotateHits } from '../tags.js'
 import { effectiveSettings } from '../settings.js'
 import { composeWorkflows } from '../workflowCompose.js'
@@ -937,10 +938,22 @@ export interface ToolOutcome {
 // Per-invocation context (caller's own platform key for pw CLI executions)
 // travels via AsyncLocalStorage so nested helpers stay signature-stable and
 // concurrent tool calls from different users cannot cross-contaminate.
-const toolContext = new AsyncLocalStorage<{ userKey: string | null; conversationId?: string | null; userId?: string | null }>()
+const toolContext = new AsyncLocalStorage<{ userKey: string | null; conversationId?: string | null; userId?: string | null; library?: string | null }>()
 
-export async function executeTool(name: string, argsJson: string, ctx?: { labelScope?: string[]; userKey?: string | null; model?: string | null; conversationId?: string | null; userId?: string | null }): Promise<ToolOutcome> {
-  return toolContext.run({ userKey: ctx?.userKey ?? null, conversationId: ctx?.conversationId ?? null, userId: ctx?.userId ?? null }, () => executeToolImpl(name, argsJson, ctx))
+export async function executeTool(name: string, argsJson: string, ctx?: { labelScope?: string[]; userKey?: string | null; model?: string | null; library?: string | null; conversationId?: string | null; userId?: string | null }): Promise<ToolOutcome> {
+  return toolContext.run({ userKey: ctx?.userKey ?? null, conversationId: ctx?.conversationId ?? null, userId: ctx?.userId ?? null, library: ctx?.library ?? null }, () => executeToolImpl(name, argsJson, ctx))
+}
+
+// The library a tool call reads from: the primary unless the conversation
+// named one. A library without files on this host can be searched but its
+// files cannot be opened, and the tool says so instead of failing oddly.
+function toolLibrary() {
+  return getLibrary(toolContext.getStore()?.library ?? null)
+}
+function toolSourceRoot(): string {
+  const lib = toolLibrary()
+  if (!lib.sourceRoot) throw new Error(`${lib.label} has no files on this host; it can be searched but not read`)
+  return lib.sourceRoot
 }
 
 async function executeToolImpl(name: string, argsJson: string, ctx?: { labelScope?: string[]; userKey?: string | null; model?: string | null }): Promise<ToolOutcome> {
@@ -953,9 +966,9 @@ async function executeToolImpl(name: string, argsJson: string, ctx?: { labelScop
         const limit = Math.min(Number(args.limit) || 10, 25)
         if (gufiAvailable()) {
           const [fts, names, vec] = await Promise.all([
-            searchFts(query, limit),
-            searchNames(query, 5),
-            searchVector(query, Math.min(limit, 8)).catch(() => []),
+            searchFts(query, limit, toolLibrary().indexRoot).catch(() => []),
+            searchNames(query, 5, toolLibrary().indexRoot),
+            searchVector(query, Math.min(limit, 8), toolLibrary().indexRoot).catch(() => []),
           ])
           // A conversation-level label scope is enforced here regardless of
           // what the model asked for; model-requested tags narrow further
@@ -979,7 +992,7 @@ async function executeToolImpl(name: string, argsJson: string, ctx?: { labelScop
       case 'read_kb_file': {
         const rel = String(args.path ?? '')
         const offset = Math.max(0, Number(args.offset) || 0)
-        const fc = await readFileContent(rel)
+        const fc = await readFileContent(rel, toolSourceRoot())
         if (fc.content == null) {
           return { result: `Binary file (${fc.kind}, ${fc.size} bytes); no text available.`, summary: 'binary' }
         }
@@ -989,7 +1002,7 @@ async function executeToolImpl(name: string, argsJson: string, ctx?: { labelScop
         return { result: slice + more, summary: `${slice.length} chars (${fc.source})` }
       }
       case 'list_kb_dir': {
-        const entries = await listDir(String(args.path ?? ''))
+        const entries = await listDir(String(args.path ?? ''), toolSourceRoot())
         const result = entries.map(e => `${e.type === 'dir' ? 'd' : '-'} ${e.path}${e.type === 'dir' ? '/' : ` (${e.size}b)`}`).join('\n')
         return { result: result || '(empty)', summary: `${entries.length} entries` }
       }