diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 17e6622..14664f2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,3 +39,23 @@ jobs: run: | pnpm --filter @activate-studio/server --prod deploy --legacy /tmp/deploy-check node testdata/check-packaging.mjs /tmp/deploy-check + + macos: + # The Studio is expected to run from source on a laptop, and macOS is + # where that claim rots quietly: a Linux-only assumption in the server + # or a tool that exists only under apt shows up here and nowhere else. + # GUFI is not built in this job, which takes far longer than the tests + # and has its own upstream CI; what is checked is that the server and + # web build and the suite passes on macOS. + runs-on: macos-15 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 22 + - uses: pnpm/action-setup@v4 + - name: install extraction tools + run: brew install tesseract poppler + - run: pnpm install --frozen-lockfile + - run: pnpm build + - run: pnpm test diff --git a/README.md b/README.md index 97441ee..a63bc14 100644 --- a/README.md +++ b/README.md @@ -54,11 +54,19 @@ Environment: `KB_ROOT` (corpus directory), `KB_LABEL`, `APP_NAME`, `APP_ICON` (p ## Running on macOS The server and web app run from source on macOS the same way as the -standalone quick start above; the Linux-specific pieces are optional. -GUFI does not build on macOS, and the app runs without it: browsing, -grep search, chat, and viewers all work, while the indexed search and -query surfaces answer with empty results and a note until an index -exists. Document +standalone quick start above, and `indexer/setup_gufi.sh` builds GUFI +there too: it installs the Homebrew toolchain that GUFI's own macOS CI +uses (LLVM rather than Apple clang, libomp, and the GNU utilities its +scripts expect) and configures the build against them. + +GUFI's AI dependencies, `sqlite-vec` and `sqlite-lembed` with +llama.cpp, are what semantic search needs and are also the hard part of +the build on macOS. They are on by default; `GUFI_AI=0 +indexer/setup_gufi.sh` skips them for a quick build that keeps +metadata, filename, and full-text search and loses only the semantic +blend. The app also runs with no index at all: browsing, grep search, +chat, and viewers work, while the indexed search and query surfaces +answer with empty results and a note until an index exists. Document extraction and previews use whatever tools are present: `brew install --cask libreoffice` provides `soffice` for DOCX/PPTX previews, and Word, PowerPoint, Excel and PDF files extract to Markdown through diff --git a/indexer/setup_gufi.sh b/indexer/setup_gufi.sh index 6bbb854..0bab95f 100755 --- a/indexer/setup_gufi.sh +++ b/indexer/setup_gufi.sh @@ -4,10 +4,48 @@ set -euo pipefail SRC="${GUFI_SRC:-$HOME/gufi-src}" PREFIX="${GUFI_PREFIX:-/opt/gufi}" +# GUFI's AI dependencies (sqlite-vec and sqlite-lembed, which carries +# llama.cpp) are what semantic search needs, and they are also the hard part +# of the build. DEP_AI defaults to On upstream, so a plain cmake attempts +# them; GUFI_AI=0 turns them off for a quick build that keeps full-text and +# metadata search and loses only the semantic blend. +AI="${GUFI_AI:-1}" +JOBS="$(getconf _NPROCESSORS_ONLN 2>/dev/null || echo 4)" -sudo DEBIAN_FRONTEND=noninteractive apt-get install -y \ - cmake libsqlite3-dev pkg-config zlib1g-dev libpcre2-dev libattr1-dev attr \ - autoconf automake libtool tesseract-ocr poppler-utils +case "$(uname -s)" in + Darwin) + # Matching GUFI's own macOS CI: Homebrew's LLVM rather than Apple clang + # (libomp is not wired into the system toolchain), plus the GNU + # utilities its scripts assume. The GNU tools go first on PATH for this + # build only, which is why the caller's PATH is left alone. + command -v brew >/dev/null 2>&1 || { echo "Homebrew is required: https://brew.sh"; exit 1; } + brew install cmake pkgconf pcre2 sqlite autoconf automake libtool \ + llvm libomp gettext coreutils findutils gnu-sed gpatch grep diffutils \ + tesseract poppler + for tool in coreutils findutils gnu-sed grep; do + PATH="$(brew --prefix "$tool")/libexec/gnubin:$PATH" + done + PATH="$(brew --prefix diffutils)/bin:$PATH" + export PATH + export CC="$(brew --prefix llvm)/bin/clang" + export CXX="$(brew --prefix llvm)/bin/clang++" + OMP_PREFIX="$(brew --prefix libomp)" + EXTRA_CMAKE=( + -DCMAKE_OSX_SYSROOT=macosx + -DOpenMP_C_LIB_NAMES=libomp -DOpenMP_C_FLAGS=-fopenmp + -DOpenMP_CXX_LIB_NAMES=libomp -DOpenMP_CXX_FLAGS=-fopenmp + -DOpenMP_libomp_LIBRARY="$OMP_PREFIX/lib/libomp.dylib" + -DCMAKE_C_FLAGS="-I$OMP_PREFIX/include" + -DCMAKE_CXX_FLAGS="-I$OMP_PREFIX/include" + ) + ;; + *) + sudo DEBIAN_FRONTEND=noninteractive apt-get install -y \ + cmake libsqlite3-dev pkg-config zlib1g-dev libpcre2-dev libattr1-dev attr \ + autoconf automake libtool tesseract-ocr poppler-utils + EXTRA_CMAKE=() + ;; +esac if [ ! -d "$SRC" ]; then git clone https://github.com/mar-file-system/GUFI.git "$SRC" @@ -15,16 +53,23 @@ fi cd "$SRC" git fetch --tags mkdir -p build && cd build -cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX="$PREFIX" -make -j"$(nproc)" -sudo make install +cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX="$PREFIX" \ + -DDEP_AI="$([ "$AI" = "1" ] && echo On || echo Off)" \ + ${EXTRA_CMAKE+"${EXTRA_CMAKE[@]}"} +make -j"$JOBS" +# A prefix the user owns needs no sudo, which is the common case on a laptop. +if [ -w "$(dirname "$PREFIX")" ]; then make install; else sudo make install; fi "$PREFIX/bin/gufi_query" -h >/dev/null 2>&1 || true echo "GUFI installed at $PREFIX" # Embedding model for the vector layer (384-dim all-MiniLM, per the GUFI master doc). PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -MODEL_DIR="$PROJECT_ROOT/index/models" +MODEL_DIR="${INDEX_BASE:-$PROJECT_ROOT/index}/models" +if [ "$AI" != "1" ]; then + echo "Built without the AI dependencies: full-text and metadata search work, semantic search does not." + exit 0 +fi mkdir -p "$MODEL_DIR" if [ ! -f "$MODEL_DIR/minilm384.gguf" ]; then curl -sL -o "$MODEL_DIR/minilm384.gguf" \