diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 773f33a..4aad4d5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -73,6 +73,20 @@ jobs: - name: Test (release, -DNDEBUG) run: make -C tests run-release + quickstart: + name: Quickstart example (make + CMake) + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + - name: Build and run via make + run: make -C examples run + - name: Build and run via CMake (consumer integration path) + run: | + cmake -B build -DEDGEVECTOR_BUILD_EXAMPLES=ON + cmake --build build + ./build/quickstart + asan-ubsan: name: ASan + UBSan runs-on: ubuntu-24.04 diff --git a/.gitignore b/.gitignore index 1e36723..fd347fc 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,10 @@ tests/ev_bench_*.evhg # WSL validation/benchmark logs (regenerated by scripts, not source) tests/wsl_results/ + +# Example build artifacts +examples/quickstart +examples/quickstart.exe +examples/*.evec +examples/*.evhg +build/ diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..87468bb --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,43 @@ +# EdgeVector: header-only, so consumption is one INTERFACE target. +# +# FetchContent: +# FetchContent_Declare(edgevector +# GIT_REPOSITORY https://github.com/JonathanKash/EdgeVector.git +# GIT_TAG main) +# FetchContent_MakeAvailable(edgevector) +# target_link_libraries(your_app PRIVATE edgevector::edgevector) +# +# Or vendor the repo and: add_subdirectory(EdgeVector) +# +# -DEDGEVECTOR_BUILD_EXAMPLES=ON builds examples/quickstart. +cmake_minimum_required(VERSION 3.14) +project(edgevector VERSION 0.8.0 LANGUAGES CXX) + +if(MSVC) + message(FATAL_ERROR + "EdgeVector requires GCC or Clang (it uses __builtin_popcountll and " + "__builtin_prefetch). On Windows, use MinGW-w64 or clang.") +endif() + +add_library(edgevector INTERFACE) +add_library(edgevector::edgevector ALIAS edgevector) +target_include_directories(edgevector INTERFACE + $ + $) +target_compile_features(edgevector INTERFACE cxx_std_17) + +# Concurrent build/queries use std::thread; harmless where already implicit. +find_package(Threads REQUIRED) +target_link_libraries(edgevector INTERFACE Threads::Threads) + +include(GNUInstallDirs) +install(DIRECTORY include/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) + +option(EDGEVECTOR_BUILD_EXAMPLES "Build the EdgeVector examples" OFF) +if(EDGEVECTOR_BUILD_EXAMPLES) + add_executable(quickstart examples/quickstart.cpp) + target_link_libraries(quickstart PRIVATE edgevector::edgevector) + # -march=native activates POPCNT/AVX2 (x86) or NEON codegen; correctness + # does not depend on it, speed does. + target_compile_options(quickstart PRIVATE -O2 -march=native -Wall -Wextra) +endif() diff --git a/README.md b/README.md index f150922..f81f57b 100644 --- a/README.md +++ b/README.md @@ -211,6 +211,45 @@ performance — run `make -C tests bench` on your actual silicon (the Makefile's `ARCH`/`RUNNER` knobs make that a ten-minute job) before committing a product. +## Getting it into your project + +Three routes, all ending at `#include ` (one +umbrella header; or include the individual headers you use). Requires GCC or +Clang with C++17 — MSVC is not supported. + +**1. Vendor the headers (firmware-style).** Copy `include/edgevector/` into +your tree, add the directory to your include path. Done — there is nothing +to link. + +**2. CMake FetchContent:** + +```cmake +include(FetchContent) +FetchContent_Declare(edgevector + GIT_REPOSITORY https://github.com/JonathanKash/EdgeVector.git + GIT_TAG main) # or pin a commit +FetchContent_MakeAvailable(edgevector) +target_link_libraries(your_app PRIVATE edgevector::edgevector) +``` + +**3. CMake subdirectory:** vendor the repo and `add_subdirectory(EdgeVector)`; +the same `edgevector::edgevector` target appears. + +Then start from **`examples/quickstart.cpp`** — a complete, runnable, +self-checking tour of the whole pipeline (quantize → multi-threaded build → +persist → reload → all three search modes → filtering → deletion → slot +reclamation), built and executed by CI on every push so it can never drift +from the library: + +```sh +make -C examples run # or: +cmake -B build -DEDGEVECTOR_BUILD_EXAMPLES=ON && cmake --build build && ./build/quickstart +``` + +Compile flags that matter: `-O2 -march=native` (or `-march=armv8-a` when +cross-compiling) so the Hamming kernel gets POPCNT/AVX2 or NEON. +`EDGEVECTOR_VERSION` in `edgevector.hpp` identifies the vendored version. + ## Quick start Everything is three `#include`s; no linking, no build step for the library diff --git a/examples/Makefile b/examples/Makefile new file mode 100644 index 0000000..11daab0 --- /dev/null +++ b/examples/Makefile @@ -0,0 +1,23 @@ +# Build and run the EdgeVector quickstart. Same knobs as tests/Makefile: +# ARCH (default -march=native), CXX, RUNNER for cross/emulated runs. +CXX ?= g++ +ARCH ?= -march=native +CXXFLAGS := -std=c++17 -O2 $(ARCH) -Wall -Wextra +RUNNER ?= + +ifeq ($(OS),Windows_NT) + EXE := .exe +else + EXE := +endif + +quickstart$(EXE): quickstart.cpp $(wildcard ../include/edgevector/*.hpp) + $(CXX) $(CXXFLAGS) -I../include quickstart.cpp -o $@ + +run: quickstart$(EXE) + $(RUNNER) ./quickstart$(EXE) + +clean: + rm -f quickstart$(EXE) quickstart.evec quickstart.evhg + +.PHONY: run clean diff --git a/examples/quickstart.cpp b/examples/quickstart.cpp new file mode 100644 index 0000000..b64275d --- /dev/null +++ b/examples/quickstart.cpp @@ -0,0 +1,180 @@ +// ============================================================================ +// EdgeVector quickstart: a complete, runnable tour of the library. +// +// Build and run (from examples/): +// make run +// or directly: +// g++ -std=c++17 -O2 -march=native -I../include quickstart.cpp -o quickstart +// ./quickstart +// or via CMake (from the repo root): +// cmake -B build -DEDGEVECTOR_BUILD_EXAMPLES=ON && cmake --build build +// ./build/quickstart +// +// The program synthesizes an embedding-like dataset, then walks the whole +// production pipeline: quantize -> build (multi-threaded) -> persist -> +// reload with zero rebuild -> search three ways (Hamming, asymmetric +// re-rank, float-exact re-rank) -> filter -> delete -> reclaim a slot. +// Every step checks its own results and the program exits non-zero if +// anything is off, so this file doubles as a smoke test (CI runs it). +// ============================================================================ + +#include + +#include +#include +#include +#include +#include + +using namespace edgevector; + +static int g_failures = 0; +static void expect(bool ok, const char* what) { + std::printf(" %s %s\n", ok ? "ok " : "FAIL", what); + if (!ok) ++g_failures; +} + +int main() { + std::printf("EdgeVector %s quickstart\n\n", EDGEVECTOR_VERSION); + + // ------------------------------------------------------------------ + // 0. A dataset. In real life these are your embeddings; here we make + // 10,000 clustered 512-d vectors so the search has structure to find. + // ------------------------------------------------------------------ + const std::size_t dim = 512; + const std::uint32_t n = 10000; + const std::size_t rb = padded_bytes(dim); // 64 bytes per vector at 512-d + + std::mt19937 rng(42); + std::normal_distribution gauss(0.0f, 1.0f); + std::vector centers(100 * dim); + for (float& c : centers) c = gauss(rng); + + std::vector floats(static_cast(n) * dim); + for (std::uint32_t i = 0; i < n; ++i) { + const float* c = centers.data() + (i % 100) * dim; + for (std::size_t d = 0; d < dim; ++d) + floats[i * dim + d] = c[d] + 0.5f * gauss(rng); + } + + // ------------------------------------------------------------------ + // 1. Quantize: 2,048 float bytes -> 64 code bytes per vector (32x). + // Codes must live in an 8-byte-aligned block; backing the buffer + // with uint64_t guarantees that. + // ------------------------------------------------------------------ + std::vector code_words((rb / 8) * n); + auto* codes = reinterpret_cast(code_words.data()); + for (std::uint32_t i = 0; i < n; ++i) + quantize(floats.data() + i * dim, dim, codes + i * rb); + std::printf("[1] quantized %u vectors: %.1f KB of codes (float32: %.1f KB)\n", + n, n * rb / 1024.0, n * dim * 4 / 1024.0); + + // ------------------------------------------------------------------ + // 2. Build the index using every hardware thread, then persist BOTH + // artifacts: the vector file (EVEC) and the graph file (EVHG). + // ------------------------------------------------------------------ + HNSWGraph builder(codes, rb, dim, n); // M=16, ef_construction=200 defaults + std::vector ids(n); + std::iota(ids.begin(), ids.end(), 0u); + expect(builder.insert_batch(ids.data(), n, 0) == n, "built the index"); + expect(builder.validate_integrity(), "index passes integrity validation"); + expect(write_storage_file("quickstart.evec", dim, n, codes) == + StorageStatus::ok, "vector file written"); + expect(builder.save_graph("quickstart.evhg") == GraphIoStatus::ok, + "graph file written"); + + // ------------------------------------------------------------------ + // 3. Device startup: mmap the vectors, load the graph. No rebuild. + // ------------------------------------------------------------------ + MMapStorage store; + expect(store.open("quickstart.evec") == StorageStatus::ok, + "vector file mapped (zero-copy)"); + HNSWGraph graph(store.vector(0), store.record_bytes(), + static_cast(store.dim()), + static_cast(store.count())); + expect(graph.load_graph("quickstart.evhg") == GraphIoStatus::ok, + "graph loaded (no rebuild)"); + + // ------------------------------------------------------------------ + // 4. Search, three ways. The query arrives as floats; quantize it. + // Mode 1: Hamming - fastest, binary-metric ranking. + // Mode 2: asymmetric re-rank - float-grade ranking, zero extra memory. + // Mode 3: exact re-rank - float32-exact results; the float corpus can + // stay on flash (here it is just our in-RAM array). + // ------------------------------------------------------------------ + const float* qf = floats.data() + 7 * dim; // query with vector #7 itself + alignas(8) std::uint8_t qbits[64]; + quantize(qf, dim, qbits); + + SearchResult hits[10]; + std::uint32_t found = graph.search(qbits, 10, /*ef=*/50, hits); + expect(found == 10 && hits[0].id == 7 && hits[0].distance == 0, + "Hamming search: the query's own vector ranks first"); + + ScoredResult best[10]; + found = graph.search_reranked(qbits, qf, 10, 50, best); + expect(found == 10 && best[0].id == 7, + "asymmetric re-rank agrees (dot(q, sign(x)) scoring)"); + + found = graph.search_exact_reranked(qbits, qf, floats.data(), dim, + 10, 100, best); + expect(found == 10 && best[0].id == 7, + "exact re-rank agrees (true float32 cosine over the pool)"); + + // ------------------------------------------------------------------ + // 5. Concurrency: one SearchContext per querying thread. The graph + // itself is shared and const during queries. + // ------------------------------------------------------------------ + SearchContext ctx = graph.make_context(); + found = graph.search(ctx, qbits, 10, 50, hits, nullptr); + expect(found == 10 && hits[0].id == 7, + "context-based search (thread-safe form) agrees"); + + // ------------------------------------------------------------------ + // 6. Filtering: an allow-bitmap restricts results (bit per id). + // ------------------------------------------------------------------ + std::vector allow((n + 63) / 64, 0); + for (std::uint32_t id = 0; id < n; id += 2) // permit even ids only + allow[id >> 6] |= (1ull << (id & 63)); + found = graph.search(ctx, qbits, 10, 100, hits, allow.data()); + bool only_even = (found == 10); + for (std::uint32_t i = 0; i < found; ++i) + if (hits[i].id % 2 != 0) only_even = false; + expect(only_even, "filtered search returns only permitted ids"); + + // ------------------------------------------------------------------ + // 7. Dynamics: soft-delete, then reclaim the slot for a NEW vector + // (remove -> overwrite the bytes at that index -> reinsert). + // ------------------------------------------------------------------ + expect(graph.remove(7), "vector 7 soft-deleted"); + found = graph.search(qbits, 10, 50, hits); + bool absent = true; + for (std::uint32_t i = 0; i < found; ++i) + if (hits[i].id == 7) absent = false; + expect(absent, "deleted vector no longer returned"); + + // NOTE: reclaiming rewrites vector bytes, so it needs a writable block - + // here we rebuild the RAM copy's slot and reinsert in `builder`, which + // owns the writable codes. (A mapped file is read-only by design.) + std::vector fresh(dim); + for (std::size_t d = 0; d < dim; ++d) fresh[d] = gauss(rng); + builder.remove(7); + quantize(fresh.data(), dim, codes + 7 * rb); + expect(builder.reinsert(7), "slot 7 reclaimed for a brand-new vector"); + alignas(8) std::uint8_t fresh_bits[64]; + quantize(fresh.data(), dim, fresh_bits); + found = builder.search(fresh_bits, 1, 50, hits); + expect(found == 1 && hits[0].id == 7 && hits[0].distance == 0, + "reclaimed slot serves the new vector"); + + // ------------------------------------------------------------------ + // Cleanup. + // ------------------------------------------------------------------ + store.close(); + std::remove("quickstart.evec"); + std::remove("quickstart.evhg"); + + std::printf("\n%s\n", g_failures == 0 ? "All quickstart checks passed." + : "QUICKSTART FAILURES - see above."); + return g_failures == 0 ? 0 : 1; +} diff --git a/include/edgevector/edgevector.hpp b/include/edgevector/edgevector.hpp new file mode 100644 index 0000000..faee41e --- /dev/null +++ b/include/edgevector/edgevector.hpp @@ -0,0 +1,29 @@ +#ifndef EDGEVECTOR_EDGEVECTOR_HPP +#define EDGEVECTOR_EDGEVECTOR_HPP + +// ============================================================================ +// EdgeVector :: edgevector.hpp - the single-include umbrella header. +// +// #include +// +// pulls in the whole library: +// quantize_math.hpp - binary quantization, Hamming kernel, asymmetric scorer +// mmap_storage.hpp - zero-copy memory-mapped vector files (EVEC format) +// hnsw_graph.hpp - the HNSW index: build, search, persistence, dynamics +// itq_rotation.hpp - optional learned rotation for anisotropic data +// +// Each header also stands alone; include only what you use if you prefer. +// See examples/quickstart.cpp for a complete, runnable tour. +// ============================================================================ + +#define EDGEVECTOR_VERSION_MAJOR 0 +#define EDGEVECTOR_VERSION_MINOR 8 +#define EDGEVECTOR_VERSION_PATCH 0 +#define EDGEVECTOR_VERSION "0.8.0" + +#include "edgevector/quantize_math.hpp" +#include "edgevector/mmap_storage.hpp" +#include "edgevector/hnsw_graph.hpp" +#include "edgevector/itq_rotation.hpp" + +#endif // EDGEVECTOR_EDGEVECTOR_HPP