Selectivity-adaptive filtered search: sparse filters get exact results

include/edgevector/hnsw_graph.hpp

@@ -40,9 +40,17 @@
 //                       memory. Requires the float query alongside the
 //                       quantized one.
 // Both accept an optional caller-owned allow-bitmap (bit id set = id may be
-// returned) and always exclude soft-deleted nodes; filtered-out nodes still
-// route traversal. NOTE: a highly selective filter degrades toward a scan of
-// the reachable graph, as in every filtered-HNSW implementation.
+// returned) and always exclude soft-deleted nodes.
+//
+// FILTERED SEARCH IS SELECTIVITY-ADAPTIVE. A sparse filter makes graph
+// traversal the wrong algorithm: too few eligible nodes ever enter the beam,
+// pruning never engages, and the walk decays toward scanning the reachable
+// graph - expensively and approximately. So when the allowed population is
+// small relative to the beam (popcount(allow) <= 16 * ef; the popcount is a
+// ~microsecond pass over the bitmap), the search switches to a direct scan
+// of the allowed ids, which is both cheaper AND exact - recall 1.0 over the
+// allowed set by construction. Larger filters keep the graph traversal, with
+// filtered-out nodes still routing. Both paths are zero-allocation.
 //
 // DELETION AND SLOT RECLAMATION
 // -----------------------------
@@ -631,7 +639,7 @@ class HNSWGraph {
         }
         clamp_kef(k, ef);
 
-        std::uint32_t found = beam_layer0(ctx, q, ef, allow);
+        std::uint32_t found = collect_pool(ctx, q, ef, allow);
         while (found > k) { // keep only the k best
             ctx.res_heap_.pop();
             --found;
@@ -671,7 +679,7 @@ class HNSWGraph {
         }
         clamp_kef(k, ef);
 
-        const std::uint32_t found = beam_layer0(ctx, q_bits, ef, allow);
+        const std::uint32_t found = collect_pool(ctx, q_bits, ef, allow);
         if (found == 0u) {
             return 0u;
         }
@@ -738,7 +746,7 @@ class HNSWGraph {
         }
         clamp_kef(k, ef);
 
-        const std::uint32_t found = beam_layer0(ctx, q_bits, ef, allow);
+        const std::uint32_t found = collect_pool(ctx, q_bits, ef, allow);
         if (found == 0u) {
             return 0u;
         }
@@ -1305,6 +1313,74 @@ class HNSWGraph {
         return search_layer(ctx, q, ep, ef, 0u, allow, true);
     }
 
+    // Population of an allow-bitmap over the valid id range (bits beyond
+    // capacity are masked off, whatever the caller left there).
+    std::uint64_t popcount_allow(const std::uint64_t* allow) const noexcept {
+        const std::size_t full = static_cast<std::size_t>(capacity_) / 64u;
+        std::uint64_t total = 0u;
+        for (std::size_t w = 0u; w < full; ++w) {
+            total += static_cast<std::uint64_t>(__builtin_popcountll(allow[w]));
+        }
+        const std::uint32_t rem = capacity_ & 63u;
+        if (rem != 0u) {
+            const std::uint64_t mask = (1ull << rem) - 1ull;
+            total += static_cast<std::uint64_t>(
+                __builtin_popcountll(allow[full] & mask));
+        }
+        return total;
+    }
+
+    // Exact scan over the allowed ids: the optimal strategy for sparse
+    // filters. Fills ctx.res_heap_ with the ef best eligible candidates -
+    // EXACT over the allowed set, since every allowed id is scored. Ids are
+    // visited ascending and ties skip the later id, which reproduces the
+    // (distance, id) selection order of the brute-force baseline exactly.
+    // Zero allocation.
+    std::uint32_t filtered_scan(SearchContext& ctx, const std::uint8_t* q,
+                                std::uint32_t ef,
+                                const std::uint64_t* allow) const noexcept {
+        ctx.res_heap_.clear();
+        const std::size_t words =
+            (static_cast<std::size_t>(capacity_) + 63u) / 64u;
+        const std::uint32_t rem = capacity_ & 63u;
+        for (std::size_t w = 0u; w < words; ++w) {
+            std::uint64_t bits = allow[w];
+            if (rem != 0u && w == words - 1u) {
+                bits &= (1ull << rem) - 1ull;
+            }
+            while (bits != 0ull) {
+                const std::uint64_t low = bits & (0ull - bits);
+                bits ^= low;
+                const std::uint32_t id = static_cast<std::uint32_t>(
+                    w * 64u + static_cast<std::size_t>(__builtin_ctzll(low)));
+                if (levels_[id] == kNotInserted || is_deleted(id)) {
+                    continue;
+                }
+                const std::uint32_t d = distance_to(q, id);
+                if (ctx.res_heap_.size() < ef) {
+                    ctx.res_heap_.push(detail::Candidate{d, id});
+                } else if (d < ctx.res_heap_.top().dist) {
+                    ctx.res_heap_.push(detail::Candidate{d, id});
+                    ctx.res_heap_.pop();
+                }
+            }
+        }
+        return ctx.res_heap_.size();
+    }
+
+    // The candidate-pool collector behind all three search modes: picks the
+    // scan for sparse filters (exact and cheaper), the beam otherwise.
+    std::uint32_t collect_pool(SearchContext& ctx, const std::uint8_t* q,
+                               std::uint32_t ef,
+                               const std::uint64_t* allow) const noexcept {
+        if (allow != nullptr &&
+            popcount_allow(allow) <=
+                static_cast<std::uint64_t>(ef) * 16u) {
+            return filtered_scan(ctx, q, ef, allow);
+        }
+        return beam_layer0(ctx, q, ef, allow);
+    }
+
     // Beam search on one layer (Algorithm 2), with result-eligibility
     // filtering: ineligible nodes (tombstoned, or cleared in `allow`) still
     // route within the beam bound but never enter the result heap. Results

tests/benchmark_100k.cpp

@@ -371,6 +371,67 @@ void run_scenario(const char* name, bool clustered, bool full_report) {
     }
     std::printf("\n");
     std::fflush(stdout);
+
+    // Filtered search across selectivities (allow every stride-th id).
+    // Recall is measured against the exact brute-force top-10 over the
+    // allowed set; sparse filters take the adaptive exact-scan path.
+    if (full_report) {
+        std::printf("Filtered search (ef = 100, k = 10, 300 queries; recall "
+                    "vs exact filtered ground truth):\n\n");
+        std::printf("| selectivity | allowed ids | recall@10 | mean latency | QPS (1 thread) |\n");
+        std::printf("|---|---|---|---|---|\n");
+        const std::size_t strides[] = {1000u, 100u, 10u}; // 0.1%, 1%, 10%
+        const std::size_t nq = 300;
+        edgevector::SearchContext fctx = graph.make_context();
+        for (const std::size_t stride : strides) {
+            std::vector<std::uint64_t> af((kN + 63u) / 64u, 0u);
+            std::vector<std::uint32_t> allowed;
+            for (std::size_t id = 0; id < kN; id += stride) {
+                af[id >> 6u] |= (1ull << (id & 63u));
+                allowed.push_back(static_cast<std::uint32_t>(id));
+            }
+
+            std::size_t fhits = 0;
+            std::vector<std::pair<std::uint32_t, std::uint32_t>> brute;
+            brute.reserve(allowed.size());
+            double total_s = 0.0;
+            for (std::size_t qi = 0; qi < nq; ++qi) {
+                const std::uint8_t* q = qbase + qi * qstride;
+                t0 = Clock::now();
+                const std::uint32_t found = graph.search(
+                    fctx, q, kK, 100u, results.data(), af.data());
+                total_s += seconds_since(t0);
+
+                brute.clear();
+                for (const std::uint32_t id : allowed) {
+                    brute.emplace_back(
+                        edgevector::hamming_distance(q, data.record(id), kDim),
+                        id);
+                }
+                std::partial_sort(brute.begin(),
+                                  brute.begin() +
+                                      static_cast<std::ptrdiff_t>(kK),
+                                  brute.end());
+                for (std::uint32_t i = 0; i < found; ++i) {
+                    got_ids[i] = results[i].id;
+                }
+                std::uint32_t truth10[kK];
+                for (std::size_t t = 0; t < kK; ++t) {
+                    truth10[t] = brute[t].second;
+                }
+                fhits += overlap10(got_ids, found, truth10);
+            }
+            std::printf("| %.1f%% | %zu | %.3f | %.0f us | %.0f |\n",
+                        100.0 / static_cast<double>(stride), allowed.size(),
+                        static_cast<double>(fhits) /
+                            static_cast<double>(nq * kK),
+                        1.0e6 * total_s / static_cast<double>(nq),
+                        static_cast<double>(nq) / total_s);
+            std::fflush(stdout);
+        }
+        std::printf("\n");
+        std::fflush(stdout);
+    }
 }
 
 // ---------------------------------------------------------------------------

tests/test_hnsw_graph.cpp

@@ -488,6 +488,56 @@ void test_delete_and_filter(std::size_t dim) {
         }
     }
     check(composed, "deletion and allow-bitmap compose correctly");
+
+    // Sparse filters take the exact-scan path: results must EQUAL the
+    // brute-force top-k over the allowed set, ids and distances both.
+    std::vector<std::uint64_t> sparse((n + 63u) / 64u, 0u);
+    std::vector<std::uint32_t> allowed_ids;
+    for (std::uint32_t id = 4u; id < n; id += 17u) { // ~18 sparse survivors
+        sparse[id >> 6u] |= (1ull << (id & 63u));
+        allowed_ids.push_back(id);
+    }
+    found = graph.search(ctx, q, 10u, 100u, res, sparse.data());
+
+    std::vector<std::pair<std::uint32_t, std::uint32_t>> brute;
+    for (const std::uint32_t id : allowed_ids) {
+        if (graph.is_deleted(id)) {
+            continue;
+        }
+        brute.emplace_back(
+            edgevector::hamming_distance(q, data.record(id), dim), id);
+    }
+    std::sort(brute.begin(), brute.end());
+    bool exact = (found == 10u && brute.size() >= 10u);
+    for (std::uint32_t i = 0; exact && i < found; ++i) {
+        exact = (res[i].id == brute[i].second) &&
+                (res[i].distance == brute[i].first);
+    }
+    check(exact,
+          "sparse filter returns the EXACT brute-force top-10 (scan path)");
+
+    // The re-ranked modes run over the same exact pool.
+    edgevector::ScoredResult sres[10];
+    std::vector<float> qraw(dim, 0.25f); // any float query paired with q bits
+    const std::uint32_t sf = graph.search_reranked(
+        ctx, q, qraw.data(), 10u, 100u, sres, sparse.data());
+    bool subset = (sf == 10u);
+    for (std::uint32_t i = 0; subset && i < sf; ++i) {
+        bool in_allowed = false;
+        for (const std::uint32_t id : allowed_ids) {
+            if (sres[i].id == id) {
+                in_allowed = true;
+                break;
+            }
+        }
+        subset = in_allowed;
+    }
+    check(subset, "re-ranked sparse-filtered results stay within the filter");
+
+    // An empty filter returns nothing, quickly.
+    std::vector<std::uint64_t> empty((n + 63u) / 64u, 0u);
+    check(graph.search(ctx, q, 10u, 100u, res, empty.data()) == 0u,
+          "empty allow-bitmap returns 0 results");
 }
 
 // ---------------------------------------------------------------------------