Size a segment's head reserve without holding its records

README.md

@@ -85,13 +85,15 @@ let seg = Blob::open(MmapBytes::open(path)?)?;
 seg.read_all(b"term", |v| { /* zero-copy borrow into the mapping */ })?;
 ```
 
-With the whole input in hand, `SegmentWriter::write_sorted` writes the same
-bytes and sizes the segment's head reserve exactly, so a reader's first probe
-covers the index without a second round trip and a small segment does not
-carry a large one's worth of zeroes. `SegmentWrite` carries the per-file
-settings -- compression, inline runs, sync spreading, and whether the reserve
-holds a copy of the directory. `supdb::reserve` answers the sizing question on
-its own, from lengths or from totals.
+A segment's head reserve is what lets a reader's first probe cover the index
+without a second round trip, and it has to be sized before the first key is
+written. `supdb::reserve` computes it rather than guessing, so a small segment
+does not carry a large one's worth of zeroes. With the whole input in hand,
+`SegmentWriter::write_sorted` does it for you. Without: run the lengths through
+`reserve::Planner`, which holds aggregates rather than records, then stream
+through `SegmentWriter::create_with(path, opts, write, reserve)`, which takes
+the per-file settings -- compression, inline runs, sync spreading -- and the
+reserve together, since all of them must be set before the first key.
 
 The same segment in a browser, over ranged HTTP from a Web Worker:
 

src/db.rs

@@ -904,6 +904,52 @@ fn superblock(fields: &[u64; 16]) -> [u8; crate::format::SUPER_BYTES] {
 }
 
 impl SegmentWriter {
+    /// Open `path` for a fresh segment with every per-file setting applied
+    /// and the head reserve set, so nothing is left to remember.
+    ///
+    /// The four things a segment writer can be configured to do -- inline
+    /// runs, compression, spread syncs, the head reserve -- all have to be set
+    /// before the first key, and each is silent when forgotten: a plain
+    /// segment where a compressed one was wanted, or a reserve of zero that
+    /// costs the sparse reader a round trip and raises nothing. Setting them
+    /// at construction is what makes forgetting one impossible.
+    ///
+    /// `reserve` is a number rather than a policy, because the caller may
+    /// know it exactly. `reserve::for_lengths` computes it for input in hand,
+    /// and its `Reserve` prices the hash-directory copy separately:
+    ///
+    /// ```no_run
+    /// # use supdb::{SegmentOptions, SegmentWrite, SegmentWriter, reserve};
+    /// # let (path, opts, write) = (std::path::Path::new("s.sup"), SegmentOptions::default(), SegmentWrite::default());
+    /// # let lengths: Vec<(usize, usize)> = Vec::new();
+    /// let r = reserve::for_lengths(&lengths, opts.block_size, write.inline_max).unwrap();
+    /// // `r.bytes()` for a lookup that plans from the probe; `without_directory`
+    /// // to save four bytes a key and let a lookup fetch the directory itself.
+    /// let w = SegmentWriter::create_with(path, &opts, &write, r.bytes())?;
+    /// # Ok::<(), std::io::Error>(())
+    /// ```
+    pub fn create_with(
+        path: &Path,
+        opts: &SegmentOptions,
+        write: &SegmentWrite,
+        reserve: usize,
+    ) -> Result<SegmentWriter> {
+        let mut w = SegmentWriter::create(path, opts)?;
+        // Every field of `SegmentWrite`, and the reserve. If a setter is
+        // added to this writer without a field beside it here, this stops
+        // compiling, which is the point of the struct.
+        let SegmentWrite {
+            inline_max,
+            compress,
+            sync_every,
+        } = *write;
+        w.set_inline_max(inline_max);
+        w.set_compress(compress);
+        w.set_sync_every(sync_every);
+        w.set_head_reserve(reserve);
+        Ok(w)
+    }
+
     /// Write a whole segment from input already in hand, sizing the head
     /// reserve exactly instead of guessing at it.
     ///
@@ -914,9 +960,10 @@ impl SegmentWriter {
     /// reserve exactly, which is what `reserve::for_lengths` does and what
     /// this does for you.
     ///
-    /// `write` carries the per-file settings; every one of this writer's
-    /// setters has a field there, so nothing it can be configured to do is
-    /// unreachable from here.
+    /// The reserve it computes holds the hash-directory copy, so a lookup
+    /// plans its records from the probe. To trade that for four bytes a key,
+    /// take `reserve::for_lengths(..).without_directory()` and stream through
+    /// [`SegmentWriter::create_with`] instead.
     ///
     /// `items` must be sorted by key, as the streaming API requires. Returns
     /// the reserve it used, since a caller measuring segments wants to know.
@@ -939,21 +986,11 @@ impl SegmentWriter {
         // block count -- and the table sized by it -- is the same either way.
         // What compression moves is where the key section lands, and the row
         // is already taken at its worst alignment.
-        let plan = crate::reserve::for_lengths(&lengths, opts.block_size, write.inline_max)
-            .ok_or_else(|| err("segment writer: this input cannot be a segment"))?;
-        let reserve = if write.directory_in_reserve {
-            plan.bytes()
-        } else {
-            plan.without_directory()
-        };
+        let reserve = crate::reserve::for_lengths(&lengths, opts.block_size, write.inline_max)
+            .ok_or_else(|| err("segment writer: this input cannot be a segment"))?
+            .bytes();
 
-        let mut w = SegmentWriter::create(path, opts)?;
-        // Every setter, in the order they must be called: all of these want
-        // to be set before the first key.
-        w.set_inline_max(write.inline_max);
-        w.set_compress(write.compress);
-        w.set_sync_every(write.sync_every);
-        w.set_head_reserve(reserve);
+        let mut w = SegmentWriter::create_with(path, opts, write, reserve)?;
         for (k, vals) in items {
             w.begin(k)?;
             for v in *vals {
@@ -1571,23 +1608,24 @@ impl SegmentWriter {
     }
 }
 
-/// The per-file settings a segment is written with: every one of
-/// `SegmentWriter`'s setters, gathered so a batch write can apply them.
+/// The per-file settings a segment is written with: one field for every one
+/// of `SegmentWriter`'s setters, and nothing else.
+///
+/// That invariant is the point. Every setter here must be called before the
+/// first key, and forgetting one is silent -- so
+/// [`SegmentWriter::create_with`] takes this struct and applies all of it,
+/// and a new setter that does not appear here is a compile error there rather
+/// than a quiet default. Nothing lives in this struct that `create_with` does
+/// not apply, which is why the choice of what the head reserve holds is not
+/// here: `create_with` is given the reserve as a number.
 ///
 /// These are separate from [`SegmentOptions`] on purpose, and the separation
 /// is the same one that struct's own note draws: `SegmentOptions` is the
 /// engine's configuration, carried to the writer for every piece it seals,
 /// while these describe one file. A term index built to be downloaded wants
 /// compression and inline runs; the segments the seal writes and its own
 /// merge reads back want neither.
-///
-/// It exists because the first `write_sorted` took `inline_max` as a bare
-/// argument and had nowhere to put the rest, so compression silently did
-/// nothing -- which is the failure `SegmentOptions` refuses a compression
-/// field to avoid, reproduced one layer up. A struct with a field per setter
-/// makes the next setter's absence a compile error in
-/// `SegmentWriter::write_sorted` rather than a quiet default.
-#[derive(Clone, Debug)]
+#[derive(Clone, Debug, Default)]
 pub struct SegmentWrite {
     /// Runs up to this many bytes go inline in the index record, and the
     /// segment is written records-first so they stream. Zero keeps every run
@@ -1598,23 +1636,6 @@ pub struct SegmentWrite {
     /// fdatasync every this many bytes of blocks rather than once at the end.
     /// Zero for the single sync.
     pub sync_every: usize,
-    /// Put a copy of the hash directory in the head reserve. Four bytes a
-    /// key, and it is the difference between a lookup that plans its records
-    /// from the probe and one that fetches the directory first. On by
-    /// default: a segment written through this path is one whose whole input
-    /// was in hand, which is the shape that gets downloaded and read cold.
-    pub directory_in_reserve: bool,
-}
-
-impl Default for SegmentWrite {
-    fn default() -> SegmentWrite {
-        SegmentWrite {
-            inline_max: 0,
-            compress: false,
-            sync_every: 0,
-            directory_in_reserve: true,
-        }
-    }
 }
 
 /// How a segment file is written.

src/flatindex.rs

@@ -91,7 +91,7 @@ const FENCE_MIN_STRIDE: usize = 16;
 
 /// Stride for `n` keys: enough entries to narrow the search hard, few enough
 /// that the fence stays small enough to sit in cache.
-fn fence_stride(n: usize) -> usize {
+pub fn fence_stride(n: usize) -> usize {
     let want = n.div_ceil(fence_target()).max(FENCE_MIN_STRIDE);
     want.next_power_of_two()
 }
@@ -490,7 +490,7 @@ fn record_len(klen: usize, next: usize) -> usize {
 
 /// `record_len` plus the record's tail: the bytes of its inline runs, padded
 /// so the next record stays 4-aligned.
-fn record_len_tail(klen: usize, next: usize, tail: usize) -> usize {
+pub fn record_len_tail(klen: usize, next: usize, tail: usize) -> usize {
     record_len(klen, next) + align_up(tail, REC_ALIGN)
 }
 
@@ -653,12 +653,6 @@ pub fn plan_inline(
     insert_slack: usize,
     record_slack: bool,
 ) -> Option<Plan> {
-    let mut cap = 1usize;
-    while cap < all.len() * 2 {
-        cap = cap.checked_mul(2)?;
-    }
-    cap = cap.max(16);
-
     let mut rec_offs = Vec::with_capacity(all.len());
     let mut at = 0usize;
     for (k, exts) in all {
@@ -676,38 +670,95 @@ pub fn plan_inline(
         let tail = tails.get(rec_offs.len() - 1).map_or(0, |t| t.len());
         at = at.checked_add(record_len_tail(k.len(), n, tail))?;
     }
-    if at > MAX_RECS {
+    // The fence samples every `stride`-th key. `fence_n + 1` offsets, so an
+    // entry's key is the span between its offset and the next.
+    let stride = fence_stride(all.len());
+    let fence_n = all.len().div_ceil(stride);
+    let fence_blob_len: usize = (0..fence_n)
+        .map(|i| all[i * stride].0.len())
+        .try_fold(0usize, |a, b| a.checked_add(b))?;
+
+    let l = section_layout(
+        all.len(),
+        at,
+        fence_n,
+        fence_blob_len,
+        insert_slack,
+        record_slack,
+    )?;
+    Some(Plan {
+        hash_cap: l.hash_cap,
+        dir_cap: l.dir_cap,
+        recs_len: at,
+        recs_cap: l.recs_cap,
+        rec_offs,
+        fence_n,
+        fence_stride: stride,
+        fence_offs_off: l.fence_offs_off,
+        fence_blob_off: l.fence_blob_off,
+        fence_blob_len,
+        total: l.total,
+        written: l.recs_off + at,
+    })
+}
+
+/// Where a section's regions land, from the four aggregates that decide them:
+/// the key count, the record bytes, and the fence's entry count and blob
+/// length. Nothing here needs the keys themselves.
+///
+/// Public and separate because the head reserve has to be sized before a key
+/// is written, and a caller streaming its input cannot hand over slices of
+/// everything. `plan_inline` calls this after counting; `reserve::Planner`
+/// calls it after accumulating. Two copies of this arithmetic would be two
+/// definitions of the section, and they would drift.
+pub struct SectionLayout {
+    pub hash_cap: usize,
+    pub dir_cap: usize,
+    pub fence_offs_off: usize,
+    pub fence_blob_off: usize,
+    pub recs_off: usize,
+    pub recs_cap: usize,
+    pub total: usize,
+}
+
+pub fn section_layout(
+    keys: usize,
+    rec_bytes: usize,
+    fence_n: usize,
+    fence_blob_len: usize,
+    insert_slack: usize,
+    record_slack: bool,
+) -> Option<SectionLayout> {
+    if rec_bytes > MAX_RECS {
         return None;
     }
+    let mut cap = 1usize;
+    while cap < keys * 2 {
+        cap = cap.checked_mul(2)?;
+    }
+    cap = cap.max(16);
+
     // Half again, so a store whose keys gain extents can publish updates
     // without rewriting anything -- unless the caller says the section is
     // never edited in place.
     let slack = if record_slack {
-        at * SLACK_NUM / SLACK_DEN
+        rec_bytes * SLACK_NUM / SLACK_DEN
     } else {
         0
     };
-    let recs_cap = at.checked_add(slack)?;
+    let recs_cap = rec_bytes.checked_add(slack)?;
     if recs_cap > MAX_RECS {
         return None;
     }
 
-    // The fence samples every `stride`-th key. `fence_n + 1` offsets, so an
-    // entry's key is the span between its offset and the next.
-    let stride = fence_stride(all.len());
-    let fence_n = all.len().div_ceil(stride);
-    let fence_blob_len: usize = (0..fence_n)
-        .map(|i| all[i * stride].0.len())
-        .try_fold(0usize, |a, b| a.checked_add(b))?;
-
     // One buffer when nothing asked for insert room, two when something did.
     let dir_cap = if insert_slack == 0 {
         0
     } else {
-        all.len().checked_add(insert_slack)?
+        keys.checked_add(insert_slack)?
     };
     let dir_bytes = if dir_cap == 0 {
-        all.len() * 4
+        keys * 4
     } else {
         dir_cap.checked_mul(8)?
     };
@@ -717,20 +768,14 @@ pub fn plan_inline(
     // Records are 4-aligned within the section, and the blob is bytes, so the
     // record region is realigned after it.
     let recs_off = align_up(fence_blob_off + fence_blob_len, REC_ALIGN);
-    let total = recs_off + recs_cap;
-    Some(Plan {
+    Some(SectionLayout {
         hash_cap: cap,
         dir_cap,
-        recs_len: at,
-        recs_cap,
-        rec_offs,
-        fence_n,
-        fence_stride: stride,
         fence_offs_off,
         fence_blob_off,
-        fence_blob_len,
-        total,
-        written: recs_off + at,
+        recs_off,
+        recs_cap,
+        total: recs_off + recs_cap,
     })
 }