From a277b0e22dc463d12b2520f5f1b00d045bc5d112 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lorenz=20H=C3=BCbschle?= Date: Fri, 7 Aug 2026 14:26:58 +0200 Subject: [PATCH 1/2] feat(orc): expose stripe stream ranges and stop seeking to the stripe start A caller that owns its own I/O needs to know which bytes a stripe read will touch before liborc asks for them, so it can fetch them up front and serve the reads from memory. Add ORCFileReader::GetStripeStreamRanges, which reports the data streams of every column liborc selects for a set of include_indices. The selected set is taken from RowReader::getSelectedColumns rather than recomputed, because selection expands beyond the requested indices to their descendants and all of their ancestors -- an ancestor's PRESENT stream carries the nullability of the struct holding a requested leaf -- and a second implementation of that rule could only drift from the one the row reader uses. Building the row reader costs no stripe I/O: it selects off the file footer. Index streams are excluded. They are only needed to seek inside a stripe or to evaluate a search argument, and NextStripeReader was requesting them even when it had neither reason to: it seeks unconditionally, and seekToRow loads the stripe index even for a zero-row seek, which drags in the bloom filters alongside the row indexes. The bloom filters can be an order of magnitude larger than the indexes and nothing reads them without a search argument. Skip the seek when already positioned at the stripe's first row; the row reader starts there regardless, and previousRow_ is only observable through getRowNumber(), which next() recomputes. Also relax CordedRandomAccessFile::ReadAt(position, nbytes, void*) from final to override. The ORC adapter reaches a file through liborc's InputStream::read, which copies into a caller-owned buffer and so can never take the corded path; leaving it final forces such a file to be a separate class. The Buffer-returning overload stays final. --- cpp/src/arrow/adapters/orc/adapter.cc | 64 ++++++++++++++++++++++++++- cpp/src/arrow/adapters/orc/adapter.h | 25 +++++++++++ cpp/src/arrow/io/interfaces.h | 7 ++- 3 files changed, 93 insertions(+), 3 deletions(-) diff --git a/cpp/src/arrow/adapters/orc/adapter.cc b/cpp/src/arrow/adapters/orc/adapter.cc index 51cca497485c..bd5e767a69fb 100644 --- a/cpp/src/arrow/adapters/orc/adapter.cc +++ b/cpp/src/arrow/adapters/orc/adapter.cc @@ -247,6 +247,56 @@ class ORCFileReader::Impl { return stripes_[static_cast(stripe)]; } + Result> GetStripeStreamRanges( + int64_t stripe, const std::vector& include_indices) { + if (stripe < 0 || static_cast(stripe) >= stripes_.size()) { + return Status::Invalid("Out of bounds stripe: ", stripe); + } + + liborc::RowReaderOptions opts = DefaultRowReaderOptions(); + if (!include_indices.empty()) { + RETURN_NOT_OK(SelectIndices(&opts, include_indices)); + } + + std::vector ranges; + + ORC_BEGIN_CATCH_NOT_OK + // Let liborc say which columns these options select, rather than reproducing the + // walk here: selection reaches beyond include_indices to their descendants and + // ancestors, and a second implementation of that could only drift from this one. + // Building the row reader parses no stripe -- it selects off the file footer, and + // the stripe is not opened until the first read. + const std::vector selected = + reader_->createRowReader(opts)->getSelectedColumns(); + + const std::unique_ptr stripe_info = + reader_->getStripe(static_cast(stripe)); + + for (uint64_t i = 0; i < stripe_info->getNumberOfStreams(); ++i) { + const std::unique_ptr stream = + stripe_info->getStreamInformation(i); + const uint64_t column = stream->getColumnId(); + if (column >= selected.size() || !selected[column]) { + continue; + } + switch (stream->getKind()) { + case liborc::StreamKind_PRESENT: + case liborc::StreamKind_DATA: + case liborc::StreamKind_LENGTH: + case liborc::StreamKind_DICTIONARY_DATA: + case liborc::StreamKind_SECONDARY: + ranges.push_back({static_cast(stream->getOffset()), + static_cast(stream->getLength())}); + break; + default: + break; + } + } + ORC_END_CATCH_NOT_OK + + return ranges; + } + FileVersion GetFileVersion() { liborc::FileVersion orc_file_version = reader_->getFormatVersion(); return FileVersion(orc_file_version.getMajor(), orc_file_version.getMinor()); @@ -518,7 +568,14 @@ class ORCFileReader::Impl { ORC_BEGIN_CATCH_NOT_OK row_reader = reader_->createRowReader(opts); - row_reader->seekToRow(current_row_); + // Only seek when we are not already at the start of the selected stripe. The row + // reader begins at the first row of its range anyway, but seekToRow loads the stripe + // index even for a zero-row seek -- and that pulls in the bloom filter streams + // alongside the row indexes, which can dwarf them and which nothing reads unless a + // search argument was set. + if (current_row_ != stripe_info.first_row_id) { + row_reader->seekToRow(current_row_); + } current_row_ = stripe_info.first_row_id + stripe_info.num_rows; ORC_END_CATCH_NOT_OK @@ -638,6 +695,11 @@ StripeInformation ORCFileReader::GetStripeInformation(int64_t stripe) { return impl_->GetStripeInformation(stripe); } +Result> ORCFileReader::GetStripeStreamRanges( + int64_t stripe, const std::vector& include_indices) { + return impl_->GetStripeStreamRanges(stripe, include_indices); +} + FileVersion ORCFileReader::GetFileVersion() { return impl_->GetFileVersion(); } std::string ORCFileReader::GetSoftwareVersion() { return impl_->GetSoftwareVersion(); } diff --git a/cpp/src/arrow/adapters/orc/adapter.h b/cpp/src/arrow/adapters/orc/adapter.h index 4ffff81f355f..a9602db13a4e 100644 --- a/cpp/src/arrow/adapters/orc/adapter.h +++ b/cpp/src/arrow/adapters/orc/adapter.h @@ -183,6 +183,31 @@ class ARROW_EXPORT ORCFileReader { /// \brief StripeInformation for each stripe. StripeInformation GetStripeInformation(int64_t stripe); + /// \brief Byte ranges of the data streams a read of `include_indices` will touch in + /// `stripe`, so that a caller doing its own I/O can fetch them up front. + /// + /// Covers the PRESENT, DATA, LENGTH, DICTIONARY_DATA and SECONDARY streams of every + /// column liborc selects. Selection is wider than `include_indices`: it also takes in + /// their descendants and all of their ancestors, because an ancestor's PRESENT stream + /// carries the nullability of the struct holding the requested leaf. The set is + /// obtained from liborc rather than recomputed here, so it cannot drift from what the + /// row reader goes on to read. + /// + /// Index streams (ROW_INDEX, BLOOM_FILTER*) are deliberately excluded: NextStripeReader + /// only loads them when it has to seek within the stripe or when a search argument was + /// set. Neither applies to a caller reading a stripe from its first row. + /// + /// Requires the stripe footer to be readable, as it names the streams -- callers doing + /// their own I/O must have the range + /// `[offset + index_length + data_length, + footer_length)` available first, which + /// GetStripeInformation reports without reading anything. + /// + /// \param[in] stripe the stripe index + /// \param[in] include_indices the selected field indices, empty to select all + /// \return the byte ranges, in ascending offset order + Result> GetStripeStreamRanges( + int64_t stripe, const std::vector& include_indices); + /// \brief Get the format version of the file. /// Currently known values are 0.11 and 0.12. /// diff --git a/cpp/src/arrow/io/interfaces.h b/cpp/src/arrow/io/interfaces.h index 6d20d04330dd..69f31c9089da 100644 --- a/cpp/src/arrow/io/interfaces.h +++ b/cpp/src/arrow/io/interfaces.h @@ -360,8 +360,11 @@ class ARROW_EXPORT CordedRandomAccessFile : public CordedInputStream, public: virtual Result ReadCordedAt(int64_t position, int64_t nbytes) = 0; - // Provide NotImplemented versions of the non-corded read functions - Result ReadAt(int64_t position, int64_t nbytes, void* out) final; + // Provide NotImplemented versions of the non-corded read functions. The copying + // overload is overridable rather than final: the ORC adapter reaches a file only + // through liborc's InputStream::read, which copies into a caller-owned buffer and + // so cannot use ReadCordedAt. Everything else stays on the corded path. + Result ReadAt(int64_t position, int64_t nbytes, void* out) override; Result> ReadAt(int64_t position, int64_t nbytes) final; }; From c3bd3d806b9bbb3e7557cd99e2e85b361bdaa4ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lorenz=20H=C3=BCbschle?= Date: Fri, 7 Aug 2026 15:47:59 +0200 Subject: [PATCH 2/2] feat(orc): report a stripe's footer range without reading it GetStripeStreamRanges cannot answer until the stripe footer is readable, since the footer is what names the streams. A caller doing its own I/O therefore needs the footer's extent before it has the footer -- and StripeInformation, which it already has, does not carry it. Report it from the file footer, where the offset and the index, data and footer lengths all sit and are already parsed. This reads nothing, so it composes with GetStripeStreamRanges: ask for the range, make it available, then ask what the streams are. --- cpp/src/arrow/adapters/orc/adapter.cc | 21 +++++++++++++++++++++ cpp/src/arrow/adapters/orc/adapter.h | 10 ++++++++++ 2 files changed, 31 insertions(+) diff --git a/cpp/src/arrow/adapters/orc/adapter.cc b/cpp/src/arrow/adapters/orc/adapter.cc index bd5e767a69fb..6351b1911abb 100644 --- a/cpp/src/arrow/adapters/orc/adapter.cc +++ b/cpp/src/arrow/adapters/orc/adapter.cc @@ -247,6 +247,23 @@ class ORCFileReader::Impl { return stripes_[static_cast(stripe)]; } + Result GetStripeFooterRange(int64_t stripe) { + if (stripe < 0 || static_cast(stripe) >= stripes_.size()) { + return Status::Invalid("Out of bounds stripe: ", stripe); + } + io::ReadRange range{}; + ORC_BEGIN_CATCH_NOT_OK + // Offsets and lengths come off the file footer; the stripe footer itself stays + // unread, which is the point -- the caller needs this to make it readable. + const std::unique_ptr info = + reader_->getStripe(static_cast(stripe)); + range.offset = static_cast(info->getOffset() + info->getIndexLength() + + info->getDataLength()); + range.length = static_cast(info->getFooterLength()); + ORC_END_CATCH_NOT_OK + return range; + } + Result> GetStripeStreamRanges( int64_t stripe, const std::vector& include_indices) { if (stripe < 0 || static_cast(stripe) >= stripes_.size()) { @@ -695,6 +712,10 @@ StripeInformation ORCFileReader::GetStripeInformation(int64_t stripe) { return impl_->GetStripeInformation(stripe); } +Result ORCFileReader::GetStripeFooterRange(int64_t stripe) { + return impl_->GetStripeFooterRange(stripe); +} + Result> ORCFileReader::GetStripeStreamRanges( int64_t stripe, const std::vector& include_indices) { return impl_->GetStripeStreamRanges(stripe, include_indices); diff --git a/cpp/src/arrow/adapters/orc/adapter.h b/cpp/src/arrow/adapters/orc/adapter.h index a9602db13a4e..4bfc3304bba3 100644 --- a/cpp/src/arrow/adapters/orc/adapter.h +++ b/cpp/src/arrow/adapters/orc/adapter.h @@ -183,6 +183,16 @@ class ARROW_EXPORT ORCFileReader { /// \brief StripeInformation for each stripe. StripeInformation GetStripeInformation(int64_t stripe); + /// \brief Byte range of a stripe's footer, which is what names its streams. + /// + /// Reported from the file footer alone, so this reads nothing -- which is what lets a + /// caller doing its own I/O make the footer available before calling + /// GetStripeStreamRanges, which cannot answer without it. + /// + /// \param[in] stripe the stripe index + /// \return the byte range of the stripe footer + Result GetStripeFooterRange(int64_t stripe); + /// \brief Byte ranges of the data streams a read of `include_indices` will touch in /// `stripe`, so that a caller doing its own I/O can fetch them up front. ///