diff --git a/cpp/src/arrow/adapters/orc/adapter.cc b/cpp/src/arrow/adapters/orc/adapter.cc index 51cca497485c..6351b1911abb 100644 --- a/cpp/src/arrow/adapters/orc/adapter.cc +++ b/cpp/src/arrow/adapters/orc/adapter.cc @@ -247,6 +247,73 @@ class ORCFileReader::Impl { return stripes_[static_cast(stripe)]; } + Result GetStripeFooterRange(int64_t stripe) { + if (stripe < 0 || static_cast(stripe) >= stripes_.size()) { + return Status::Invalid("Out of bounds stripe: ", stripe); + } + io::ReadRange range{}; + ORC_BEGIN_CATCH_NOT_OK + // Offsets and lengths come off the file footer; the stripe footer itself stays + // unread, which is the point -- the caller needs this to make it readable. + const std::unique_ptr info = + reader_->getStripe(static_cast(stripe)); + range.offset = static_cast(info->getOffset() + info->getIndexLength() + + info->getDataLength()); + range.length = static_cast(info->getFooterLength()); + ORC_END_CATCH_NOT_OK + return range; + } + + Result> GetStripeStreamRanges( + int64_t stripe, const std::vector& include_indices) { + if (stripe < 0 || static_cast(stripe) >= stripes_.size()) { + return Status::Invalid("Out of bounds stripe: ", stripe); + } + + liborc::RowReaderOptions opts = DefaultRowReaderOptions(); + if (!include_indices.empty()) { + RETURN_NOT_OK(SelectIndices(&opts, include_indices)); + } + + std::vector ranges; + + ORC_BEGIN_CATCH_NOT_OK + // Let liborc say which columns these options select, rather than reproducing the + // walk here: selection reaches beyond include_indices to their descendants and + // ancestors, and a second implementation of that could only drift from this one. + // Building the row reader parses no stripe -- it selects off the file footer, and + // the stripe is not opened until the first read. + const std::vector selected = + reader_->createRowReader(opts)->getSelectedColumns(); + + const std::unique_ptr stripe_info = + reader_->getStripe(static_cast(stripe)); + + for (uint64_t i = 0; i < stripe_info->getNumberOfStreams(); ++i) { + const std::unique_ptr stream = + stripe_info->getStreamInformation(i); + const uint64_t column = stream->getColumnId(); + if (column >= selected.size() || !selected[column]) { + continue; + } + switch (stream->getKind()) { + case liborc::StreamKind_PRESENT: + case liborc::StreamKind_DATA: + case liborc::StreamKind_LENGTH: + case liborc::StreamKind_DICTIONARY_DATA: + case liborc::StreamKind_SECONDARY: + ranges.push_back({static_cast(stream->getOffset()), + static_cast(stream->getLength())}); + break; + default: + break; + } + } + ORC_END_CATCH_NOT_OK + + return ranges; + } + FileVersion GetFileVersion() { liborc::FileVersion orc_file_version = reader_->getFormatVersion(); return FileVersion(orc_file_version.getMajor(), orc_file_version.getMinor()); @@ -518,7 +585,14 @@ class ORCFileReader::Impl { ORC_BEGIN_CATCH_NOT_OK row_reader = reader_->createRowReader(opts); - row_reader->seekToRow(current_row_); + // Only seek when we are not already at the start of the selected stripe. The row + // reader begins at the first row of its range anyway, but seekToRow loads the stripe + // index even for a zero-row seek -- and that pulls in the bloom filter streams + // alongside the row indexes, which can dwarf them and which nothing reads unless a + // search argument was set. + if (current_row_ != stripe_info.first_row_id) { + row_reader->seekToRow(current_row_); + } current_row_ = stripe_info.first_row_id + stripe_info.num_rows; ORC_END_CATCH_NOT_OK @@ -638,6 +712,15 @@ StripeInformation ORCFileReader::GetStripeInformation(int64_t stripe) { return impl_->GetStripeInformation(stripe); } +Result ORCFileReader::GetStripeFooterRange(int64_t stripe) { + return impl_->GetStripeFooterRange(stripe); +} + +Result> ORCFileReader::GetStripeStreamRanges( + int64_t stripe, const std::vector& include_indices) { + return impl_->GetStripeStreamRanges(stripe, include_indices); +} + FileVersion ORCFileReader::GetFileVersion() { return impl_->GetFileVersion(); } std::string ORCFileReader::GetSoftwareVersion() { return impl_->GetSoftwareVersion(); } diff --git a/cpp/src/arrow/adapters/orc/adapter.h b/cpp/src/arrow/adapters/orc/adapter.h index 4ffff81f355f..4bfc3304bba3 100644 --- a/cpp/src/arrow/adapters/orc/adapter.h +++ b/cpp/src/arrow/adapters/orc/adapter.h @@ -183,6 +183,41 @@ class ARROW_EXPORT ORCFileReader { /// \brief StripeInformation for each stripe. StripeInformation GetStripeInformation(int64_t stripe); + /// \brief Byte range of a stripe's footer, which is what names its streams. + /// + /// Reported from the file footer alone, so this reads nothing -- which is what lets a + /// caller doing its own I/O make the footer available before calling + /// GetStripeStreamRanges, which cannot answer without it. + /// + /// \param[in] stripe the stripe index + /// \return the byte range of the stripe footer + Result GetStripeFooterRange(int64_t stripe); + + /// \brief Byte ranges of the data streams a read of `include_indices` will touch in + /// `stripe`, so that a caller doing its own I/O can fetch them up front. + /// + /// Covers the PRESENT, DATA, LENGTH, DICTIONARY_DATA and SECONDARY streams of every + /// column liborc selects. Selection is wider than `include_indices`: it also takes in + /// their descendants and all of their ancestors, because an ancestor's PRESENT stream + /// carries the nullability of the struct holding the requested leaf. The set is + /// obtained from liborc rather than recomputed here, so it cannot drift from what the + /// row reader goes on to read. + /// + /// Index streams (ROW_INDEX, BLOOM_FILTER*) are deliberately excluded: NextStripeReader + /// only loads them when it has to seek within the stripe or when a search argument was + /// set. Neither applies to a caller reading a stripe from its first row. + /// + /// Requires the stripe footer to be readable, as it names the streams -- callers doing + /// their own I/O must have the range + /// `[offset + index_length + data_length, + footer_length)` available first, which + /// GetStripeInformation reports without reading anything. + /// + /// \param[in] stripe the stripe index + /// \param[in] include_indices the selected field indices, empty to select all + /// \return the byte ranges, in ascending offset order + Result> GetStripeStreamRanges( + int64_t stripe, const std::vector& include_indices); + /// \brief Get the format version of the file. /// Currently known values are 0.11 and 0.12. /// diff --git a/cpp/src/arrow/io/interfaces.h b/cpp/src/arrow/io/interfaces.h index 6d20d04330dd..69f31c9089da 100644 --- a/cpp/src/arrow/io/interfaces.h +++ b/cpp/src/arrow/io/interfaces.h @@ -360,8 +360,11 @@ class ARROW_EXPORT CordedRandomAccessFile : public CordedInputStream, public: virtual Result ReadCordedAt(int64_t position, int64_t nbytes) = 0; - // Provide NotImplemented versions of the non-corded read functions - Result ReadAt(int64_t position, int64_t nbytes, void* out) final; + // Provide NotImplemented versions of the non-corded read functions. The copying + // overload is overridable rather than final: the ORC adapter reaches a file only + // through liborc's InputStream::read, which copies into a caller-owned buffer and + // so cannot use ReadCordedAt. Everything else stays on the corded path. + Result ReadAt(int64_t position, int64_t nbytes, void* out) override; Result> ReadAt(int64_t position, int64_t nbytes) final; };