From f5a4ed42b14217b92ce8a5baa3a228db7a3fa829 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Wed, 19 Aug 2026 13:14:22 -0700 Subject: [PATCH 01/30] Implemented by claude: added a Python interface for Monarch3 --- CMakeLists.txt | 13 + Documentation/UsageMonarch3Python.rst | 243 +++++++++++++++++++ Documentation/index.rst | 1 + Monarch3/CMakeLists.txt | 5 + Monarch3/python/CMakeLists.txt | 37 +++ Monarch3/python/add_lib_python_path.sh.in | 3 + Monarch3/python/m3constants_pybind.hh | 44 ++++ Monarch3/python/m3header_pybind.hh | 145 +++++++++++ Monarch3/python/m3monarch_pybind.hh | 146 +++++++++++ Monarch3/python/m3record_pybind.hh | 47 ++++ Monarch3/python/m3stream_pybind.hh | 138 +++++++++++ Monarch3/python/monarch3_binding_helpers.hh | 15 ++ Monarch3/python/monarch3_namespace_pybind.cc | 45 ++++ 13 files changed, 882 insertions(+) create mode 100644 Documentation/UsageMonarch3Python.rst create mode 100644 Monarch3/python/CMakeLists.txt create mode 100644 Monarch3/python/add_lib_python_path.sh.in create mode 100644 Monarch3/python/m3constants_pybind.hh create mode 100644 Monarch3/python/m3header_pybind.hh create mode 100644 Monarch3/python/m3monarch_pybind.hh create mode 100644 Monarch3/python/m3record_pybind.hh create mode 100644 Monarch3/python/m3stream_pybind.hh create mode 100644 Monarch3/python/monarch3_binding_helpers.hh create mode 100644 Monarch3/python/monarch3_namespace_pybind.cc diff --git a/CMakeLists.txt b/CMakeLists.txt index 3352070..6be6789 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -18,6 +18,7 @@ add_definitions( -DEgg_VERSION=${Egg_VERSION} ) # Monarch library options set( Monarch_BUILD_MONARCH2 FALSE CACHE BOOL "Build Monarch2 library (requires Protobuf)" ) set( Monarch_BUILD_MONARCH3 TRUE CACHE BOOL "Build Monarch3 library (requires HDF5)" ) +set( Monarch_BUILD_PYTHON FALSE CACHE BOOL "Build Monarch3 Python bindings (requires pybind11)" ) ######################## @@ -132,6 +133,18 @@ if( Monarch_BUILD_MONARCH2 ) endif( Monarch_BUILD_MONARCH2 ) +########## +# Python # +########## + +if( Monarch_BUILD_PYTHON ) + find_package( Python3 REQUIRED COMPONENTS Interpreter Development ) + set( PYBIND11_PYTHON_VERSION ${Python3_VERSION} CACHE STRING "" ) + find_package( pybind11 REQUIRED ) + include_directories( ${Python3_INCLUDE_DIRS} ) +endif( Monarch_BUILD_PYTHON ) + + ##################### # Prepare for Build # ##################### diff --git a/Documentation/UsageMonarch3Python.rst b/Documentation/UsageMonarch3Python.rst new file mode 100644 index 0000000..0deaa94 --- /dev/null +++ b/Documentation/UsageMonarch3Python.rst @@ -0,0 +1,243 @@ +How to use Monarch3 from Python +================================ + +The ``monarch3`` Python module is a pybind11 binding of the Monarch3 C++ library. +It exposes the same read/write workflow as the C++ API but with Pythonic conventions: +method and property names use ``snake_case``, raw data buffers are returned as +``numpy.ndarray`` views (``uint8`` by default), and files can be managed with Python's +``with`` statement. + +.. note:: + Record data is returned as a ``numpy.ndarray`` of ``uint8`` bytes. + Use ``array.view(dtype)`` to reinterpret the buffer as the appropriate element type + (e.g. ``np.uint16``, ``np.float32``). The element type and size are available from + the stream header's ``data_type_size`` and ``data_format`` properties. + +Thread safety follows the same rules as the C++ library: calling ``read_record()`` and +``write_record()`` is thread-safe (the GIL is released during disk I/O), but all other +operations are not thread-safe. + + +Constants +--------- + +The following constants are provided by the ``monarch3`` module: + +**Data format** + +* ``sDigitizedUS`` -- unsigned integer samples +* ``sDigitizedS`` -- signed integer samples +* ``sAnalog`` -- floating-point samples + +**Bit alignment** + +* ``sBitsAlignedLeft`` -- significant bits are aligned to the MSB of the sample word +* ``sBitsAlignedRight`` -- significant bits are aligned to the LSB of the sample word + +**Channel format (multi-channel streams)** + +* ``sInterleaved`` -- channel samples are interleaved within the stream record +* ``sSeparate`` -- each channel has its own contiguous block within the stream record + + +Reading Egg3 Files +------------------ + +1. Open the file:: + + m = monarch3.Monarch3.open_for_reading( filename ) + + Or use the context manager (recommended; closes the file automatically):: + + with monarch3.Monarch3.open_for_reading( filename ) as m: + ... + +2. Read the header:: + + m.read_header() + +3. Inspect the header:: + + hdr = m.get_header() + print( hdr.egg_version ) + print( hdr.run_duration ) + print( hdr.timestamp ) + print( hdr.description ) + + Stream and channel metadata are available through header lists:: + + stream_hdr = hdr.stream_headers[0] + print( stream_hdr.n_channels, stream_hdr.acquisition_rate, stream_hdr.record_size ) + print( stream_hdr.data_type_size, stream_hdr.data_format ) + + channel_hdr = hdr.channel_headers[0] + print( channel_hdr.voltage_offset, channel_hdr.voltage_range ) + +4. Get a stream:: + + stream = m.get_stream( stream_index ) + n_records = stream.n_records_in_file + +5. Access data record by record. + Use ``get_channel_data( channel )`` to obtain a ``numpy.ndarray`` of ``uint8`` bytes + for a single channel, or ``get_stream_data()`` for the full (potentially interleaved) + stream buffer:: + + while stream.read_record(): + arr = stream.get_channel_data( 0 ) + # reinterpret bytes as the actual element type, e.g.: + samples = arr.view( np.uint16 ) + print( samples ) + + The ``offset`` parameter to ``read_record`` controls navigation within the file + (same semantics as the C++ API): + + * ``offset == 0`` (default): advance to the next record + * ``offset == -1``: re-read the current record + * ``offset < -1``: step backward + * ``offset > 0``: skip forward + + ``read_record()`` returns ``True`` on success and ``False`` when the requested + position is past the end (or before the beginning) of the file. + +6. Close the file (not needed when using the ``with`` statement):: + + m.finish_reading() + + +Writing Egg3 Files +------------------ + +1. Open the file:: + + m = monarch3.Monarch3.open_for_writing( filename ) + + Or use the context manager:: + + with monarch3.Monarch3.open_for_writing( filename ) as m: + ... + +2. Configure the header:: + + hdr = m.get_header() + hdr.filename = filename + hdr.run_duration = 1000 # milliseconds + hdr.timestamp = "2024-01-01T00:00:00" + hdr.description = "My data" + +3. Add streams. + For a single-channel stream:: + + stream_num = hdr.add_stream( + source = "my-digitizer", + acq_rate = 200, # MHz + rec_size = 4096, # samples per record + sample_size = 1, # elements per sample + data_type_size = 2, # bytes per element + data_format = monarch3.sDigitizedUS, + bit_depth = 14, + bit_alignment = monarch3.sBitsAlignedRight, + ) + + For a multi-channel stream:: + + stream_num = hdr.add_stream( + source = "my-2ch-digitizer", + n_channels = 2, + channel_format = monarch3.sInterleaved, + acq_rate = 200, + rec_size = 4096, + sample_size = 1, + data_type_size = 2, + data_format = monarch3.sDigitizedUS, + bit_depth = 14, + bit_alignment = monarch3.sBitsAlignedRight, + ) + +4. Write the header (this creates the HDF5 structure and allocates stream objects):: + + m.write_header() + +5. Get the stream object:: + + stream = m.get_stream( stream_num ) + +6. For each record, fill the data buffer and write to disk:: + + # get a writable uint8 view; reinterpret as the target element type + buf = stream.get_channel_data( 0 ).view( np.uint16 ) + buf[:] = my_samples # numpy array of uint16 + + stream.write_record( is_new_acquisition=True ) # True = start new acquisition + stream.write_record( is_new_acquisition=False ) # False = continue acquisition + +7. Close the file (not needed when using the ``with`` statement):: + + m.finish_writing() + + +Complete Examples +----------------- + +**Writing a file** + +.. code-block:: python + + import monarch3 + import numpy as np + + filename = "output.egg" + + with monarch3.Monarch3.open_for_writing( filename ) as m: + hdr = m.get_header() + hdr.filename = filename + hdr.run_duration = 1000 + hdr.timestamp = "2024-01-01T00:00:00" + hdr.description = "Example egg file" + + stream_num = hdr.add_stream( + source = "my-digitizer", + acq_rate = 200, + rec_size = 1024, + sample_size = 1, + data_type_size = 1, + data_format = monarch3.sDigitizedUS, + bit_depth = 8, + bit_alignment = monarch3.sBitsAlignedLeft, + ) + + m.write_header() + + stream = m.get_stream( stream_num ) + + # Acquisition 0, two records + buf = stream.get_channel_data( 0 ) + buf[:] = 42 + stream.write_record( True ) + + buf[:] = 100 + stream.write_record( False ) + + +**Reading a file** + +.. code-block:: python + + import monarch3 + import numpy as np + + with monarch3.Monarch3.open_for_reading( "output.egg" ) as m: + m.read_header() + hdr = m.get_header() + print( "Egg version:", hdr.egg_version ) + print( "Timestamp: ", hdr.timestamp ) + + stream_hdr = hdr.stream_headers[0] + dtype = np.uint8 # choose based on stream_hdr.data_type_size and data_format + + stream = m.get_stream( 0 ) + print( f"Stream 0: {stream.n_records_in_file} record(s)" ) + + while stream.read_record(): + arr = stream.get_channel_data( 0 ).view( dtype ) + print( f" acq={stream.acquisition_id} rec={stream.record_count_in_acq} data={arr}" ) diff --git a/Documentation/index.rst b/Documentation/index.rst index 4121b74..6bea2cd 100644 --- a/Documentation/index.rst +++ b/Documentation/index.rst @@ -8,6 +8,7 @@ Contents: Monarch_versions UsageMonarch3 + UsageMonarch3Python .. end of toc .. (you must not remove or modify the above comment line, it is required by the API Doc generation) diff --git a/Monarch3/CMakeLists.txt b/Monarch3/CMakeLists.txt index bed9c3e..036b9a7 100644 --- a/Monarch3/CMakeLists.txt +++ b/Monarch3/CMakeLists.txt @@ -48,3 +48,8 @@ pbuilder_install_headers( ${MONARCH3_HEADERFILES} ) # Executables add_subdirectory( Executables ) + +# Python bindings +if( Monarch_BUILD_PYTHON ) + add_subdirectory( python ) +endif( Monarch_BUILD_PYTHON ) diff --git a/Monarch3/python/CMakeLists.txt b/Monarch3/python/CMakeLists.txt new file mode 100644 index 0000000..554900e --- /dev/null +++ b/Monarch3/python/CMakeLists.txt @@ -0,0 +1,37 @@ +# CMakeLists.txt for Monarch3/python +# Author: N.S. Oblath + +message( "Building Monarch3 Python bindings" ) + +set( dir ${CMAKE_CURRENT_SOURCE_DIR} ) + +set( PYBINDING_HEADERFILES + ${dir}/monarch3_binding_helpers.hh + ${dir}/m3constants_pybind.hh + ${dir}/m3header_pybind.hh + ${dir}/m3record_pybind.hh + ${dir}/m3stream_pybind.hh + ${dir}/m3monarch_pybind.hh +) + +set( PYBINDING_SOURCEFILES + ${dir}/monarch3_namespace_pybind.cc +) + +set( PYBINDING_PROJECT_LIBRARIES Monarch3 ) + +pbuilder_add_pybind11_module( + MODULE_NAME monarch3 + SOURCE_FILES ${PYBINDING_SOURCEFILES} + PROJECT_LIBRARIES ${PYBINDING_PROJECT_LIBRARIES} +) + +pbuilder_install_headers( ${PYBINDING_HEADERFILES} ) + +if( NOT DEFINED PBUILDER_PY_INSTALL_IN_SITELIB ) + message( STATUS "Installing add_lib_python_path.sh since python modules will not be installed in the site-package directory" ) + configure_file( add_lib_python_path.sh.in add_lib_python_path.sh @ONLY ) + install( FILES ${CMAKE_CURRENT_BINARY_DIR}/add_lib_python_path.sh DESTINATION ${BIN_INSTALL_DIR} ) +endif( NOT DEFINED PBUILDER_PY_INSTALL_IN_SITELIB ) + +message( "Done with Monarch3 Python bindings" ) diff --git a/Monarch3/python/add_lib_python_path.sh.in b/Monarch3/python/add_lib_python_path.sh.in new file mode 100644 index 0000000..68aff4b --- /dev/null +++ b/Monarch3/python/add_lib_python_path.sh.in @@ -0,0 +1,3 @@ +# Adds the lib install directory to the python path + +export PYTHONPATH=@LIB_INSTALL_DIR@:$PYTHONPATH diff --git a/Monarch3/python/m3constants_pybind.hh b/Monarch3/python/m3constants_pybind.hh new file mode 100644 index 0000000..566fb1d --- /dev/null +++ b/Monarch3/python/m3constants_pybind.hh @@ -0,0 +1,44 @@ +#ifndef M3CONSTANTS_PYBIND_HH_ +#define M3CONSTANTS_PYBIND_HH_ + +#include "M3Constants.hh" + +#include "pybind11/pybind11.h" + +namespace monarch3_pybind +{ + + std::list< std::string > export_constants( pybind11::module& mod ) + { + std::list< std::string > all_items; + + // Data format constants + all_items.push_back( "sDigitizedUS" ); + mod.attr( "sDigitizedUS" ) = monarch3::sDigitizedUS; + + all_items.push_back( "sDigitizedS" ); + mod.attr( "sDigitizedS" ) = monarch3::sDigitizedS; + + all_items.push_back( "sAnalog" ); + mod.attr( "sAnalog" ) = monarch3::sAnalog; + + // Bit alignment constants + all_items.push_back( "sBitsAlignedLeft" ); + mod.attr( "sBitsAlignedLeft" ) = monarch3::sBitsAlignedLeft; + + all_items.push_back( "sBitsAlignedRight" ); + mod.attr( "sBitsAlignedRight" ) = monarch3::sBitsAlignedRight; + + // Channel format constants + all_items.push_back( "sInterleaved" ); + mod.attr( "sInterleaved" ) = monarch3::sInterleaved; + + all_items.push_back( "sSeparate" ); + mod.attr( "sSeparate" ) = monarch3::sSeparate; + + return all_items; + } + +} /* namespace monarch3_pybind */ + +#endif /* M3CONSTANTS_PYBIND_HH_ */ diff --git a/Monarch3/python/m3header_pybind.hh b/Monarch3/python/m3header_pybind.hh new file mode 100644 index 0000000..72b8894 --- /dev/null +++ b/Monarch3/python/m3header_pybind.hh @@ -0,0 +1,145 @@ +#ifndef M3HEADER_PYBIND_HH_ +#define M3HEADER_PYBIND_HH_ + +#include "M3Header.hh" + +#include "pybind11/pybind11.h" +#include "pybind11/stl.h" + +#include + +namespace monarch3_pybind +{ + + std::list< std::string > export_header( pybind11::module& mod ) + { + std::list< std::string > all_items; + + // M3StreamHeader + all_items.push_back( "M3StreamHeader" ); + pybind11::class_< monarch3::M3StreamHeader >( mod, "M3StreamHeader", + "Header information for a single data stream" ) + .def_property_readonly( "number", &monarch3::M3StreamHeader::GetNumber ) + .def_property_readonly( "source", []( const monarch3::M3StreamHeader& h ) { return h.Source(); } ) + .def_property_readonly( "n_channels", &monarch3::M3StreamHeader::GetNChannels ) + .def_property_readonly( "channels", []( const monarch3::M3StreamHeader& h ) { return h.Channels(); } ) + .def_property_readonly( "channel_format", &monarch3::M3StreamHeader::GetChannelFormat ) + .def_property_readonly( "acquisition_rate",&monarch3::M3StreamHeader::GetAcquisitionRate ) + .def_property_readonly( "record_size", &monarch3::M3StreamHeader::GetRecordSize ) + .def_property_readonly( "sample_size", &monarch3::M3StreamHeader::GetSampleSize ) + .def_property_readonly( "data_type_size", &monarch3::M3StreamHeader::GetDataTypeSize ) + .def_property_readonly( "data_format", &monarch3::M3StreamHeader::GetDataFormat ) + .def_property_readonly( "bit_depth", &monarch3::M3StreamHeader::GetBitDepth ) + .def_property_readonly( "bit_alignment", &monarch3::M3StreamHeader::GetBitAlignment ) + .def_property_readonly( "n_acquisitions", &monarch3::M3StreamHeader::GetNAcquisitions ) + .def_property_readonly( "n_records", &monarch3::M3StreamHeader::GetNRecords ) + .def( "__repr__", []( const monarch3::M3StreamHeader& h ) { + std::ostringstream out; + out << h; + return out.str(); + } ) + ; + + // M3ChannelHeader + all_items.push_back( "M3ChannelHeader" ); + pybind11::class_< monarch3::M3ChannelHeader >( mod, "M3ChannelHeader", + "Header information for a single data channel" ) + .def_property_readonly( "number", &monarch3::M3ChannelHeader::GetNumber ) + .def_property_readonly( "source", []( const monarch3::M3ChannelHeader& h ) { return h.Source(); } ) + .def_property_readonly( "acquisition_rate",&monarch3::M3ChannelHeader::GetAcquisitionRate ) + .def_property_readonly( "record_size", &monarch3::M3ChannelHeader::GetRecordSize ) + .def_property_readonly( "sample_size", &monarch3::M3ChannelHeader::GetSampleSize ) + .def_property_readonly( "data_type_size", &monarch3::M3ChannelHeader::GetDataTypeSize ) + .def_property_readonly( "data_format", &monarch3::M3ChannelHeader::GetDataFormat ) + .def_property_readonly( "bit_depth", &monarch3::M3ChannelHeader::GetBitDepth ) + .def_property_readonly( "bit_alignment", &monarch3::M3ChannelHeader::GetBitAlignment ) + .def_property_readonly( "voltage_offset", &monarch3::M3ChannelHeader::GetVoltageOffset ) + .def_property_readonly( "voltage_range", &monarch3::M3ChannelHeader::GetVoltageRange ) + .def_property_readonly( "dac_gain", &monarch3::M3ChannelHeader::GetDACGain ) + .def_property_readonly( "frequency_min", &monarch3::M3ChannelHeader::GetFrequencyMin ) + .def_property_readonly( "frequency_range", &monarch3::M3ChannelHeader::GetFrequencyRange ) + .def( "__repr__", []( const monarch3::M3ChannelHeader& h ) { + std::ostringstream out; + out << h; + return out.str(); + } ) + ; + + // M3Header + all_items.push_back( "M3Header" ); + pybind11::class_< monarch3::M3Header >( mod, "M3Header", + "Egg file header: run metadata and stream/channel configuration" ) + // Read-only properties + .def_property_readonly( "egg_version", []( const monarch3::M3Header& h ) { return h.EggVersion(); } ) + .def_property_readonly( "n_channels", &monarch3::M3Header::GetNChannels ) + .def_property_readonly( "n_streams", &monarch3::M3Header::GetNStreams ) + .def_property_readonly( "stream_headers", + []( const monarch3::M3Header& h ) { return h.StreamHeaders(); } ) + .def_property_readonly( "channel_headers", + []( const monarch3::M3Header& h ) { return h.ChannelHeaders(); } ) + // Read-write properties (ref-accessor pattern) + .def_property( "filename", + []( const monarch3::M3Header& h ) { return h.Filename(); }, + []( monarch3::M3Header& h, const std::string& v ) { h.Filename() = v; } ) + .def_property( "run_duration", + &monarch3::M3Header::GetRunDuration, + &monarch3::M3Header::SetRunDuration ) + .def_property( "timestamp", + []( const monarch3::M3Header& h ) { return h.Timestamp(); }, + []( monarch3::M3Header& h, const std::string& v ) { h.Timestamp() = v; } ) + .def_property( "description", + []( const monarch3::M3Header& h ) { return h.Description(); }, + []( monarch3::M3Header& h, const std::string& v ) { h.Description() = v; } ) + // AddStream overloads + .def( "add_stream", + ( unsigned ( monarch3::M3Header::* )( + const std::string&, + uint32_t, uint32_t, uint32_t, + uint32_t, uint32_t, + uint32_t, uint32_t, + std::vector< unsigned >* ) ) + &monarch3::M3Header::AddStream, + pybind11::arg( "source" ), + pybind11::arg( "acq_rate" ), + pybind11::arg( "rec_size" ), + pybind11::arg( "sample_size" ), + pybind11::arg( "data_type_size" ), + pybind11::arg( "data_format" ), + pybind11::arg( "bit_depth" ), + pybind11::arg( "bit_alignment" ), + pybind11::arg( "chan_vec" ) = nullptr, + "Add a single-channel stream; returns the stream number" ) + .def( "add_stream", + ( unsigned ( monarch3::M3Header::* )( + const std::string&, + uint32_t, uint32_t, + uint32_t, uint32_t, uint32_t, + uint32_t, uint32_t, + uint32_t, uint32_t, + std::vector< unsigned >* ) ) + &monarch3::M3Header::AddStream, + pybind11::arg( "source" ), + pybind11::arg( "n_channels" ), + pybind11::arg( "channel_format" ), + pybind11::arg( "acq_rate" ), + pybind11::arg( "rec_size" ), + pybind11::arg( "sample_size" ), + pybind11::arg( "data_type_size" ), + pybind11::arg( "data_format" ), + pybind11::arg( "bit_depth" ), + pybind11::arg( "bit_alignment" ), + pybind11::arg( "chan_vec" ) = nullptr, + "Add a multi-channel stream; returns the stream number" ) + .def( "__repr__", []( const monarch3::M3Header& h ) { + std::ostringstream out; + out << h; + return out.str(); + } ) + ; + + return all_items; + } + +} /* namespace monarch3_pybind */ + +#endif /* M3HEADER_PYBIND_HH_ */ diff --git a/Monarch3/python/m3monarch_pybind.hh b/Monarch3/python/m3monarch_pybind.hh new file mode 100644 index 0000000..1b50144 --- /dev/null +++ b/Monarch3/python/m3monarch_pybind.hh @@ -0,0 +1,146 @@ +#ifndef M3MONARCH_PYBIND_HH_ +#define M3MONARCH_PYBIND_HH_ + +#include "M3Monarch.hh" + +#include "monarch3_binding_helpers.hh" + +#include "pybind11/pybind11.h" + +#include + +namespace monarch3_pybind +{ + + // Custom deleter: call FinishReading/FinishWriting before deleting so HDF5 file is + // closed properly even when Python's garbage collector destroys the object. + struct Monarch3Deleter + { + void operator()( monarch3::Monarch3* m ) const + { + if( m == nullptr ) return; + auto state = m->GetState(); + if( state == monarch3::Monarch3::eReadyToRead || state == monarch3::Monarch3::eOpenToRead ) + { + try { m->FinishReading(); } catch(...) {} + } + else if( state == monarch3::Monarch3::eReadyToWrite || state == monarch3::Monarch3::eOpenToWrite ) + { + try { m->FinishWriting(); } catch(...) {} + } + delete m; + } + }; + + // Convenience alias + using Monarch3Ptr = std::unique_ptr< monarch3::Monarch3, Monarch3Deleter >; + + + std::list< std::string > export_monarch( pybind11::module& mod ) + { + std::list< std::string > all_items; + + all_items.push_back( "Monarch3" ); + pybind11::class_< monarch3::Monarch3, Monarch3Ptr >( mod, "Monarch3", + "Top-level egg v3 file handle.\n\n" + "Use open_for_reading() or open_for_writing() to obtain an instance.\n" + "Supports the context manager protocol ('with' statement)." ) + + // ---- Factory methods ---- + .def_static( "open_for_reading", + []( const std::string& filename ) -> Monarch3Ptr { + return Monarch3Ptr( Monarch3::OpenForReading( filename ) ); + }, + pybind11::arg( "filename" ), + "Open an existing egg file for reading.\n" + "Returns a Monarch3 instance in the eOpenToRead state.", + MONARCH3_BIND_CALL_GUARD_STREAMS ) + .def_static( "open_for_writing", + []( const std::string& filename ) -> Monarch3Ptr { + return Monarch3Ptr( Monarch3::OpenForWriting( filename ) ); + }, + pybind11::arg( "filename" ), + "Create or overwrite an egg file for writing.\n" + "Returns a Monarch3 instance in the eOpenToWrite state.", + MONARCH3_BIND_CALL_GUARD_STREAMS ) + + // ---- Context manager ---- + .def( "__enter__", + []( monarch3::Monarch3* self ) { return self; }, + pybind11::return_value_policy::reference, + "Enter context manager; returns self." ) + .def( "__exit__", + []( monarch3::Monarch3* self, + pybind11::object /*exc_type*/, + pybind11::object /*exc_val*/, + pybind11::object /*exc_tb*/ ) + { + auto state = self->GetState(); + if( state == monarch3::Monarch3::eReadyToRead || state == monarch3::Monarch3::eOpenToRead ) + { + self->FinishReading(); + } + else if( state == monarch3::Monarch3::eReadyToWrite || state == monarch3::Monarch3::eOpenToWrite ) + { + self->FinishWriting(); + } + return false; // do not suppress exceptions + }, + "Exit context manager; calls finish_reading() or finish_writing() as appropriate." ) + + // ---- State ---- + .def_property_readonly( "state", + &monarch3::Monarch3::GetState, + "Current state of the file handle" ) + + // ---- Reading ---- + .def( "read_header", + &monarch3::Monarch3::ReadHeader, + "Read header information from the file.\n" + "Must be called after open_for_reading() before accessing data.", + MONARCH3_BIND_CALL_GUARD_STREAMS ) + .def( "get_header", + ( const monarch3::M3Header* ( monarch3::Monarch3::* )() const ) + &monarch3::Monarch3::GetHeader, + pybind11::return_value_policy::reference_internal, + "Return the file header (read access)" ) + .def( "get_stream", + ( const monarch3::M3Stream* ( monarch3::Monarch3::* )( unsigned ) const ) + &monarch3::Monarch3::GetStream, + pybind11::arg( "stream" ), + pybind11::return_value_policy::reference_internal, + "Return the stream object for the given stream index (read access)" ) + .def( "finish_reading", + &monarch3::Monarch3::FinishReading, + "Close the file after reading.", + MONARCH3_BIND_CALL_GUARD_STREAMS ) + + // ---- Writing ---- + .def( "write_header", + &monarch3::Monarch3::WriteHeader, + "Write the header to file and prepare streams for writing.\n" + "Must be called after configuring the header and adding streams.", + MONARCH3_BIND_CALL_GUARD_STREAMS ) + .def( "get_header", + ( monarch3::M3Header* ( monarch3::Monarch3::* )() ) + &monarch3::Monarch3::GetHeader, + pybind11::return_value_policy::reference_internal, + "Return the file header (write access)" ) + .def( "get_stream", + ( monarch3::M3Stream* ( monarch3::Monarch3::* )( unsigned ) ) + &monarch3::Monarch3::GetStream, + pybind11::arg( "stream" ), + pybind11::return_value_policy::reference_internal, + "Return the stream object for the given stream index (write access)" ) + .def( "finish_writing", + &monarch3::Monarch3::FinishWriting, + "Flush and close the file after writing.", + MONARCH3_BIND_CALL_GUARD_STREAMS ) + ; + + return all_items; + } + +} /* namespace monarch3_pybind */ + +#endif /* M3MONARCH_PYBIND_HH_ */ diff --git a/Monarch3/python/m3record_pybind.hh b/Monarch3/python/m3record_pybind.hh new file mode 100644 index 0000000..59a111d --- /dev/null +++ b/Monarch3/python/m3record_pybind.hh @@ -0,0 +1,47 @@ +#ifndef M3RECORD_PYBIND_HH_ +#define M3RECORD_PYBIND_HH_ + +#include "M3Record.hh" + +#include "pybind11/pybind11.h" +#include "pybind11/numpy.h" + +namespace monarch3_pybind +{ + + std::list< std::string > export_record( pybind11::module& mod ) + { + std::list< std::string > all_items; + + all_items.push_back( "M3Record" ); + pybind11::class_< monarch3::M3Record >( mod, "M3Record", + "A single data record: ID, timestamp (ns), and raw data bytes" ) + .def_property_readonly( "record_id", + &monarch3::M3Record::GetRecordId, + "Record ID (uint64)" ) + .def_property_readonly( "time", + &monarch3::M3Record::GetTime, + "Timestamp in nanoseconds since the start of the run (uint64)" ) + .def( "get_data", + []( const monarch3::M3Record& r, unsigned nbytes ) { + // Zero-copy view: the numpy array does not own the data. + // The caller must ensure the record (and its parent stream) stays alive. + return pybind11::array_t< monarch3::byte_type >( + { static_cast< pybind11::ssize_t >( nbytes ) }, + { sizeof( monarch3::byte_type ) }, + r.GetData(), + pybind11::cast( r ) // keep-alive base object + ); + }, + pybind11::arg( "nbytes" ), + "Return a writable numpy uint8 view of the data buffer.\n" + "nbytes should equal stream.channel_record_n_bytes (or stream_record_n_bytes).\n" + "Use numpy.ndarray.view(dtype) to reinterpret as a different element type." ) + ; + + return all_items; + } + +} /* namespace monarch3_pybind */ + +#endif /* M3RECORD_PYBIND_HH_ */ diff --git a/Monarch3/python/m3stream_pybind.hh b/Monarch3/python/m3stream_pybind.hh new file mode 100644 index 0000000..e53cbc8 --- /dev/null +++ b/Monarch3/python/m3stream_pybind.hh @@ -0,0 +1,138 @@ +#ifndef M3STREAM_PYBIND_HH_ +#define M3STREAM_PYBIND_HH_ + +#include "M3Stream.hh" + +#include "monarch3_binding_helpers.hh" + +#include "pybind11/pybind11.h" +#include "pybind11/numpy.h" + +namespace monarch3_pybind +{ + + std::list< std::string > export_stream( pybind11::module& mod ) + { + std::list< std::string > all_items; + + all_items.push_back( "M3Stream" ); + pybind11::class_< monarch3::M3Stream >( mod, "M3Stream", + "Read/write access for a single data stream" ) + + // ---- State query properties ---- + .def_property_readonly( "n_channels", + &monarch3::M3Stream::GetNChannels, + "Number of channels in this stream" ) + .def_property_readonly( "n_acquisitions", + &monarch3::M3Stream::GetNAcquisitions, + "Number of acquisitions in this stream (valid after ReadRecord or after writing)" ) + .def_property_readonly( "acquisition_id", + &monarch3::M3Stream::GetAcquisitionId, + "ID of the most recently accessed acquisition" ) + .def_property_readonly( "record_count_in_acq", + &monarch3::M3Stream::GetRecordCountInAcq, + "Number of records read/written in the current acquisition" ) + .def_property_readonly( "n_records_in_file", + &monarch3::M3Stream::GetNRecordsInFile, + "Total number of records across all acquisitions in the file" ) + .def_property_readonly( "n_records_in_acquisition", + &monarch3::M3Stream::GetNRecordsInAcquisition, + "Number of records in the current acquisition" ) + .def_property_readonly( "data_type_size", + &monarch3::M3Stream::GetDataTypeSize, + "Size in bytes of each sample element" ) + .def_property_readonly( "sample_size", + &monarch3::M3Stream::GetSampleSize, + "Number of elements per sample (1 for real, 2 for complex)" ) + .def_property_readonly( "channel_record_size", + &monarch3::M3Stream::GetChannelRecordSize, + "Number of samples in a channel record" ) + .def_property_readonly( "channel_record_n_bytes", + &monarch3::M3Stream::GetChannelRecordNBytes, + "Size in bytes of a channel record data buffer" ) + .def_property_readonly( "stream_record_size", + &monarch3::M3Stream::GetStreamRecordSize, + "Number of samples in the full (interleaved) stream record" ) + .def_property_readonly( "stream_record_n_bytes", + &monarch3::M3Stream::GetStreamRecordNBytes, + "Size in bytes of the full stream record data buffer" ) + .def_property_readonly( "is_interleaved", + &monarch3::M3Stream::GetIsInterleaved, + "True if multi-channel data is stored interleaved in the file" ) + + // ---- Record access: reading ---- + .def( "read_record", + &monarch3::M3Stream::ReadRecord, + pybind11::arg( "offset" ) = 0, + pybind11::arg( "if_new_acq_start_at_first_rec" ) = true, + "Read a record from the file.\n\n" + "If the last record read was [J], reads record [J+1+offset].\n" + "offset=0 (default): next record; offset=-1: reread current;\n" + "offset<-1: step backward; offset>0: skip forward.\n" + "Returns True on success, False when past end/start of file.\n" + "Raises Monarch3Exception on error.", + MONARCH3_BIND_CALL_GUARD_STREAMS_AND_GIL ) + .def( "get_stream_record", + ( monarch3::M3Record* ( monarch3::M3Stream::* )() ) + &monarch3::M3Stream::GetStreamRecord, + pybind11::return_value_policy::reference_internal, + "Return the stream-level record object (all channels interleaved)" ) + .def( "get_channel_record", + ( monarch3::M3Record* ( monarch3::M3Stream::* )( unsigned ) ) + &monarch3::M3Stream::GetChannelRecord, + pybind11::arg( "channel" ), + pybind11::return_value_policy::reference_internal, + "Return the record object for the given channel index" ) + + // ---- Convenience: typed numpy data access ---- + .def( "get_stream_data", + []( monarch3::M3Stream& s ) { + monarch3::M3Record* r = s.GetStreamRecord(); + unsigned nbytes = s.GetStreamRecordNBytes(); + return pybind11::array_t< monarch3::byte_type >( + { static_cast< pybind11::ssize_t >( nbytes ) }, + { sizeof( monarch3::byte_type ) }, + r->GetData(), + pybind11::cast( s ) + ); + }, + pybind11::return_value_policy::reference_internal, + "Return a writable numpy uint8 view of the full (interleaved) stream record.\n" + "Call after read_record(). Use .view(dtype) to reinterpret elements." ) + .def( "get_channel_data", + []( monarch3::M3Stream& s, unsigned channel ) { + monarch3::M3Record* r = s.GetChannelRecord( channel ); + if( r == nullptr ) + { + throw monarch3::M3Exception() << "Channel " << channel << " does not exist in this stream"; + } + unsigned nbytes = s.GetChannelRecordNBytes(); + return pybind11::array_t< monarch3::byte_type >( + { static_cast< pybind11::ssize_t >( nbytes ) }, + { sizeof( monarch3::byte_type ) }, + r->GetData(), + pybind11::cast( s ) + ); + }, + pybind11::arg( "channel" ), + pybind11::return_value_policy::reference_internal, + "Return a writable numpy uint8 view of the data buffer for the given channel.\n" + "Call after read_record(). Use .view(dtype) to reinterpret elements." ) + + // ---- Record access: writing ---- + .def( "write_record", + &monarch3::M3Stream::WriteRecord, + pybind11::arg( "is_new_acquisition" ), + "Write the current record contents to the file.\n\n" + "is_new_acquisition=True starts a new acquisition group;\n" + "is_new_acquisition=False continues the current acquisition.\n" + "Returns True on success. Raises Monarch3Exception on error.", + MONARCH3_BIND_CALL_GUARD_STREAMS_AND_GIL ) + ; + + return all_items; + } + +} /* namespace monarch3_pybind */ + +#endif /* M3STREAM_PYBIND_HH_ */ diff --git a/Monarch3/python/monarch3_binding_helpers.hh b/Monarch3/python/monarch3_binding_helpers.hh new file mode 100644 index 0000000..9d4fb67 --- /dev/null +++ b/Monarch3/python/monarch3_binding_helpers.hh @@ -0,0 +1,15 @@ +#ifndef MONARCH3_PYBIND_BINDING_HELPERS_HH_ +#define MONARCH3_PYBIND_BINDING_HELPERS_HH_ + +#include "pybind11/iostream.h" + +#define MONARCH3_BIND_CALL_GUARD_STREAMS \ + pybind11::call_guard< pybind11::scoped_ostream_redirect, pybind11::scoped_estream_redirect >() + +#define MONARCH3_BIND_CALL_GUARD_GIL \ + pybind11::call_guard< pybind11::gil_scoped_release >() + +#define MONARCH3_BIND_CALL_GUARD_STREAMS_AND_GIL \ + pybind11::call_guard< pybind11::scoped_ostream_redirect, pybind11::scoped_estream_redirect, pybind11::gil_scoped_release >() + +#endif /* MONARCH3_PYBIND_BINDING_HELPERS_HH_ */ diff --git a/Monarch3/python/monarch3_namespace_pybind.cc b/Monarch3/python/monarch3_namespace_pybind.cc new file mode 100644 index 0000000..60624db --- /dev/null +++ b/Monarch3/python/monarch3_namespace_pybind.cc @@ -0,0 +1,45 @@ +/* + * monarch3_namespace_pybind.cc + * + * Pybind11 module entry point for the monarch3 Python binding. + */ + +#include "pybind11/pybind11.h" +#include "pybind11/stl.h" +#include "pybind11/numpy.h" + +#include "M3Exception.hh" + +#include "m3constants_pybind.hh" +#include "m3header_pybind.hh" +#include "m3record_pybind.hh" +#include "m3stream_pybind.hh" +#include "m3monarch_pybind.hh" + +PYBIND11_MODULE( monarch3, mod ) +{ + mod.doc() = "Python bindings for the Monarch3 egg v3 file library"; + + std::list< std::string > all_members; + + // Exception + pybind11::register_exception< monarch3::M3Exception >( mod, "Monarch3Exception" ); + all_members.push_back( "Monarch3Exception" ); + + // Constants + all_members.splice( all_members.end(), monarch3_pybind::export_constants( mod ) ); + + // Header classes + all_members.splice( all_members.end(), monarch3_pybind::export_header( mod ) ); + + // Record class + all_members.splice( all_members.end(), monarch3_pybind::export_record( mod ) ); + + // Stream class + all_members.splice( all_members.end(), monarch3_pybind::export_stream( mod ) ); + + // Top-level Monarch3 class + all_members.splice( all_members.end(), monarch3_pybind::export_monarch( mod ) ); + + mod.attr( "__all__" ) = all_members; +} From 05619057fd9402aa799d8baa63b6a951ec52b840 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Wed, 19 Aug 2026 14:07:04 -0700 Subject: [PATCH 02/30] Added validation testing (courtesy of claude) --- CMakeLists.txt | 4 + Documentation/TestingMonarch3.rst | 137 ++++++++++++++ Documentation/index.rst | 1 + Monarch3/CMakeLists.txt | 5 + Monarch3/Tests/CMakeLists.txt | 1 + Monarch3/Tests/Validation/CMakeLists.txt | 37 ++++ Monarch3/Tests/Validation/conftest.py | 74 ++++++++ .../Validation/test_cpp_write_python_read.py | 167 ++++++++++++++++++ .../Validation/test_python_write_cpp_read.py | 100 +++++++++++ .../test_python_write_python_read.py | 91 ++++++++++ 10 files changed, 617 insertions(+) create mode 100644 Documentation/TestingMonarch3.rst create mode 100644 Monarch3/Tests/CMakeLists.txt create mode 100644 Monarch3/Tests/Validation/CMakeLists.txt create mode 100644 Monarch3/Tests/Validation/conftest.py create mode 100644 Monarch3/Tests/Validation/test_cpp_write_python_read.py create mode 100644 Monarch3/Tests/Validation/test_python_write_cpp_read.py create mode 100644 Monarch3/Tests/Validation/test_python_write_python_read.py diff --git a/CMakeLists.txt b/CMakeLists.txt index 6be6789..398ed9c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -151,6 +151,10 @@ endif( Monarch_BUILD_PYTHON ) pbuilder_prepare_project() +if( Monarch_ENABLE_TESTING ) + enable_testing() +endif( Monarch_ENABLE_TESTING ) + ######## # Scarab diff --git a/Documentation/TestingMonarch3.rst b/Documentation/TestingMonarch3.rst new file mode 100644 index 0000000..00bb7ab --- /dev/null +++ b/Documentation/TestingMonarch3.rst @@ -0,0 +1,137 @@ +Testing Monarch3 +================ + +Overview +-------- + +Monarch3 includes two categories of tests: + +**C++ test executables** (``M3WriteTest``, ``M3ReadTest``, ``M3WriteSpeedTest``, +``M3MultithreadingTest``) are built when ``Monarch_ENABLE_TESTING`` is ON. +They exercise the C++ library directly and do not require Python. + +**Python validation tests** live in ``Monarch3/Tests/Validation/`` and require both +``Monarch_ENABLE_TESTING`` and ``Monarch_BUILD_PYTHON`` to be ON. They use +`pytest `_ as the test runner and verify cross-language +compatibility between the C++ library and the Python binding. The validation tests +are registered with CTest so they run automatically alongside the C++ tests. + + +Building for Testing +-------------------- + +To build all tests — C++ executables and Python validation — configure CMake with:: + + cmake \ + -DMonarch_BUILD_MONARCH3=ON \ + -DMonarch_ENABLE_TESTING=ON \ + -DMonarch_BUILD_PYTHON=ON \ + /path/to/monarch/source + + make + +.. note:: + ``Monarch_BUILD_PYTHON`` requires `pybind11 `_ + and Python 3 development headers (``python3-dev`` or equivalent). + ``numpy`` and ``pytest`` must also be available in the active Python environment. + +To build only the C++ test executables (no Python binding or validation tests):: + + cmake -DMonarch_BUILD_MONARCH3=ON -DMonarch_ENABLE_TESTING=ON /path/to/monarch/source + make + + +Running Tests +------------- + +**Via CTest** (recommended; runs everything registered in the build):: + + cd build + ctest --output-on-failure + +This runs the C++ test executables and, if the Python binding was built, the pytest +validation suite as a single CTest entry named ``monarch3_validation``. + +**Running the Python validation tests directly** (after building and installing):: + + cd build + source bin/add_lib_python_path.sh # makes monarch3.so importable + pytest Monarch3/Tests/Validation/ -v + +Run a specific test file:: + + pytest Monarch3/Tests/Validation/test_cpp_write_python_read.py -v + +Run a specific test function:: + + pytest Monarch3/Tests/Validation/test_cpp_write_python_read.py::test_stream0_data -v + + +Validation +---------- + +The validation tests verify cross-language compatibility: that egg v3 files written by +C++ are correctly read by Python, and that files written by Python are correctly read by +C++. All three test modules use the same gold-standard 4-stream data structure defined +in the ``M3WriteTest`` C++ executable. + +Gold-Standard Data +~~~~~~~~~~~~~~~~~~ + +Every validation test is written against a common set of known values: + ++--------+----------------------------+---------+-----------+------+------+----------+---------------------+ +| Stream | Source | Channels| Format | Rate | Rec | Type | Record values | ++========+============================+=========+===========+======+======+==========+=====================+ +| 0 | "1-channel device" | 1 | — | 500 | 10 | uint8 | rec0=1, rec1=10 | ++--------+----------------------------+---------+-----------+------+------+----------+---------------------+ +| 1 | "2-channel device" | 2 | interleaved| 250 | 5 | uint16 | rec0=(1,2), | +| | | | | | | | rec1=(1000,2000), | +| | | | | | | | rec2=(10000,20000) | ++--------+----------------------------+---------+-----------+------+------+----------+---------------------+ +| 2 | "3-channel device" | 3 | separate | 100 | 5 | uint8 | rec0=(1,2,3), | +| | | | | | | | rec1=(10,20,30) | ++--------+----------------------------+---------+-----------+------+------+----------+---------------------+ +| 3 | "Floating-point device" | 1 | — | 100 | 10 | float32 | rec0=π, rec1=e | ++--------+----------------------------+---------+-----------+------+------+----------+---------------------+ + +Header fields: ``run_duration=8675309``, ``timestamp="Stardate 33515"``, +``description="Bigger on the inside"``. + +Stream 1 has two acquisitions: acquisition 0 contains record 0; acquisition 1 contains +records 1 and 2. Stream 3 has two acquisitions, one record each. + +The shared constants are defined in ``conftest.py`` and imported by all test modules. + +Test Scenarios +~~~~~~~~~~~~~~ + +**C++ write → Python read** (``test_cpp_write_python_read.py``) + +The fixture ``cpp_written_egg`` invokes the ``M3WriteTest`` executable to produce a +known-good egg file in a temporary directory. The tests then open that file with the +Python binding and verify: + +- Header fields (``run_duration``, ``timestamp``, ``description``, ``n_streams``) +- Per-stream structure (channel count, acquisition count, record count, record size) +- Sample values for each stream and channel, including float comparison with + ``numpy.isclose`` +- Offset-based record navigation for stream 1: forward skips, rewinds, and requests + past the end or before the beginning of the file (mirrors the checks in + ``M3ReadTest.cc``) + +**Python write → C++ read** (``test_python_write_cpp_read.py``) + +The helper function ``write_standard_egg(path)`` uses the Python binding to write the +same 4-stream structure as ``M3WriteTest``. The test then runs the ``M3ReadTest`` +executable against that file and asserts that it exits with return code 0. Since +``M3ReadTest`` checks record counts, acquisition counts, channel counts, and data values +internally, a clean exit confirms byte-level compatibility. + +**Python write → Python read** (``test_python_write_python_read.py``) + +Uses ``write_standard_egg`` (imported from ``test_python_write_cpp_read.py``) to produce +a file, then reads it back entirely with the Python binding. This isolates the binding +from the C++ executables and verifies that the write and read paths are internally +consistent. The same structure and value checks as the C++ write → Python read scenario +are applied. diff --git a/Documentation/index.rst b/Documentation/index.rst index 6bea2cd..83a3241 100644 --- a/Documentation/index.rst +++ b/Documentation/index.rst @@ -9,6 +9,7 @@ Contents: Monarch_versions UsageMonarch3 UsageMonarch3Python + TestingMonarch3 .. end of toc .. (you must not remove or modify the above comment line, it is required by the API Doc generation) diff --git a/Monarch3/CMakeLists.txt b/Monarch3/CMakeLists.txt index 036b9a7..c535d58 100644 --- a/Monarch3/CMakeLists.txt +++ b/Monarch3/CMakeLists.txt @@ -53,3 +53,8 @@ add_subdirectory( Executables ) if( Monarch_BUILD_PYTHON ) add_subdirectory( python ) endif( Monarch_BUILD_PYTHON ) + +# Tests +if( Monarch_ENABLE_TESTING ) + add_subdirectory( Tests ) +endif( Monarch_ENABLE_TESTING ) diff --git a/Monarch3/Tests/CMakeLists.txt b/Monarch3/Tests/CMakeLists.txt new file mode 100644 index 0000000..3b90ebd --- /dev/null +++ b/Monarch3/Tests/CMakeLists.txt @@ -0,0 +1 @@ +add_subdirectory( Validation ) diff --git a/Monarch3/Tests/Validation/CMakeLists.txt b/Monarch3/Tests/Validation/CMakeLists.txt new file mode 100644 index 0000000..81df3d5 --- /dev/null +++ b/Monarch3/Tests/Validation/CMakeLists.txt @@ -0,0 +1,37 @@ +# CMakeLists.txt for Monarch3/Tests/Validation +# Author: N.S. Oblath + +if( Monarch_BUILD_PYTHON ) + + message( STATUS "Configuring Monarch3 validation tests" ) + + set( VALIDATION_TEST_FILES + conftest.py + test_cpp_write_python_read.py + test_python_write_cpp_read.py + test_python_write_python_read.py + ) + + # Copy Python test files to build tree so pytest can find the built monarch3 module + foreach( tfile ${VALIDATION_TEST_FILES} ) + configure_file( ${tfile} ${CMAKE_CURRENT_BINARY_DIR}/${tfile} COPYONLY ) + endforeach() + + # Register validation tests with CTest + # pytest is invoked in the build-tree copy so it can find monarch3.so and the executables + add_test( + NAME monarch3_validation + COMMAND ${Python3_EXECUTABLE} -m pytest ${CMAKE_CURRENT_BINARY_DIR} -v + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + ) + + # Make the built monarch3 module and test executables available at test time + set_tests_properties( monarch3_validation PROPERTIES + ENVIRONMENT "PYTHONPATH=${LIB_INSTALL_DIR}:$ENV{PYTHONPATH};PATH=${BIN_INSTALL_DIR}:$ENV{PATH}" + ) + +else( Monarch_BUILD_PYTHON ) + + message( STATUS "Skipping Monarch3 Python validation tests (Monarch_BUILD_PYTHON is OFF)" ) + +endif( Monarch_BUILD_PYTHON ) diff --git a/Monarch3/Tests/Validation/conftest.py b/Monarch3/Tests/Validation/conftest.py new file mode 100644 index 0000000..7ab52e9 --- /dev/null +++ b/Monarch3/Tests/Validation/conftest.py @@ -0,0 +1,74 @@ +# conftest.py +# Shared fixtures and constants for Monarch3 validation tests. +# All data constants match the exact values written by the M3WriteTest C++ executable. + +import pytest +import subprocess + +# ---- Gold-standard data constants from M3WriteTest ---- + +# Header +HEADER_RUN_DURATION = 8675309 +HEADER_TIMESTAMP = "Stardate 33515" +HEADER_DESCRIPTION = "Bigger on the inside" + +# Stream 0: 1 channel, uint8, 10 samples/record, 1 acquisition, 2 records +STREAM0_N_CHANNELS = 1 +STREAM0_N_ACQUISITIONS = 1 +STREAM0_N_RECORDS = 2 +STREAM0_REC_SIZE = 10 +STREAM0_VALUES = [1, 10] # one uniform value per record + +# Stream 1: 2 channels, uint16, interleaved, 5 samples/record, 2 acquisitions, 3 records +# acq 0: rec 0 (WriteRecord true) +# acq 1: rec 1 (WriteRecord true), rec 2 (WriteRecord false) +STREAM1_N_CHANNELS = 2 +STREAM1_N_ACQUISITIONS = 2 +STREAM1_N_RECORDS = 3 +STREAM1_REC_SIZE = 5 +STREAM1_VALUES = [ + (1, 2), # rec 0: ch0=1, ch1=2 + (1000, 2000), # rec 1: ch0=1000, ch1=2000 + (10000, 20000), # rec 2: ch0=10000, ch1=20000 +] + +# Stream 2: 3 channels, uint8, separate, 5 samples/record, 1 acquisition, 2 records +STREAM2_N_CHANNELS = 3 +STREAM2_N_ACQUISITIONS = 1 +STREAM2_N_RECORDS = 2 +STREAM2_REC_SIZE = 5 +STREAM2_VALUES = [ + (1, 2, 3), # rec 0: ch0=1, ch1=2, ch2=3 + (10, 20, 30), # rec 1: ch0=10, ch1=20, ch2=30 +] + +# Stream 3: 1 channel, float32, 10 samples/record, 2 acquisitions, 2 records (one each) +STREAM3_N_CHANNELS = 1 +STREAM3_N_ACQUISITIONS = 2 +STREAM3_N_RECORDS = 2 +STREAM3_REC_SIZE = 10 +STREAM3_VALUES = [3.1415926535898, 2.71828182846] # pi, e + +# ---- Fixtures ---- + +@pytest.fixture +def tmp_egg(tmp_path): + """Provide a temporary .egg file path, cleaned up automatically after the test.""" + return str(tmp_path / "test.egg") + + +@pytest.fixture +def cpp_written_egg(tmp_path): + """Run M3WriteTest to produce the gold-standard egg file; return its path.""" + path = str(tmp_path / "cpp_written.egg") + result = subprocess.run( + ["M3WriteTest", path], + capture_output=True, + text=True, + ) + assert result.returncode == 0, ( + f"M3WriteTest failed with return code {result.returncode}:\n" + f"stdout: {result.stdout}\n" + f"stderr: {result.stderr}" + ) + return path diff --git a/Monarch3/Tests/Validation/test_cpp_write_python_read.py b/Monarch3/Tests/Validation/test_cpp_write_python_read.py new file mode 100644 index 0000000..8d26c35 --- /dev/null +++ b/Monarch3/Tests/Validation/test_cpp_write_python_read.py @@ -0,0 +1,167 @@ +# test_cpp_write_python_read.py +# Validates that the Python binding can correctly read a file produced by the C++ +# M3WriteTest executable. The structure and values checked here mirror M3ReadTest.cc. + +import numpy as np +import pytest +import monarch3 + +from conftest import ( + HEADER_RUN_DURATION, HEADER_TIMESTAMP, HEADER_DESCRIPTION, + STREAM0_N_CHANNELS, STREAM0_N_ACQUISITIONS, STREAM0_N_RECORDS, STREAM0_REC_SIZE, STREAM0_VALUES, + STREAM1_N_CHANNELS, STREAM1_N_ACQUISITIONS, STREAM1_N_RECORDS, STREAM1_REC_SIZE, STREAM1_VALUES, + STREAM2_N_CHANNELS, STREAM2_N_ACQUISITIONS, STREAM2_N_RECORDS, STREAM2_REC_SIZE, STREAM2_VALUES, + STREAM3_N_CHANNELS, STREAM3_N_ACQUISITIONS, STREAM3_N_RECORDS, STREAM3_REC_SIZE, STREAM3_VALUES, +) + + +# ---- Header ---- + +def test_header(cpp_written_egg): + with monarch3.Monarch3.open_for_reading(cpp_written_egg) as m: + m.read_header() + hdr = m.get_header() + assert hdr.run_duration == HEADER_RUN_DURATION + assert hdr.timestamp == HEADER_TIMESTAMP + assert hdr.description == HEADER_DESCRIPTION + assert hdr.n_streams == 4 + assert hdr.n_channels == 7 # 1 + 2 + 3 + 1 + + +# ---- Stream 0: single-channel uint8 ---- + +def test_stream0_structure(cpp_written_egg): + with monarch3.Monarch3.open_for_reading(cpp_written_egg) as m: + m.read_header() + stream = m.get_stream(0) + assert stream.n_channels == STREAM0_N_CHANNELS + assert stream.n_acquisitions == STREAM0_N_ACQUISITIONS + assert stream.n_records_in_file == STREAM0_N_RECORDS + assert stream.channel_record_size == STREAM0_REC_SIZE + + +def test_stream0_data(cpp_written_egg): + with monarch3.Monarch3.open_for_reading(cpp_written_egg) as m: + m.read_header() + stream = m.get_stream(0) + for expected_val in STREAM0_VALUES: + assert stream.read_record(), "Unexpected end of stream 0" + arr = stream.get_channel_data(0).view(np.uint8) + assert np.all(arr == expected_val), \ + f"Stream 0: expected all {expected_val}, got {arr}" + assert not stream.read_record(), "Stream 0 should be exhausted" + + +# ---- Stream 1: two-channel uint16, interleaved, offset navigation ---- + +def test_stream1_structure(cpp_written_egg): + with monarch3.Monarch3.open_for_reading(cpp_written_egg) as m: + m.read_header() + stream = m.get_stream(1) + assert stream.n_channels == STREAM1_N_CHANNELS + assert stream.n_acquisitions == STREAM1_N_ACQUISITIONS + assert stream.n_records_in_file == STREAM1_N_RECORDS + assert stream.channel_record_size == STREAM1_REC_SIZE + + +def test_stream1_data_sequential(cpp_written_egg): + with monarch3.Monarch3.open_for_reading(cpp_written_egg) as m: + m.read_header() + stream = m.get_stream(1) + for v0, v1 in STREAM1_VALUES: + assert stream.read_record(), "Unexpected end of stream 1" + ch0 = stream.get_channel_data(0).view(np.uint16) + ch1 = stream.get_channel_data(1).view(np.uint16) + assert np.all(ch0 == v0), f"Stream 1 ch0: expected {v0}, got {ch0}" + assert np.all(ch1 == v1), f"Stream 1 ch1: expected {v1}, got {ch1}" + + +def test_stream1_offset_navigation(cpp_written_egg): + """Verify seeking with non-zero offsets (mirrors M3ReadTest Test 2).""" + with monarch3.Monarch3.open_for_reading(cpp_written_egg) as m: + m.read_header() + stream = m.get_stream(1) + + # Read record 0 (offset=0 from start) + assert stream.read_record(0), "Expected record 0" + ch0 = stream.get_channel_data(0).view(np.uint16) + assert np.all(ch0 == STREAM1_VALUES[0][0]) + + # Skip forward: offset=1 steps to record 2 (crossing to acquisition 1) + assert stream.read_record(1), "Expected record 2" + ch0 = stream.get_channel_data(0).view(np.uint16) + assert np.all(ch0 == STREAM1_VALUES[2][0]) + + # Reread record 2 (offset=-1) + assert stream.read_record(-1), "Expected reread of record 2" + ch0 = stream.get_channel_data(0).view(np.uint16) + assert np.all(ch0 == STREAM1_VALUES[2][0]) + + # Step back to record 1 (offset=-2) + assert stream.read_record(-2), "Expected record 1" + ch0 = stream.get_channel_data(0).view(np.uint16) + assert np.all(ch0 == STREAM1_VALUES[1][0]) + + # Request past end of file + assert not stream.read_record(5), "Expected False for out-of-bounds forward seek" + + # Request before beginning of file + assert not stream.read_record(-5), "Expected False for out-of-bounds backward seek" + + +# ---- Stream 2: three-channel uint8, separate ---- + +def test_stream2_structure(cpp_written_egg): + with monarch3.Monarch3.open_for_reading(cpp_written_egg) as m: + m.read_header() + stream = m.get_stream(2) + assert stream.n_channels == STREAM2_N_CHANNELS + assert stream.n_acquisitions == STREAM2_N_ACQUISITIONS + assert stream.n_records_in_file == STREAM2_N_RECORDS + assert stream.channel_record_size == STREAM2_REC_SIZE + + +def test_stream2_data(cpp_written_egg): + with monarch3.Monarch3.open_for_reading(cpp_written_egg) as m: + m.read_header() + stream = m.get_stream(2) + for v0, v1, v2 in STREAM2_VALUES: + assert stream.read_record(), "Unexpected end of stream 2" + assert np.all(stream.get_channel_data(0).view(np.uint8) == v0) + assert np.all(stream.get_channel_data(1).view(np.uint8) == v1) + assert np.all(stream.get_channel_data(2).view(np.uint8) == v2) + + +def test_stream2_skip_to_second_record(cpp_written_egg): + """Skip directly to record 1 using offset=1 (mirrors M3ReadTest Test 3).""" + with monarch3.Monarch3.open_for_reading(cpp_written_egg) as m: + m.read_header() + stream = m.get_stream(2) + assert stream.read_record(1), "Expected record 1 via skip" + v0, v1, v2 = STREAM2_VALUES[1] + assert np.all(stream.get_channel_data(0).view(np.uint8) == v0) + assert np.all(stream.get_channel_data(1).view(np.uint8) == v1) + assert np.all(stream.get_channel_data(2).view(np.uint8) == v2) + + +# ---- Stream 3: single-channel float32 ---- + +def test_stream3_structure(cpp_written_egg): + with monarch3.Monarch3.open_for_reading(cpp_written_egg) as m: + m.read_header() + stream = m.get_stream(3) + assert stream.n_channels == STREAM3_N_CHANNELS + assert stream.n_acquisitions == STREAM3_N_ACQUISITIONS + assert stream.n_records_in_file == STREAM3_N_RECORDS + assert stream.channel_record_size == STREAM3_REC_SIZE + + +def test_stream3_data(cpp_written_egg): + with monarch3.Monarch3.open_for_reading(cpp_written_egg) as m: + m.read_header() + stream = m.get_stream(3) + for expected_val in STREAM3_VALUES: + assert stream.read_record(), "Unexpected end of stream 3" + arr = stream.get_channel_data(0).view(np.float32) + assert np.all(np.isclose(arr, expected_val, rtol=1e-6)), \ + f"Stream 3: expected ~{expected_val}, got {arr}" diff --git a/Monarch3/Tests/Validation/test_python_write_cpp_read.py b/Monarch3/Tests/Validation/test_python_write_cpp_read.py new file mode 100644 index 0000000..1243a29 --- /dev/null +++ b/Monarch3/Tests/Validation/test_python_write_cpp_read.py @@ -0,0 +1,100 @@ +# test_python_write_cpp_read.py +# Validates that a file written by the Python binding is correctly read by the C++ +# M3ReadTest executable. The Python write reproduces the same 4-stream structure as +# M3WriteTest so that M3ReadTest's built-in checks pass without modification. + +import numpy as np +import subprocess +import monarch3 + +from conftest import ( + HEADER_RUN_DURATION, HEADER_TIMESTAMP, HEADER_DESCRIPTION, + STREAM0_REC_SIZE, STREAM0_VALUES, + STREAM1_REC_SIZE, STREAM1_VALUES, + STREAM2_REC_SIZE, STREAM2_VALUES, + STREAM3_REC_SIZE, STREAM3_VALUES, +) + + +def write_standard_egg(path): + """Write the same 4-stream structure as M3WriteTest using the Python binding. + + This is also imported by test_python_write_python_read.py to avoid + duplicating the write logic. + """ + with monarch3.Monarch3.open_for_writing(path) as m: + hdr = m.get_header() + hdr.filename = path + hdr.run_duration = HEADER_RUN_DURATION + hdr.timestamp = HEADER_TIMESTAMP + hdr.description = HEADER_DESCRIPTION + + s0 = hdr.add_stream( + "1-channel device", + 500, STREAM0_REC_SIZE, 1, 1, + monarch3.sDigitizedUS, 8, monarch3.sBitsAlignedLeft, + ) + s1 = hdr.add_stream( + "2-channel device", + 2, monarch3.sInterleaved, + 250, STREAM1_REC_SIZE, 1, 2, + monarch3.sDigitizedUS, 16, monarch3.sBitsAlignedLeft, + ) + s2 = hdr.add_stream( + "3-channel device", + 3, monarch3.sSeparate, + 100, STREAM2_REC_SIZE, 1, 1, + monarch3.sDigitizedUS, 8, monarch3.sBitsAlignedLeft, + ) + s3 = hdr.add_stream( + "Floating-point device", + 100, STREAM3_REC_SIZE, 1, 4, + monarch3.sAnalog, 8, monarch3.sBitsAlignedLeft, + ) + m.write_header() + + # Stream 0: 2 records in 1 acquisition + stream = m.get_stream(s0) + for i, val in enumerate(STREAM0_VALUES): + stream.get_channel_data(0)[:] = val + stream.write_record(i == 0) + + # Stream 1: 3 records in 2 acquisitions + # acq 0: rec 0 (is_new=True) + # acq 1: rec 1 (is_new=True), rec 2 (is_new=False) + stream = m.get_stream(s1) + for i, (v0, v1) in enumerate(STREAM1_VALUES): + stream.get_channel_data(0).view(np.uint16)[:] = v0 + stream.get_channel_data(1).view(np.uint16)[:] = v1 + is_new = (i == 0 or i == 1) + stream.write_record(is_new) + + # Stream 2: 2 records in 1 acquisition + stream = m.get_stream(s2) + for i, (v0, v1, v2) in enumerate(STREAM2_VALUES): + stream.get_channel_data(0)[:] = v0 + stream.get_channel_data(1)[:] = v1 + stream.get_channel_data(2)[:] = v2 + stream.write_record(i == 0) + + # Stream 3: 2 records, each in its own acquisition + stream = m.get_stream(s3) + for val in STREAM3_VALUES: + stream.get_channel_data(0).view(np.float32)[:] = val + stream.write_record(True) + + +def test_python_write_cpp_read(tmp_egg): + """Write the standard 4-stream file from Python, then validate with M3ReadTest.""" + write_standard_egg(tmp_egg) + + result = subprocess.run( + ["M3ReadTest", tmp_egg], + capture_output=True, + text=True, + ) + assert result.returncode == 0, ( + f"M3ReadTest failed on Python-written file (return code {result.returncode}):\n" + f"stdout:\n{result.stdout}\n" + f"stderr:\n{result.stderr}" + ) diff --git a/Monarch3/Tests/Validation/test_python_write_python_read.py b/Monarch3/Tests/Validation/test_python_write_python_read.py new file mode 100644 index 0000000..d7ac312 --- /dev/null +++ b/Monarch3/Tests/Validation/test_python_write_python_read.py @@ -0,0 +1,91 @@ +# test_python_write_python_read.py +# Pure Python round-trip: write with the Python binding, read back with the Python +# binding. Isolates the binding from the C++ executables. + +import numpy as np +import monarch3 + +from conftest import ( + HEADER_RUN_DURATION, HEADER_DESCRIPTION, + STREAM0_N_CHANNELS, STREAM0_N_ACQUISITIONS, STREAM0_N_RECORDS, STREAM0_REC_SIZE, STREAM0_VALUES, + STREAM1_N_CHANNELS, STREAM1_N_ACQUISITIONS, STREAM1_N_RECORDS, STREAM1_REC_SIZE, STREAM1_VALUES, + STREAM2_N_CHANNELS, STREAM2_N_ACQUISITIONS, STREAM2_N_RECORDS, STREAM2_REC_SIZE, STREAM2_VALUES, + STREAM3_N_CHANNELS, STREAM3_N_ACQUISITIONS, STREAM3_N_RECORDS, STREAM3_REC_SIZE, STREAM3_VALUES, +) +from test_python_write_cpp_read import write_standard_egg + + +def test_header_round_trip(tmp_egg): + write_standard_egg(tmp_egg) + with monarch3.Monarch3.open_for_reading(tmp_egg) as m: + m.read_header() + hdr = m.get_header() + assert hdr.run_duration == HEADER_RUN_DURATION + assert hdr.description == HEADER_DESCRIPTION + assert hdr.n_streams == 4 + + +def test_stream0_round_trip(tmp_egg): + write_standard_egg(tmp_egg) + with monarch3.Monarch3.open_for_reading(tmp_egg) as m: + m.read_header() + stream = m.get_stream(0) + assert stream.n_channels == STREAM0_N_CHANNELS + assert stream.n_acquisitions == STREAM0_N_ACQUISITIONS + assert stream.n_records_in_file == STREAM0_N_RECORDS + assert stream.channel_record_size == STREAM0_REC_SIZE + for expected_val in STREAM0_VALUES: + assert stream.read_record(), "Unexpected end of stream 0" + arr = stream.get_channel_data(0).view(np.uint8) + assert np.all(arr == expected_val), \ + f"Stream 0: expected all {expected_val}, got {arr}" + assert not stream.read_record(), "Stream 0 should be exhausted" + + +def test_stream1_round_trip(tmp_egg): + write_standard_egg(tmp_egg) + with monarch3.Monarch3.open_for_reading(tmp_egg) as m: + m.read_header() + stream = m.get_stream(1) + assert stream.n_channels == STREAM1_N_CHANNELS + assert stream.n_acquisitions == STREAM1_N_ACQUISITIONS + assert stream.n_records_in_file == STREAM1_N_RECORDS + assert stream.channel_record_size == STREAM1_REC_SIZE + for v0, v1 in STREAM1_VALUES: + assert stream.read_record(), "Unexpected end of stream 1" + ch0 = stream.get_channel_data(0).view(np.uint16) + ch1 = stream.get_channel_data(1).view(np.uint16) + assert np.all(ch0 == v0), f"Stream 1 ch0: expected {v0}, got {ch0}" + assert np.all(ch1 == v1), f"Stream 1 ch1: expected {v1}, got {ch1}" + + +def test_stream2_round_trip(tmp_egg): + write_standard_egg(tmp_egg) + with monarch3.Monarch3.open_for_reading(tmp_egg) as m: + m.read_header() + stream = m.get_stream(2) + assert stream.n_channels == STREAM2_N_CHANNELS + assert stream.n_acquisitions == STREAM2_N_ACQUISITIONS + assert stream.n_records_in_file == STREAM2_N_RECORDS + assert stream.channel_record_size == STREAM2_REC_SIZE + for v0, v1, v2 in STREAM2_VALUES: + assert stream.read_record(), "Unexpected end of stream 2" + assert np.all(stream.get_channel_data(0).view(np.uint8) == v0) + assert np.all(stream.get_channel_data(1).view(np.uint8) == v1) + assert np.all(stream.get_channel_data(2).view(np.uint8) == v2) + + +def test_stream3_round_trip(tmp_egg): + write_standard_egg(tmp_egg) + with monarch3.Monarch3.open_for_reading(tmp_egg) as m: + m.read_header() + stream = m.get_stream(3) + assert stream.n_channels == STREAM3_N_CHANNELS + assert stream.n_acquisitions == STREAM3_N_ACQUISITIONS + assert stream.n_records_in_file == STREAM3_N_RECORDS + assert stream.channel_record_size == STREAM3_REC_SIZE + for expected_val in STREAM3_VALUES: + assert stream.read_record(), "Unexpected end of stream 3" + arr = stream.get_channel_data(0).view(np.float32) + assert np.all(np.isclose(arr, expected_val, rtol=1e-6)), \ + f"Stream 3: expected ~{expected_val}, got {arr}" From dee71ba02040be14dafcf64f82cb2340eb3f3a6b Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Wed, 19 Aug 2026 14:46:15 -0700 Subject: [PATCH 03/30] Update HDF5 usage: 1.10.1 is now the minimum required version; v2.0.0 and up just use provided CMake targets. --- CMakeLists.txt | 117 ++++++++++++++++++++++--------------------------- 1 file changed, 52 insertions(+), 65 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 398ed9c..7122d55 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -37,82 +37,69 @@ if( Monarch_BUILD_MONARCH3 ) find_package( HDF5 REQUIRED COMPONENTS CXX ) include_directories( BEFORE ${PROJECT_SOURCE_DIR}/Monarch3 ) - # HDF5 doesn't currently define imported targets. Therefore we have to do it ourselves. + if( HDF5_VERSION VERSION_LESS "1.10.1" ) + message( FATAL_ERROR "HDF5 must be v1.10.1 or newer" ) + elseif( HDF5_VERSION VERSION_GREATER_EQUAL "2.0.0" ) + list( APPEND PUBLIC_EXT_LIBS_M3 hdf5::hdf5_cpp ) + else() # 1.10.1 ... <2.0.0 - # Library variables returned by the FindHDF5 module are paths to the actual libraries. - # We'll store all libraries, regardless of platform, in a single list. - set( ALL_HDF5_LIBRARIES ) - if( WIN32 ) + # HDF5 doesn't currently define imported targets. Therefore we have to do it ourselves. - # In testing on 3/24/16 I found that the .dlls were being found, not the .libs. I don't understand why, but this fixes the problem if it occurs. - # HDF5_CXX_LIBRARIES was blank in testing, so I commented out the first line. - #string( REPLACE ".dll" ".lib" HDF5_CXX_LIBRARIES ${HDF5_CXX_LIBRARIES} ) - string( REPLACE ".dll" ".lib" HDF5_C_LIBRARY ${HDF5_C_LIBRARY} ) - string( REPLACE ".dll" ".lib" HDF5_CXX_LIBRARY ${HDF5_CXX_LIBRARY} ) + # Library variables returned by the FindHDF5 module are paths to the actual libraries. + # We'll store all libraries, regardless of platform, in a single list. + set( ALL_HDF5_LIBRARIES ) + if( WIN32 ) - list( APPEND ALL_HDF5_LIBRARIES ${HDF5_C_LIBRARY} ${HDF5_CXX_LIBRARY} ) + # In testing on 3/24/16 I found that the .dlls were being found, not the .libs. I don't understand why, but this fixes the problem if it occurs. + # HDF5_CXX_LIBRARIES was blank in testing, so I commented out the first line. + #string( REPLACE ".dll" ".lib" HDF5_CXX_LIBRARIES ${HDF5_CXX_LIBRARIES} ) + string( REPLACE ".dll" ".lib" HDF5_C_LIBRARY ${HDF5_C_LIBRARY} ) + string( REPLACE ".dll" ".lib" HDF5_CXX_LIBRARY ${HDF5_CXX_LIBRARY} ) - message( STATUS "HDF5_C_LIBRARY: ${HDF5_C_LIBRARY}" ) - message( STATUS "HDF5_CXX_LIBRARY: ${HDF5_CXX_LIBRARY}" ) + list( APPEND ALL_HDF5_LIBRARIES ${HDF5_C_LIBRARY} ${HDF5_CXX_LIBRARY} ) - else() + message( STATUS "HDF5_C_LIBRARY: ${HDF5_C_LIBRARY}" ) + message( STATUS "HDF5_CXX_LIBRARY: ${HDF5_CXX_LIBRARY}" ) - list( APPEND ALL_HDF5_LIBRARIES ${HDF5_CXX_LIBRARIES} ) - message( STATUS "HDF5_CXX_LIBRARIES: ${HDF5_CXX_LIBRARIES}" ) - - endif() - - # Now we create an imported library for the C++ library, and assign all other libraries as dependencies - add_library( HDF5CXXLib SHARED IMPORTED ) - foreach( LIBRARY_PATH IN LISTS ALL_HDF5_LIBRARIES ) - get_filename_component( LIBRARY ${LIBRARY_PATH} NAME_WE ) - string(TOLOWER ${LIBRARY} LIBRARY_LOWER ) # avoid any potential case variation - if( LIBRARY_LOWER STREQUAL "libhdf5_cpp" ) - # this is the C++ library - set_target_properties( HDF5CXXLib PROPERTIES IMPORTED_LOCATION ${LIBRARY_PATH} ) - - # We want the relevant include directories to be propagated as part of the HDF5 library properties - # HDF5_INCLUDE_DIR is deprecated, but is placed here to ensure the directory is picked up if an older version of cmake is used - target_include_directories( HDF5CXXLib INTERFACE ${HDF5_INCLUDE_DIRS} ${HDF5_INCLUDE_DIR} ) else() - # this is a dependency library - add_library( ${LIBRARY} SHARED IMPORTED ) - set_target_properties( ${LIBRARY} PROPERTIES IMPORTED_LOCATION ${LIBRARY_PATH} ) - # We want the dependency libraries to be propagated as part of the HDF5 library properties - target_link_libraries( HDF5CXXLib INTERFACE ${LIBRARY} ) + list( APPEND ALL_HDF5_LIBRARIES ${HDF5_CXX_LIBRARIES} ) + message( STATUS "HDF5_CXX_LIBRARIES: ${HDF5_CXX_LIBRARIES}" ) + endif() - endforeach() - # Add the imported library to the set of external libraries - list( APPEND PUBLIC_EXT_LIBS_M3 HDF5CXXLib ) + # Now we create an imported library for the C++ library, and assign all other libraries as dependencies + add_library( HDF5CXXLib SHARED IMPORTED ) + foreach( LIBRARY_PATH IN LISTS ALL_HDF5_LIBRARIES ) + get_filename_component( LIBRARY ${LIBRARY_PATH} NAME_WE ) + string(TOLOWER ${LIBRARY} LIBRARY_LOWER ) # avoid any potential case variation + if( LIBRARY_LOWER STREQUAL "libhdf5_cpp" ) + # this is the C++ library + set_target_properties( HDF5CXXLib PROPERTIES IMPORTED_LOCATION ${LIBRARY_PATH} ) + + # We want the relevant include directories to be propagated as part of the HDF5 library properties + # HDF5_INCLUDE_DIR is deprecated, but is placed here to ensure the directory is picked up if an older version of cmake is used + target_include_directories( HDF5CXXLib INTERFACE ${HDF5_INCLUDE_DIRS} ${HDF5_INCLUDE_DIR} ) + else() + # this is a dependency library + add_library( ${LIBRARY} SHARED IMPORTED ) + set_target_properties( ${LIBRARY} PROPERTIES IMPORTED_LOCATION ${LIBRARY_PATH} ) + + # We want the dependency libraries to be propagated as part of the HDF5 library properties + target_link_libraries( HDF5CXXLib INTERFACE ${LIBRARY} ) + endif() + endforeach() + + # Add the imported library to the set of external libraries + list( APPEND PUBLIC_EXT_LIBS_M3 HDF5CXXLib ) - # The following is a work-around to provide compatibility with the HDF5 1.8 API and the HDF5 1.10 API - # It should be removed in favor of the 1.10 API once we're able to make 1.10 a requirement - # If you upgrade your version of HDF5, you'll need to clear your CMake cache before rebuilding - # First, test for the version of HDF5 in use - include( CheckCXXSourceCompiles ) - set( CMAKE_REQUIRED_QUIET FALSE ) - set( CMAKE_REQUIRED_LIBRARIES ${HDF5_CXX_LIBRARIES} ) - set( CMAKE_REQUIRED_INCLUDES ${HDF5_INCLUDE_DIRS} ) - check_cxx_source_compiles(" - #include - int main() - { - H5::CommonFG* tester = new H5::H5File(\"new_file.h5\", H5F_ACC_TRUNC); - tester->openGroup(\"my_group\"); - return 0; - } - " HAVE_OLD_HDF5 ) - - # Then set preprocessor macros accordingly - if( ${HAVE_OLD_HDF5} ) - message( STATUS "HDF5 is version 1.10.0 or older" ) - set( HDF5_DEFINITIONS -DHAS_ATTR_IFC=H5::H5Location -DHAS_GRP_IFC=H5::CommonFG) - else( ${HAVE_OLD_HDF5} ) - message( STATUS "HDF5 is version 1.10.1 or newer" ) - set( HDF5_DEFINITIONS -DHAS_ATTR_IFC=H5::H5Object -DHAS_GRP_IFC=H5::H5Object ) - endif( ${HAVE_OLD_HDF5} ) + endif() + + # The following macros were originally introduced to handle compatibility with the HDF5 1.8 API and the HDF5 1.10.1 API + # 1.10 is now the minimum version (as of 8/19/2026) so the workaround for v1.8 has been removed. + # But these definitions still need to be used until the source code is adjusted. + message( STATUS "HDF5 is version 1.10.1 or newer" ) + set( HDF5_DEFINITIONS -DHAS_ATTR_IFC=H5::H5Object -DHAS_GRP_IFC=H5::H5Object ) endif( Monarch_BUILD_MONARCH3 ) From 868d55accdce7d97544ad3f82bff9f0c95318b0d Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Wed, 19 Aug 2026 14:46:28 -0700 Subject: [PATCH 04/30] Namespace and constness fix --- Monarch3/python/m3monarch_pybind.hh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Monarch3/python/m3monarch_pybind.hh b/Monarch3/python/m3monarch_pybind.hh index 1b50144..20eadc5 100644 --- a/Monarch3/python/m3monarch_pybind.hh +++ b/Monarch3/python/m3monarch_pybind.hh @@ -49,7 +49,7 @@ namespace monarch3_pybind // ---- Factory methods ---- .def_static( "open_for_reading", []( const std::string& filename ) -> Monarch3Ptr { - return Monarch3Ptr( Monarch3::OpenForReading( filename ) ); + return Monarch3Ptr( const_cast(monarch3::Monarch3::OpenForReading( filename )) ); }, pybind11::arg( "filename" ), "Open an existing egg file for reading.\n" @@ -57,7 +57,7 @@ namespace monarch3_pybind MONARCH3_BIND_CALL_GUARD_STREAMS ) .def_static( "open_for_writing", []( const std::string& filename ) -> Monarch3Ptr { - return Monarch3Ptr( Monarch3::OpenForWriting( filename ) ); + return Monarch3Ptr( monarch3::Monarch3::OpenForWriting( filename ) ); }, pybind11::arg( "filename" ), "Create or overwrite an egg file for writing.\n" From d665786e116c4341eab425fdcce6daa38f1610cd Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Wed, 19 Aug 2026 15:24:18 -0700 Subject: [PATCH 05/30] Fixing issues with the initial version of the validation tests --- Monarch3/Executables/M3ReadTest.cc | 18 +++--- Monarch3/Tests/Validation/conftest.py | 12 ++-- .../Validation/test_cpp_write_python_read.py | 36 ++++++++---- Monarch3/python/m3header_pybind.hh | 57 +++++++++++++------ 4 files changed, 82 insertions(+), 41 deletions(-) diff --git a/Monarch3/Executables/M3ReadTest.cc b/Monarch3/Executables/M3ReadTest.cc index 443be5e..5fb7ee2 100644 --- a/Monarch3/Executables/M3ReadTest.cc +++ b/Monarch3/Executables/M3ReadTest.cc @@ -112,35 +112,35 @@ int main( const int argc, const char** argv ) return RETURN_ERROR; } - LINFO( mlog, "Skip to the third record, crossing to the next acquisition (record 2; acquisition 1)" ); - if( ! ReadRecordCheck( tStream1, 1, tStrHeader1.GetDataFormat() ) ) + LINFO( mlog, "Skip to the second record in acquisition 1 (record 2 in file), crossing to the next acquisition" ); + if( ! tStream1->ReadRecord( 1, false ) ) { - LERROR( mlog, "Failed read record check" ); + LERROR( mlog, "Failed to read record" ); return RETURN_ERROR; } - LINFO( mlog, "Reread the third record (record 2; acquisition 1)" ); + LINFO( mlog, "Reread the current record (record 2 in file; record 1 in acquisition 1)" ); if( ! ReadRecordCheck( tStream1, -1, tStrHeader1.GetDataFormat() ) ) { LERROR( mlog, "Failed read record check" ); return RETURN_ERROR; } - LINFO( mlog, "Go backwards to the second record (record 1; acquisition 1)" ); + LINFO( mlog, "Go backwards to the first record in acquisition 1 (record 1 in file)" ); if( ! ReadRecordCheck( tStream1, -2, tStrHeader1.GetDataFormat() ) ) { LERROR( mlog, "Failed read record check" ); return RETURN_ERROR; } - LINFO( mlog, "Go backwards to the first record (record 1; acquisition 0)" ); + LINFO( mlog, "Go backwards to acquisition 0 (record 0 in file)" ); if( ! ReadRecordCheck( tStream1, -2, tStrHeader1.GetDataFormat() ) ) { LERROR( mlog, "Failed read record check" ); return RETURN_ERROR; } - LINFO( mlog, "Reread the first record (record 1; acquisition 0)" ); + LINFO( mlog, "Reread the first record (record 0 in file; acquisition 0)" ); if( ! ReadRecordCheck( tStream1, -1, tStrHeader1.GetDataFormat() ) ) { LERROR( mlog, "Failed read record check" ); @@ -177,9 +177,9 @@ int main( const int argc, const char** argv ) } LINFO( mlog, "Skipping immediately to the second record (record 1)" ); - if( ! ReadRecordCheck( tStream2, 1, tStrHeader2.GetDataFormat() ) ) + if( ! tStream2->ReadRecord( 1, false ) ) { - LERROR( mlog, "Failed read record check" ); + LERROR( mlog, "Failed to read record" ); return RETURN_ERROR; } diff --git a/Monarch3/Tests/Validation/conftest.py b/Monarch3/Tests/Validation/conftest.py index 7ab52e9..3056ead 100644 --- a/Monarch3/Tests/Validation/conftest.py +++ b/Monarch3/Tests/Validation/conftest.py @@ -57,10 +57,14 @@ def tmp_egg(tmp_path): return str(tmp_path / "test.egg") -@pytest.fixture -def cpp_written_egg(tmp_path): - """Run M3WriteTest to produce the gold-standard egg file; return its path.""" - path = str(tmp_path / "cpp_written.egg") +@pytest.fixture(scope="session") +def cpp_written_egg(tmp_path_factory): + """Run M3WriteTest once per session to produce the gold-standard egg file. + + Session scope avoids spawning multiple concurrent M3WriteTest processes (which + can be killed by macOS resource throttling when pytest runs tests back-to-back). + """ + path = str(tmp_path_factory.mktemp("cpp_written") / "cpp_written.egg") result = subprocess.run( ["M3WriteTest", path], capture_output=True, diff --git a/Monarch3/Tests/Validation/test_cpp_write_python_read.py b/Monarch3/Tests/Validation/test_cpp_write_python_read.py index 8d26c35..6fcce6a 100644 --- a/Monarch3/Tests/Validation/test_cpp_write_python_read.py +++ b/Monarch3/Tests/Validation/test_cpp_write_python_read.py @@ -77,30 +77,37 @@ def test_stream1_data_sequential(cpp_written_egg): def test_stream1_offset_navigation(cpp_written_egg): - """Verify seeking with non-zero offsets (mirrors M3ReadTest Test 2).""" + """Verify seeking with non-zero offsets (mirrors M3ReadTest Test 2). + + Key: ReadRecord uses aIfNewAcqStartAtFirstRec=True by default, which snaps to the + first record of any new acquisition entered. Pass False to land exactly on the + offset-targeted record when crossing an acquisition boundary. + """ with monarch3.Monarch3.open_for_reading(cpp_written_egg) as m: m.read_header() stream = m.get_stream(1) - # Read record 0 (offset=0 from start) + # Read record 0 (offset=0 from start); acq 0, rec 0 assert stream.read_record(0), "Expected record 0" ch0 = stream.get_channel_data(0).view(np.uint16) - assert np.all(ch0 == STREAM1_VALUES[0][0]) + assert np.all(ch0 == STREAM1_VALUES[0][0]), f"Expected {STREAM1_VALUES[0][0]}, got {ch0}" - # Skip forward: offset=1 steps to record 2 (crossing to acquisition 1) - assert stream.read_record(1), "Expected record 2" + # Skip forward with offset=1 crossing to acquisition 1. + # aIfNewAcqStartAtFirstRec=False: land on the exact target (file rec 2, acq 1 rec 1, + # values 10000/20000) rather than snapping back to the start of acq 1. + assert stream.read_record(1, False), "Expected record 2" ch0 = stream.get_channel_data(0).view(np.uint16) - assert np.all(ch0 == STREAM1_VALUES[2][0]) + assert np.all(ch0 == STREAM1_VALUES[2][0]), f"Expected {STREAM1_VALUES[2][0]}, got {ch0}" # Reread record 2 (offset=-1) assert stream.read_record(-1), "Expected reread of record 2" ch0 = stream.get_channel_data(0).view(np.uint16) - assert np.all(ch0 == STREAM1_VALUES[2][0]) + assert np.all(ch0 == STREAM1_VALUES[2][0]), f"Expected {STREAM1_VALUES[2][0]}, got {ch0}" # Step back to record 1 (offset=-2) assert stream.read_record(-2), "Expected record 1" ch0 = stream.get_channel_data(0).view(np.uint16) - assert np.all(ch0 == STREAM1_VALUES[1][0]) + assert np.all(ch0 == STREAM1_VALUES[1][0]), f"Expected {STREAM1_VALUES[1][0]}, got {ch0}" # Request past end of file assert not stream.read_record(5), "Expected False for out-of-bounds forward seek" @@ -133,16 +140,25 @@ def test_stream2_data(cpp_written_egg): def test_stream2_skip_to_second_record(cpp_written_egg): - """Skip directly to record 1 using offset=1 (mirrors M3ReadTest Test 3).""" + """Skip directly to record 1 using offset=1 (mirrors M3ReadTest Test 3). + + Stream 2 has only 1 acquisition, so aIfNewAcqStartAtFirstRec does not matter here; + the snap-to-first-in-acq only fires when entering a *new* acquisition. However, + the very first ReadRecord call always treats the stream as entering a new acquisition, + so we must pass False to land on the exact offset target (rec 1) rather than rec 0. + """ with monarch3.Monarch3.open_for_reading(cpp_written_egg) as m: m.read_header() stream = m.get_stream(2) - assert stream.read_record(1), "Expected record 1 via skip" + assert stream.read_record(1, False), "Expected record 1 via skip" v0, v1, v2 = STREAM2_VALUES[1] assert np.all(stream.get_channel_data(0).view(np.uint8) == v0) assert np.all(stream.get_channel_data(1).view(np.uint8) == v1) assert np.all(stream.get_channel_data(2).view(np.uint8) == v2) + # Verify that a backward seek past the beginning returns False + assert not stream.read_record(-3), "Expected False for out-of-bounds backward seek" + # ---- Stream 3: single-channel float32 ---- diff --git a/Monarch3/python/m3header_pybind.hh b/Monarch3/python/m3header_pybind.hh index 72b8894..d881675 100644 --- a/Monarch3/python/m3header_pybind.hh +++ b/Monarch3/python/m3header_pybind.hh @@ -90,15 +90,26 @@ namespace monarch3_pybind .def_property( "description", []( const monarch3::M3Header& h ) { return h.Description(); }, []( monarch3::M3Header& h, const std::string& v ) { h.Description() = v; } ) - // AddStream overloads + // AddStream overloads — lambda wrappers are needed because pybind11 cannot + // implicitly convert Python None to a raw std::vector* pointer. .def( "add_stream", - ( unsigned ( monarch3::M3Header::* )( - const std::string&, - uint32_t, uint32_t, uint32_t, - uint32_t, uint32_t, - uint32_t, uint32_t, - std::vector< unsigned >* ) ) - &monarch3::M3Header::AddStream, + []( monarch3::M3Header& h, + const std::string& source, + uint32_t acq_rate, uint32_t rec_size, uint32_t sample_size, + uint32_t data_type_size, uint32_t data_format, + uint32_t bit_depth, uint32_t bit_alignment, + pybind11::object chan_vec_py ) -> unsigned + { + std::vector< unsigned > buf; + std::vector< unsigned >* ptr = nullptr; + if( ! chan_vec_py.is_none() ) + { + buf = chan_vec_py.cast< std::vector< unsigned > >(); + ptr = &buf; + } + return h.AddStream( source, acq_rate, rec_size, sample_size, + data_type_size, data_format, bit_depth, bit_alignment, ptr ); + }, pybind11::arg( "source" ), pybind11::arg( "acq_rate" ), pybind11::arg( "rec_size" ), @@ -107,17 +118,27 @@ namespace monarch3_pybind pybind11::arg( "data_format" ), pybind11::arg( "bit_depth" ), pybind11::arg( "bit_alignment" ), - pybind11::arg( "chan_vec" ) = nullptr, + pybind11::arg( "chan_vec" ) = pybind11::none(), "Add a single-channel stream; returns the stream number" ) .def( "add_stream", - ( unsigned ( monarch3::M3Header::* )( - const std::string&, - uint32_t, uint32_t, - uint32_t, uint32_t, uint32_t, - uint32_t, uint32_t, - uint32_t, uint32_t, - std::vector< unsigned >* ) ) - &monarch3::M3Header::AddStream, + []( monarch3::M3Header& h, + const std::string& source, + uint32_t n_channels, uint32_t channel_format, + uint32_t acq_rate, uint32_t rec_size, uint32_t sample_size, + uint32_t data_type_size, uint32_t data_format, + uint32_t bit_depth, uint32_t bit_alignment, + pybind11::object chan_vec_py ) -> unsigned + { + std::vector< unsigned > buf; + std::vector< unsigned >* ptr = nullptr; + if( ! chan_vec_py.is_none() ) + { + buf = chan_vec_py.cast< std::vector< unsigned > >(); + ptr = &buf; + } + return h.AddStream( source, n_channels, channel_format, acq_rate, rec_size, + sample_size, data_type_size, data_format, bit_depth, bit_alignment, ptr ); + }, pybind11::arg( "source" ), pybind11::arg( "n_channels" ), pybind11::arg( "channel_format" ), @@ -128,7 +149,7 @@ namespace monarch3_pybind pybind11::arg( "data_format" ), pybind11::arg( "bit_depth" ), pybind11::arg( "bit_alignment" ), - pybind11::arg( "chan_vec" ) = nullptr, + pybind11::arg( "chan_vec" ) = pybind11::none(), "Add a multi-channel stream; returns the stream number" ) .def( "__repr__", []( const monarch3::M3Header& h ) { std::ostringstream out; From 70d24467f73b4e378634e4b1f8d571a2c2fc7661 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Wed, 19 Aug 2026 15:36:28 -0700 Subject: [PATCH 06/30] Added changelog.md --- changelog.md | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 changelog.md diff --git a/changelog.md b/changelog.md new file mode 100644 index 0000000..e42429f --- /dev/null +++ b/changelog.md @@ -0,0 +1,42 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +Types of changes: Added, Changed, Deprecated, Removed, Fixed, Security + +## [Unreleased] ([3.9.0] - 2026-08-19) + +### Added + +- Python interface for Monarch3 via pybind11 (`Monarch3/python/`), exposing `M3Header`, + `M3StreamHeader`, `M3ChannelHeader`, `M3Stream`, `M3Record`, and `Monarch3` to Python + as the `monarch3` module; record data is returned as zero-copy numpy `uint8` arrays +- Context manager (`with` statement) support for `Monarch3` objects; `FinishReading()` + or `FinishWriting()` is called automatically on exit +- `Monarch_BUILD_PYTHON` CMake option to enable the Python binding (requires pybind11 and + Python 3 development headers) +- Validation test suite (`Monarch3/Tests/Validation/`) using pytest and CTest: + - C++ write → Python read (`test_cpp_write_python_read.py`) + - Python write → C++ read (`test_python_write_cpp_read.py`) + - Python write → Python read (`test_python_write_python_read.py`) +- `Monarch_ENABLE_TESTING` CMake option now activates `enable_testing()` and the CTest + validation suite in addition to the existing C++ test executables +- Documentation pages: `UsageMonarch3Python.rst` and `TestingMonarch3.rst` +- GitHub Actions workflow (`.github/workflows/run_tests.yaml`) with separate jobs for + Monarch3 (including Python validation tests) and Monarch2, plus a Release job + +### Fixed + +- `M3ReadTest`: offset-navigation test (Test 2) now passes `aIfNewAcqStartAtFirstRec=false` + when crossing an acquisition boundary to land on the exact target record, and corrects + a `ReadRecord(-2)` call that would step before the start of the file +- `M3ReadTest`: Test 3 (stream 2 skip) likewise uses `aIfNewAcqStartAtFirstRec=false` for + the initial offset skip + +### Changed + +- HDF5 minimum version raised to 1.10.1; the v1.8 API compatibility workaround has been + removed from `CMakeLists.txt` From 00b4f87de14070c67d6354aa8387ace5afc08c83 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Wed, 19 Aug 2026 15:48:32 -0700 Subject: [PATCH 07/30] Added GHA workflow file --- .github/workflows/run_tests.yaml | 183 +++++++++++++++++++++++++++++++ 1 file changed, 183 insertions(+) create mode 100644 .github/workflows/run_tests.yaml diff --git a/.github/workflows/run_tests.yaml b/.github/workflows/run_tests.yaml new file mode 100644 index 0000000..51a85de --- /dev/null +++ b/.github/workflows/run_tests.yaml @@ -0,0 +1,183 @@ +name: Run Tests + +on: + push: + branches: [ 'main', 'develop' ] + tags: ['*'] + pull_request: + + workflow_dispatch: + +jobs: + + Monarch3Tests: + + runs-on: ${{ matrix.os }} + + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04, ubuntu-24.04-arm, ubuntu-22.04, ubuntu-22.04-arm, macos-26, macos-15-intel, macos-15] + + steps: + + - name: Checkout the repo + uses: actions/checkout@v7 + with: + submodules: recursive + + - name: Select CC version + # We require GCC 12 or higher if using GCC + # The default GCC for Ubuntu 22.04 is 11, but 12 is available + if: ${{ matrix.os == 'ubuntu-22.04' || matrix.os == 'ubuntu-22.04-arm' }} + run: | + echo "CC=gcc-12" >> $GITHUB_ENV + echo "CXX=g++-12" >> $GITHUB_ENV + + - name: Install dependencies -- Mac + # pybind11 is manually installed because we need to use v3.0.0 (as of 8/21/25) + # Fix the /usr/local directory structure + if: startsWith(matrix.os, 'macos') + run: | + brew install \ + hdf5 \ + boost \ + pybind11 \ + rapidjson \ + yaml-cpp + pip install pytest + + - name: Install dependencies -- Linux + if: startsWith(matrix.os, 'ubuntu') + run: | + sudo apt-get update + DEBIAN_FRONTEND=noninteractive sudo apt-get install -y \ + libhdf5-dev \ + libboost-all-dev \ + rapidjson-dev \ + libyaml-cpp-dev + git clone https://github.com/pybind/pybind11.git + cd pybind11 + git checkout v3.1.0 + mkdir build + cd build + sudo cmake -DPYBIND11_TEST=FALSE .. + sudo make -j2 install + cd ../.. + pip install pytest + + - name: Configure + run: | + mkdir build + cd build + cmake .. \ + -DMonarch_BUILD_MONARCH3=TRUE \ + -DMonarch_ENABLE_TESTING=TRUE \ + -DMonarch_BUILD_PYTHON=TRUE + + - name: Build + run: | + cd build + make -j2 install + + - name: Run tests + run: | + cd build + ctest --output-on-failure + +# For debugging +# - name: Setup tmate session +# if: ${{ ! success() }} +# uses: mxschmitt/action-tmate@v3 + + + Monarch2Tests: + + runs-on: ${{ matrix.os }} + + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04, ubuntu-24.04-arm, ubuntu-22.04, ubuntu-22.04-arm, macos-26, macos-15-intel, macos-15] + + steps: + + - name: Checkout the repo + uses: actions/checkout@v7 + with: + submodules: recursive + + - name: Select CC version + if: ${{ matrix.os == 'ubuntu-22.04' }} + run: | + echo "CC=gcc-12" >> $GITHUB_ENV + echo "CXX=g++-12" >> $GITHUB_ENV + + - name: Install dependencies -- Mac + if: startsWith(matrix.os, 'macos') + run: | + brew install \ + protobuf \ + boost \ + rapidjson \ + yaml-cpp + + - name: Install dependencies -- Linux + if: startsWith(matrix.os, 'ubuntu') + run: | + sudo apt-get update + DEBIAN_FRONTEND=noninteractive sudo apt-get install -y \ + protobuf-compiler \ + libprotobuf-dev \ + libboost-all-dev \ + rapidjson-dev \ + libyaml-cpp-dev + + - name: Configure + run: | + mkdir build + cd build + cmake .. \ + -DMonarch_BUILD_MONARCH2=TRUE \ + -DMonarch_BUILD_MONARCH3=FALSE \ + -DMonarch_ENABLE_TESTING=TRUE + + - name: Build + run: | + cd build + make -j2 install + +# Tests have not been implemented +# - name: Run tests +# run: | +# cd build +# ctest --output-on-failure + +# For debugging +# - name: Setup tmate session +# if: ${{ ! success() }} +# uses: mxschmitt/action-tmate@v3 + + + Release: + + runs-on: ubuntu-24.04 + + if: ${{ github.event_name == 'push' && contains(github.ref, 'refs/tags/') }} + + needs: [Monarch3Tests, Monarch2Tests] + + steps: + + - name: Checkout the repo + uses: actions/checkout@v7 + with: + submodules: recursive + + - name: Release with a changelog + uses: rasmus-saks/release-a-changelog-action@v1.2.0 + with: + github-token: '${{ secrets.GITHUB_TOKEN }}' + path: 'changelog.md' + title-template: 'Monarch v{version} -- Release Notes' + tag-template: 'v{version}' From 1ca632ffae4e10841c6a8f243590ffd084564990 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Wed, 19 Aug 2026 15:53:17 -0700 Subject: [PATCH 08/30] Disable Python in the Monarch2 builds and install numpy in the Monarch3 builds --- .github/workflows/run_tests.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/run_tests.yaml b/.github/workflows/run_tests.yaml index 51a85de..19b3903 100644 --- a/.github/workflows/run_tests.yaml +++ b/.github/workflows/run_tests.yaml @@ -45,7 +45,7 @@ jobs: pybind11 \ rapidjson \ yaml-cpp - pip install pytest + pip install pytest numpy - name: Install dependencies -- Linux if: startsWith(matrix.os, 'ubuntu') @@ -64,7 +64,7 @@ jobs: sudo cmake -DPYBIND11_TEST=FALSE .. sudo make -j2 install cd ../.. - pip install pytest + pip install pytest numpy - name: Configure run: | @@ -140,7 +140,8 @@ jobs: cmake .. \ -DMonarch_BUILD_MONARCH2=TRUE \ -DMonarch_BUILD_MONARCH3=FALSE \ - -DMonarch_ENABLE_TESTING=TRUE + -DMonarch_ENABLE_TESTING=TRUE \ + -DScarab_BUILD_PYTHON=FALSE - name: Build run: | From 4b344362d97c4b9f4475e162db109492edb57df8 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Wed, 19 Aug 2026 16:10:21 -0700 Subject: [PATCH 09/30] Remove unused protobuf::libprotoc --- CMakeLists.txt | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7122d55..a745fee 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -112,10 +112,9 @@ if( Monarch_BUILD_MONARCH2 ) find_package( Protobuf REQUIRED ) include_directories( BEFORE ${PROJECT_SOURCE_DIR}/Monarch2 ${PROJECT_BINARY_DIR}/Monarch2 ${PROJECT_BINARY_DIR} ) - list( APPEND PUBLIC_EXT_LIBS_M2 - protobuf::libprotobuf - protobuf::libprotobuf-lite - protobuf::libprotoc + list( APPEND PUBLIC_EXT_LIBS_M2 + protobuf::libprotobuf + protobuf::libprotobuf-lite ) endif( Monarch_BUILD_MONARCH2 ) From 3e7a6caeb5f52dc1db5becb79848a8a77c05a2c7 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Wed, 19 Aug 2026 16:25:42 -0700 Subject: [PATCH 10/30] Use pip3 to install python packages in mac jobs. Include ubuntu-22.04-arm in the set of jobs that switch to gcc 12 manually. --- .github/workflows/run_tests.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run_tests.yaml b/.github/workflows/run_tests.yaml index 19b3903..7c384be 100644 --- a/.github/workflows/run_tests.yaml +++ b/.github/workflows/run_tests.yaml @@ -45,7 +45,7 @@ jobs: pybind11 \ rapidjson \ yaml-cpp - pip install pytest numpy + pip3 install pytest numpy - name: Install dependencies -- Linux if: startsWith(matrix.os, 'ubuntu') @@ -108,7 +108,7 @@ jobs: submodules: recursive - name: Select CC version - if: ${{ matrix.os == 'ubuntu-22.04' }} + if: ${{ matrix.os == 'ubuntu-22.04' || matrix.os == 'ubuntu-22.04-arm' }} run: | echo "CC=gcc-12" >> $GITHUB_ENV echo "CXX=g++-12" >> $GITHUB_ENV From 1c03da09b7704951ecb0897bf1892723a60dfd57 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Wed, 19 Aug 2026 16:26:43 -0700 Subject: [PATCH 11/30] Move the M2Header operator<<() to within the monarch2 namespace --- Monarch2/M2Header.cc | 40 ++++++++++++++++++++-------------------- Monarch2/M2Header.hh | 6 +++--- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/Monarch2/M2Header.cc b/Monarch2/M2Header.cc index e421d9a..3069281 100644 --- a/Monarch2/M2Header.cc +++ b/Monarch2/M2Header.cc @@ -267,24 +267,24 @@ TimeType M2Header::GetRecordTime0() const { return fProtobufHeader->voltagerange(); } -} -std::ostream& operator<<( std::ostream& out, const monarch2::M2Header& hdr ) -{ - out << "Monarch Header Content: " << "\n"; - out << "\tFilename: " << hdr.GetFilename() << "\n"; - out << "\tAcquisition Mode (# channels): " << hdr.GetAcquisitionMode() << "\n"; - out << "\tAcquisition Rate: " << hdr.GetAcquisitionRate() << " MHz\n"; - out << "\tRun Duration: " << hdr.GetRunDuration() << " ms\n"; - out << "\tRecord Size: " << hdr.GetRecordSize() << "\n"; - out << "\tTimestamp: " << hdr.GetTimestamp() << "\n"; - out << "\tDescription: " << hdr.GetDescription() << "\n"; - out << "\tRun Type: " << hdr.GetRunType() << "\n"; - out << "\tRun Source: " << hdr.GetRunSource() << "\n"; - out << "\tFormat Mode: " << hdr.GetFormatMode() << "\n"; - out << "\tData Type Size: " << hdr.GetDataTypeSize() << " bytes\n"; - out << "\tBit Depth: " << hdr.GetBitDepth() << " bits\n"; - out << "\tVoltage Min: " << hdr.GetVoltageMin() << " V\n"; - out << "\tVoltage Range: " << hdr.GetVoltageRange() << " V\n"; - return out; -} + std::ostream& operator<<( std::ostream& out, const M2Header& hdr ) + { + out << "Monarch Header Content: " << "\n"; + out << "\tFilename: " << hdr.GetFilename() << "\n"; + out << "\tAcquisition Mode (# channels): " << hdr.GetAcquisitionMode() << "\n"; + out << "\tAcquisition Rate: " << hdr.GetAcquisitionRate() << " MHz\n"; + out << "\tRun Duration: " << hdr.GetRunDuration() << " ms\n"; + out << "\tRecord Size: " << hdr.GetRecordSize() << "\n"; + out << "\tTimestamp: " << hdr.GetTimestamp() << "\n"; + out << "\tDescription: " << hdr.GetDescription() << "\n"; + out << "\tRun Type: " << hdr.GetRunType() << "\n"; + out << "\tRun Source: " << hdr.GetRunSource() << "\n"; + out << "\tFormat Mode: " << hdr.GetFormatMode() << "\n"; + out << "\tData Type Size: " << hdr.GetDataTypeSize() << " bytes\n"; + out << "\tBit Depth: " << hdr.GetBitDepth() << " bits\n"; + out << "\tVoltage Min: " << hdr.GetVoltageMin() << " V\n"; + out << "\tVoltage Range: " << hdr.GetVoltageRange() << " V\n"; + return out; + } +} // namespace monarch2 diff --git a/Monarch2/M2Header.hh b/Monarch2/M2Header.hh index ac9fce4..be3dba3 100644 --- a/Monarch2/M2Header.hh +++ b/Monarch2/M2Header.hh @@ -91,9 +91,9 @@ namespace monarch2 }; -} + // Pretty printing method + std::ostream& operator<<( std::ostream& out, const M2Header& hdr ); -// Pretty printing method -std::ostream& operator<<( std::ostream& out, const monarch2::M2Header& hdr ); +} #endif From 6874c5a993fec1cc691c45d39d19ab9e373288ad Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Wed, 19 Aug 2026 16:30:38 -0700 Subject: [PATCH 12/30] Adapt finding of Protobuf to more modern Protobuf installations --- CMakeLists.txt | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index a745fee..f280a59 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -109,7 +109,16 @@ endif( Monarch_BUILD_MONARCH3 ) ########## if( Monarch_BUILD_MONARCH2 ) - find_package( Protobuf REQUIRED ) + # Use config mode so that protobuf's abseil dependency is propagated automatically. + # Modern protobuf (v22 / 4.x+ via brew) depends on abseil internally; the old + # FindProtobuf module mode does not surface that transitive dependency, causing + # undefined absl:: symbols at link time. + find_package( protobuf CONFIG QUIET ) + if( NOT protobuf_FOUND ) + # Fall back to module mode for older protobuf installations that only ship + # a FindProtobuf module and no config file. + find_package( Protobuf REQUIRED ) + endif() include_directories( BEFORE ${PROJECT_SOURCE_DIR}/Monarch2 ${PROJECT_BINARY_DIR}/Monarch2 ${PROJECT_BINARY_DIR} ) list( APPEND PUBLIC_EXT_LIBS_M2 From 2f7e1567874d67edf31d6a852498fa40d1ae4a4b Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Wed, 19 Aug 2026 16:36:20 -0700 Subject: [PATCH 13/30] Macs are picky about modifying the global Python environment . . . --- .github/workflows/run_tests.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/run_tests.yaml b/.github/workflows/run_tests.yaml index 7c384be..136365d 100644 --- a/.github/workflows/run_tests.yaml +++ b/.github/workflows/run_tests.yaml @@ -45,7 +45,9 @@ jobs: pybind11 \ rapidjson \ yaml-cpp - pip3 install pytest numpy + python3 -m venv $(pwd)/venv + source $(pwd)/venv/bin/activate + python3 -m pip install pytest numpy - name: Install dependencies -- Linux if: startsWith(matrix.os, 'ubuntu') From 712b0806600b6e5d3d50016b649bbe1a91bc7d4f Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Wed, 19 Aug 2026 16:36:39 -0700 Subject: [PATCH 14/30] More modernizing protobuf usage --- Monarch2/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Monarch2/CMakeLists.txt b/Monarch2/CMakeLists.txt index 0bd2c98..d6ff969 100644 --- a/Monarch2/CMakeLists.txt +++ b/Monarch2/CMakeLists.txt @@ -7,7 +7,7 @@ set( PROTO_FILES Protobuf/MonarchHeader.proto ) -protobuf_generate_cpp( Monarch_Protobuf_Sources Monarch_Protobuf_Headers ${PROTO_FILES} ) +protobuf_generate( Monarch_Protobuf_Sources Monarch_Protobuf_Headers ${PROTO_FILES} ) #################### From 654f64356f9cae3a868e30eeff1901b9e6c68a04 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Wed, 19 Aug 2026 16:41:59 -0700 Subject: [PATCH 15/30] Fixing the protobuf_generate() usage --- Monarch2/CMakeLists.txt | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/Monarch2/CMakeLists.txt b/Monarch2/CMakeLists.txt index d6ff969..d652c09 100644 --- a/Monarch2/CMakeLists.txt +++ b/Monarch2/CMakeLists.txt @@ -7,15 +7,19 @@ set( PROTO_FILES Protobuf/MonarchHeader.proto ) -protobuf_generate( Monarch_Protobuf_Sources Monarch_Protobuf_Headers ${PROTO_FILES} ) - +protobuf_generate( + LANGUAGE CPP + OUT_VAR Monarch_Protobuf_Sources #Monarch_Protobuf_Headers + PROTOS ${PROTO_FILES} +) +message( STATUS "Generating protobuf source files: ${Monarch_Protobuf_Sources}" ) #################### # monarch2 library # #################### set( MONARCH2_HEADERFILES - ${Monarch_Protobuf_Sources} +# ${Monarch_Protobuf_Sources} M2Exception.hh M2Header.hh M2IO.hh From f83a288908a5d7d38ae02ad9dba28a6d9fcbbbb0 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Wed, 19 Aug 2026 16:51:35 -0700 Subject: [PATCH 16/30] Another attempt to use protobuf_generate correctly --- Monarch2/CMakeLists.txt | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Monarch2/CMakeLists.txt b/Monarch2/CMakeLists.txt index d652c09..266d773 100644 --- a/Monarch2/CMakeLists.txt +++ b/Monarch2/CMakeLists.txt @@ -9,7 +9,7 @@ set( PROTO_FILES protobuf_generate( LANGUAGE CPP - OUT_VAR Monarch_Protobuf_Sources #Monarch_Protobuf_Headers + OUT_VAR Monarch_Protobuf_Output PROTOS ${PROTO_FILES} ) message( STATUS "Generating protobuf source files: ${Monarch_Protobuf_Sources}" ) @@ -18,8 +18,11 @@ message( STATUS "Generating protobuf source files: ${Monarch_Protobuf_Sources}" # monarch2 library # #################### +list( GET Monarch_Protobuf_Output 0 Monarch_Protobuf_Headers ) +list( GET Monarch_Protobuf_Output 1 Monarch_Protobuf_Sources ) + set( MONARCH2_HEADERFILES -# ${Monarch_Protobuf_Sources} + ${Monarch_Protobuf_Headers} M2Exception.hh M2Header.hh M2IO.hh From ace0fe5abf4c23dc44563323daa0303c2dadb6a2 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Wed, 19 Aug 2026 17:01:56 -0700 Subject: [PATCH 17/30] More details being fixed on the Python installation for Monarch3 and the protobuf usage for Monarch2 --- .github/workflows/run_tests.yaml | 5 ++++- Monarch2/CMakeLists.txt | 3 ++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run_tests.yaml b/.github/workflows/run_tests.yaml index 136365d..fb69f51 100644 --- a/.github/workflows/run_tests.yaml +++ b/.github/workflows/run_tests.yaml @@ -66,7 +66,9 @@ jobs: sudo cmake -DPYBIND11_TEST=FALSE .. sudo make -j2 install cd ../.. - pip install pytest numpy + python -m venv $(pwd)/venv + source $(pwd)/venv/bin/activate + python -m pip install pytest numpy - name: Configure run: | @@ -84,6 +86,7 @@ jobs: - name: Run tests run: | + source $(pwd)/venv/bin/activate cd build ctest --output-on-failure diff --git a/Monarch2/CMakeLists.txt b/Monarch2/CMakeLists.txt index 266d773..e341655 100644 --- a/Monarch2/CMakeLists.txt +++ b/Monarch2/CMakeLists.txt @@ -12,7 +12,7 @@ protobuf_generate( OUT_VAR Monarch_Protobuf_Output PROTOS ${PROTO_FILES} ) -message( STATUS "Generating protobuf source files: ${Monarch_Protobuf_Sources}" ) +message( STATUS "Generating protobuf source files: ${Monarch_Protobuf_Output}" ) #################### # monarch2 library # @@ -20,6 +20,7 @@ message( STATUS "Generating protobuf source files: ${Monarch_Protobuf_Sources}" list( GET Monarch_Protobuf_Output 0 Monarch_Protobuf_Headers ) list( GET Monarch_Protobuf_Output 1 Monarch_Protobuf_Sources ) +include_directories( ${CMAKE_CURRENT_BINARY_DIR}/Protobuf ) set( MONARCH2_HEADERFILES ${Monarch_Protobuf_Headers} From fb0a83399d642017d7dec6c9852ed9fc5b92e7ce Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Wed, 19 Aug 2026 17:13:50 -0700 Subject: [PATCH 18/30] With Claude's help we might have this Python thing sorted --- .github/workflows/run_tests.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/run_tests.yaml b/.github/workflows/run_tests.yaml index fb69f51..3f99136 100644 --- a/.github/workflows/run_tests.yaml +++ b/.github/workflows/run_tests.yaml @@ -34,6 +34,14 @@ jobs: echo "CC=gcc-12" >> $GITHUB_ENV echo "CXX=g++-12" >> $GITHUB_ENV + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.x' + + - name: Install Python packages + run: pip install pytest numpy + - name: Install dependencies -- Mac # pybind11 is manually installed because we need to use v3.0.0 (as of 8/21/25) # Fix the /usr/local directory structure @@ -45,9 +53,6 @@ jobs: pybind11 \ rapidjson \ yaml-cpp - python3 -m venv $(pwd)/venv - source $(pwd)/venv/bin/activate - python3 -m pip install pytest numpy - name: Install dependencies -- Linux if: startsWith(matrix.os, 'ubuntu') @@ -65,10 +70,6 @@ jobs: cd build sudo cmake -DPYBIND11_TEST=FALSE .. sudo make -j2 install - cd ../.. - python -m venv $(pwd)/venv - source $(pwd)/venv/bin/activate - python -m pip install pytest numpy - name: Configure run: | @@ -86,7 +87,6 @@ jobs: - name: Run tests run: | - source $(pwd)/venv/bin/activate cd build ctest --output-on-failure From 2cc7e8a5de7031a9f68c88715b993cf83a0bc844 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Wed, 19 Aug 2026 17:21:23 -0700 Subject: [PATCH 19/30] Attempting to fix the RTD docs build --- .readthedocs.yaml | 46 ++++++++++++++++++++++++++++++++++ Documentation/requirements.txt | 2 ++ 2 files changed, 48 insertions(+) create mode 100644 .readthedocs.yaml create mode 100644 Documentation/requirements.txt diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 0000000..fc246c7 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,46 @@ +# .readthedocs.yaml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Set the OS, Python version and other tools you might need +build: + os: ubuntu-22.04 + tools: + python: "3.12" + # You can also specify other tool versions: + # nodejs: "19" + # rust: "1.64" + # golang: "1.19" + apt_packages: + - doxygen + - graphviz + - tree +# jobs: +# pre_build: +# - git submodule update --init --recursive +# - python ./documentation/source/run_doxygen.py +# - mkdir -p $READTHEDOCS_OUTPUT/html +# - mv ./user_doxygen_out/html $READTHEDOCS_OUTPUT/html/_static +# - tree + +# Build documentation in the "Documentation/" directory with Sphinx +sphinx: + configuration: Documentation/conf.py + +submodules: + recursive: true + +# Optionally build your docs in additional formats such as PDF and ePub +# formats: +# - pdf +# - epub + +# Optional but recommended, declare the Python requirements required +# to build your documentation +# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html +python: + install: + - requirements: Documentation/requirements.txt diff --git a/Documentation/requirements.txt b/Documentation/requirements.txt new file mode 100644 index 0000000..3ef203c --- /dev/null +++ b/Documentation/requirements.txt @@ -0,0 +1,2 @@ +sphinx +furo From f015d6a4e74c0949e9e7b46c91681df7d71b4b54 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Thu, 20 Aug 2026 11:04:47 -0700 Subject: [PATCH 20/30] Trying to get the doxygen processing running --- .readthedocs.yaml | 14 +++++++------- Documentation/run_doxygen.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 7 deletions(-) create mode 100644 Documentation/run_doxygen.py diff --git a/.readthedocs.yaml b/.readthedocs.yaml index fc246c7..36726bb 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -18,13 +18,13 @@ build: - doxygen - graphviz - tree -# jobs: -# pre_build: -# - git submodule update --init --recursive -# - python ./documentation/source/run_doxygen.py -# - mkdir -p $READTHEDOCS_OUTPUT/html -# - mv ./user_doxygen_out/html $READTHEDOCS_OUTPUT/html/_static -# - tree + jobs: + pre_build: + - git submodule update --init --recursive + - python ./Documentation/run_doxygen.py + - mkdir -p $READTHEDOCS_OUTPUT/html + - mv ./user_doxygen_out/html $READTHEDOCS_OUTPUT/html/_static + - tree # Build documentation in the "Documentation/" directory with Sphinx sphinx: diff --git a/Documentation/run_doxygen.py b/Documentation/run_doxygen.py new file mode 100644 index 0000000..cdccecc --- /dev/null +++ b/Documentation/run_doxygen.py @@ -0,0 +1,28 @@ +# This script sets environment variables that are used by Doxygen + +import os +from subprocess import call, check_output + +# version +this_version = 'v?.?.?' +try: + this_version = check_output(['git', 'describe', '--abbrev=0', '--tags']).decode('utf-8').strip() +except: + pass + +# environment variables used by Doxygen +os.environ['PROJECT_NAME'] = 'Monarch' +os.environ['PROJECT_NUMBER'] = this_version +os.environ['PROJECT_BRIEF_DESC'] = 'Library for reading and writing egg files' +# located in your documentation directory, or give the relative path from the documentation directory +#os.environ['PROJECT_LOGO'] = './documentation/images/Logo_55x55.png' + +# directories in which doxygen should look for source files; if you have a `doxfiles` directory in your documentation, that should go here; string with space-separated directories +os.environ['DOXYGEN_INPUT'] = './Documentation/DoxFiles ./Monarch2 ./Monarch3' +# directories within DOXYGEN_INPUT that you want to exclude from doxygen (e.g. if there's a submodule included that you don't want to index); string with space-separated directories +os.environ['DOXYGEN_EXCLUDE'] = '' +# directories outside of DOXYGEN_INPUT that you want the C preprocessor to look in for macro definitions (e.g. if there's a submodule not included that has relevant macros); string with space-separated directories +os.environ['PREPROC_INCLUDE_PATH'] = './Scarab/library/utility ./Scarab/library/logger' + +# Doxygen +call(['doxygen', './Scarab/documentation/Doxyfile']) From 6a25378afb6efeeed373b8e9eefabbc8a589149a Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Thu, 20 Aug 2026 11:59:55 -0700 Subject: [PATCH 21/30] Remove running doxygen from conf.py --- Documentation/conf.py | 23 ++--------------------- 1 file changed, 2 insertions(+), 21 deletions(-) diff --git a/Documentation/conf.py b/Documentation/conf.py index b0a3fec..e09a46d 100644 --- a/Documentation/conf.py +++ b/Documentation/conf.py @@ -29,29 +29,10 @@ # version this_version = 'v?.?.?' try: - this_version = check_output(['git', 'describe', '--abbrev=0', '--tags']) + this_version = check_output(['git', 'describe', '--abbrev=0', '--tags']).decode('utf-8').strip() except: pass -# environment variables used by Doxygen -os.environ['PROJECT_NAME'] = 'Monarch' -os.environ['PROJECT_NUMBER'] = this_version -os.environ['PROJECT_BRIEF_DESC'] = 'Project 8 Data File Format Library' -# located in your documentation directory, or give the relative path from the documentation directory -os.environ['PROJECT_LOGO'] = '' - -# directories in which doxygen should look for source files; if you have a `doxfiles` directory in your documentation, that should go here; string with space-separated directories -os.environ['DOXYGEN_INPUT'] = 'DoxFiles ../Monarch3 ../Monarch2' -# directories within DOXYGEN_INPUT that you want to exclude from doxygen (e.g. if there's a submodule included that you don't want to index); string with space-separated directories -os.environ['DOXYGEN_EXCLUDE'] = '' -# directories outside of DOXYGEN_INPUT that you want the C preprocessor to look in for macro definitions (e.g. if there's a submodule not included that has relevant macros); string with space-separated directories -os.environ['PREPROC_INCLUDE_PATH'] = '../Scarab/library/utility ../Scarab/library/logger' - -# Doxygen -call(['doxygen', '../Scarab/documentation/cpp/Doxyfile']) -call(['mv', './user_doxygen_out/html', './_static']) - - on_rtd = os.environ.get('READTHEDOCS', None) == 'True' if on_rtd: html_theme = 'default' @@ -87,7 +68,7 @@ # General information about the project. project = u'Monarch' -copyright = u'2018, Monarch Authors' +copyright = u'2026, Monarch Authors' author = u'Project 8 Collaboration' # The version info for the project you're documenting, acts as replacement for From 374f41f420bfe430828036dbe80029873ac6012d Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Thu, 20 Aug 2026 12:40:45 -0700 Subject: [PATCH 22/30] Updated C++ usage documentation --- Documentation/UsageMonarch3.rst | 71 ------- Documentation/UsageMonarch3Cpp.rst | 298 +++++++++++++++++++++++++++++ Documentation/index.rst | 2 +- 3 files changed, 299 insertions(+), 72 deletions(-) delete mode 100644 Documentation/UsageMonarch3.rst create mode 100644 Documentation/UsageMonarch3Cpp.rst diff --git a/Documentation/UsageMonarch3.rst b/Documentation/UsageMonarch3.rst deleted file mode 100644 index 7e5a969..0000000 --- a/Documentation/UsageMonarch3.rst +++ /dev/null @@ -1,71 +0,0 @@ -How to use Monarch3 -=================== - -Thread safety: Reading and writing records (via M3Stream::ReadRecord() and M3Stream::WriteRecord(), respectively) are thread-safe -except that the HDF5 C library (on which the C++ library is built) is inherently non-thread-safe. Though multi-threaded writing may -work even most of the time, it is inherently unstable. -All other operations in Monarch (besides writing and reading records) are are explicitly not thread-safe. - -Reading Egg3 Files ------------------- - -1. Open the file: ``Monarch3::OpenForReading( [filename] )`` -2. Access the header information: ``Monarch3::ReadHeader()`` -3. Get the pointer to the header, and use as needed: ``Monarch3::GetHeader()`` -4. Get the pointer(s) to the stream(s) in the file: ``Monarch3::GetStream( [stream number] )`` -5. Setup to access the data in a stream. You can acces either the record for the entire stream with ``Monarch3::GetStreamRecord()``, - or for individual channels with ``Monarch3::GetChannelRecord( [record number] )``. - If you have only one channel in the stream, the distinction between those is irrelevant. - The record objects have a function ``M3Record::GetData()`` to get the data array. - There are two ways in which you can interact with the data array: - - * If you want to access the data as an array of bytes (e.g. because either your data is of type ``uint8_t``, or you want to use ``memcpy``), you can use the pointer returned by ``M3Record::GetData()``; - * If you want to access the data as an array of other integer or floating-point data types, you can pass the data pointer from ``M3Record::GetData()`` to an ``M3DataReader`` object, along with the data type size and data format flag. The type of the values that are returned is specified as a template argument for ``M3DataReader``; it doesn't have to match the data type in the data array exactly, but it should have at least as many bytes as the data elements, and if the data elements are integer, it should be an integer, and if the data elements are floating-point, it should be floating-point. - * If you want to access the data as an array of complex floating-point data types, you can pass the data pointer from ``M3Record::GetData()`` to an ``M3ComplexDataReader`` object, along with the data type size and data format flag (you can also specify the element size, but for complex data it should be the default, 2). The type of the values that are returned is specified as a template argument for ``M3ComplexDataReader``; it should either be f4_complex or f8_complex, or the equivalent. - -6. When moving from record to record in the file, the memory used for the data stays the same, but it gets updated with new values. - To move to a new record use the ``M3Stream::ReadRecord( [offset] )`` function. The offset parameter allows you to move forward and - backward within the file. If the last record read was ``[J]`` (``= -1`` for a just-opened file), ReadRecord will access the ``[J+1+offset]`` record. - This means that the offset parameter has the following meanings: - - * if ``offset == 0`` (default), the next record will be accessed; - * if ``offset == -1``, the current record will be reread; - * ``offset < -1`` will go backwards in the file; - * ``offset > 0`` will skip forward in the file. - - The outcomes from the call are: - - * returns true if the move was successful; - * returns false if the move was unsuccessful because it goes past the end (or beginning) of the file; - * throws an M3Exception if there was an error. - -7. When you're finished reading, use ``Monarch3::FinishReading()`` to close the file. - - -Writing Egg3 Files ------------------- - -1. Open the file: ``Monarch3::OpenForWriting( [filename] )`` -2. Get the pointer to the header, and use as needed: ``Monarch3::GetHeader()`` -3. Fill in the header information and setup the streams. For the latter, use the AddStream functions to add streams with one or multiple channels. -4. Write the header information: ``Monarch3::WriteHeader()`` -5. Get the pointer(s) to the stream(s) in the file: ``Monarch3::GetStream( [stream number] )`` -6. Setup to access the data in a stream. You can acces either the record for the entire stream with ``Monarch3::GetStreamRecord()``, - or for individual channels with ``Monarch3::GetChannelRecord( [record number] )``. - If you have only one channel in the stream, the distinction between those is irrelevant. - The record objects have a function ``M3Record::GetData()`` to get the data array. - There are two ways in which you can interact with the data array: - - * If you want to access the data as an array of bytes (e.g. because either your data is of type ``uint8_t``, or you want to use ``memcpy``), you can use the pointer returned by ``M3Record::GetData()``; - * If you want to access the data as an array of other integer or floating-point data types, you can pass the data pointer from ``M3Record::GetData()`` to an ``M3DataWriter`` object, along with the data type size and data format flag. The type of the values that are passed to the writer is specified as a template argument for ``M3DataWriter``; it doesn't have to match the data type in the data array exactly, but it should be no larger than the data elements, and if the data elements are integer, it should be an integer, and if the data elements are floating-point, it should be floating-point. - * If you want to access the data as an array of complex floating-point data types, you can pass the data pointer from ``M3Record::GetData()`` to an ``M3ComplexDataWriter`` object, along with the data type size and data format flag (you can also specify the element size, but for complex data it should be the default, 2). The type of the values that are returned is specified as a template argument for ``M3ComplexDataWriter``; it should either be ``f4_complex`` or ``f8_complex``, or the equivalent. - -7. For each record, copy the data to the stream data memory using the access method you chose above, and then write to disk with ``M3Stream::WriteRecord( [is new acquisition?] )``. - When a record is from a different acquisition than the previous record, the flag passed to ``WriteRecord`` should be ``true``; otherwise it should be ``false``. - The outcomes from the call are: - - * returns true if the write was successful; - * throws an M3Exception if there was an error; - * (should never return false). - -8. When you're finished writing, use ``Monarch3::FinishWriting()`` to close the file. diff --git a/Documentation/UsageMonarch3Cpp.rst b/Documentation/UsageMonarch3Cpp.rst new file mode 100644 index 0000000..9946de3 --- /dev/null +++ b/Documentation/UsageMonarch3Cpp.rst @@ -0,0 +1,298 @@ +How to use Monarch3 with C++ +============================ + +Thread safety: Reading and writing records (via ``M3Stream::ReadRecord()`` and ``M3Stream::WriteRecord()``, respectively) are thread-safe +except that the HDF5 C library (on which the C++ library is built) is inherently non-thread-safe. Though multi-threaded writing may +work even most of the time, it is inherently unstable. +All other operations in Monarch (besides writing and reading records) are explicitly not thread-safe. + + +Constants +--------- + +The following constants are defined in the ``monarch3`` namespace (``M3Constants.hh``): + +**Data format** + +* ``sDigitizedUS`` -- unsigned integer samples +* ``sDigitizedS`` -- signed integer samples +* ``sAnalog`` -- floating-point samples + +**Bit alignment** + +* ``sBitsAlignedLeft`` -- significant bits are aligned to the MSB of the sample word +* ``sBitsAlignedRight`` -- significant bits are aligned to the LSB of the sample word + +**Channel format (multi-channel streams)** + +* ``sInterleaved`` -- channel samples are interleaved within the stream record +* ``sSeparate`` -- each channel has its own contiguous block within the stream record + + +Reading Egg3 Files +------------------ + +1. Open the file: ``Monarch3::OpenForReading( [filename] )`` + +2. Read the header: ``Monarch3::ReadHeader()`` + +3. Get the pointer to the header and inspect it: ``Monarch3::GetHeader()`` + + Key fields on ``M3Header``: + + * ``GetEggVersion()`` / ``SetEggVersion()`` + * ``GetFilename()`` + * ``GetRunDuration()`` -- run duration in milliseconds + * ``GetTimestamp()`` + * ``GetDescription()`` + + Stream and channel metadata are available through header vectors: + + .. code-block:: cpp + + const M3Header* hdr = monarch->GetHeader(); + + // Stream-level metadata + const M3StreamHeader& streamHdr = hdr->GetStreamHeaders()[0]; + streamHdr.GetNChannels(); + streamHdr.GetAcquisitionRate(); + streamHdr.GetRecordSize(); + streamHdr.GetDataTypeSize(); + streamHdr.GetDataFormat(); + + // Channel-level metadata + const M3ChannelHeader& chanHdr = hdr->GetChannelHeaders()[0]; + chanHdr.GetVoltageOffset(); + chanHdr.GetVoltageRange(); + +4. Get the pointer(s) to the stream(s) in the file: ``Monarch3::GetStream( [stream number] )`` + + The total number of records in the stream is available via ``M3Stream::GetNRecordsInFile()``. + +5. Setup to access the data in a stream. You can access either the record for the entire stream with ``M3Stream::GetStreamRecord()``, + or for individual channels with ``M3Stream::GetChannelRecord( [channel number] )``. + If you have only one channel in the stream, the distinction between those is irrelevant. + The record objects have a function ``M3Record::GetData()`` to get the raw byte data array. + There are three ways in which you can interact with the data array: + + * If you want to access the data as an array of bytes (e.g. because either your data is of type ``uint8_t``, or you want to use ``memcpy``), you can use the pointer returned by ``M3Record::GetData()``; + * If you want to access the data as an array of other integer or floating-point data types, you can pass the data pointer from ``M3Record::GetData()`` to an ``M3DataReader`` object, along with the data type size and data format flag. The type of the values that are returned is specified as a template argument for ``M3DataReader``; it doesn't have to match the data type in the data array exactly, but it should have at least as many bytes as the data elements, and if the data elements are integer, it should be an integer, and if the data elements are floating-point, it should be floating-point. + * If you want to access the data as an array of complex floating-point data types, you can pass the data pointer from ``M3Record::GetData()`` to an ``M3ComplexDataReader`` object, along with the data type size and data format flag (you can also specify the element size, but for complex data it should be the default, 2). The type of the values that are returned is specified as a template argument for ``M3ComplexDataReader``; it should either be ``f4_complex`` or ``f8_complex``, or the equivalent. + +6. When moving from record to record in the file, the memory used for the data stays the same, but it gets updated with new values. + To move to a new record use the ``M3Stream::ReadRecord( [offset] )`` function. The offset parameter allows you to move forward and + backward within the file. If the last record read was ``[J]`` (``= -1`` for a just-opened file), ``ReadRecord`` will access the ``[J+1+offset]`` record. + This means that the offset parameter has the following meanings: + + * if ``offset == 0`` (default), the next record will be accessed; + * if ``offset == -1``, the current record will be reread; + * ``offset < -1`` will go backwards in the file; + * ``offset > 0`` will skip forward in the file. + + The outcomes from the call are: + + * returns ``true`` if the move was successful; + * returns ``false`` if the move was unsuccessful because it goes past the end (or beginning) of the file; + * throws an ``M3Exception`` if there was an error. + + After each successful ``ReadRecord``, the current acquisition and record position are available via: + + * ``M3Stream::GetAcquisitionId()`` -- index of the current acquisition + * ``M3Stream::GetRecordCountInAcq()`` -- record index within the current acquisition + + A typical read loop: + + .. code-block:: cpp + + M3Stream* stream = monarch->GetStream( 0 ); + const M3Record* record = stream->GetChannelRecord( 0 ); + + // Data type from the stream header + unsigned dataTypeSize = stream->GetDataTypeSize(); + uint32_t dataFormat = hdr->GetStreamHeaders()[0].GetDataFormat(); + + M3DataReader< uint16_t > reader( record->GetData(), dataTypeSize, dataFormat ); + + while( stream->ReadRecord() ) + { + unsigned recSize = stream->GetChannelRecordSize(); + for( unsigned i = 0; i < recSize; ++i ) + { + uint16_t sample = reader.at( i ); + // process sample ... + } + } + +7. When you're finished reading, use ``Monarch3::FinishReading()`` to close the file. + + +Writing Egg3 Files +------------------ + +1. Open the file: ``Monarch3::OpenForWriting( [filename] )`` + +2. Get the pointer to the header and configure it: ``Monarch3::GetHeader()`` + + .. code-block:: cpp + + M3Header* hdr = monarch->GetHeader(); + hdr->SetFilename( filename ); + hdr->SetRunDuration( 1000 ); // milliseconds + hdr->SetTimestamp( "2024-01-01T00:00:00" ); + hdr->SetDescription( "My data" ); + +3. Add streams using the ``AddStream`` functions. Both overloads return the stream number, + which is used to address the stream after the header is written. + + For a single-channel stream:: + + unsigned streamNum = hdr->AddStream( + source, // std::string: digitizer label + acqRate, // uint32_t: acquisition rate (MHz) + recSize, // uint32_t: samples per record + sampleSize, // uint32_t: elements per sample (1 for real, 2 for complex) + dataTypeSize, // uint32_t: bytes per element + dataFormat, // uint32_t: sDigitizedUS, sDigitizedS, or sAnalog + bitDepth, // uint32_t: number of significant bits per sample + bitAlignment // uint32_t: sBitsAlignedLeft or sBitsAlignedRight + ); + + For a multi-channel stream:: + + unsigned streamNum = hdr->AddStream( + source, // std::string: digitizer label + nChannels, // uint32_t: number of channels + channelFormat, // uint32_t: sInterleaved or sSeparate + acqRate, // uint32_t: acquisition rate (MHz) + recSize, // uint32_t: samples per record per channel + sampleSize, // uint32_t: elements per sample + dataTypeSize, // uint32_t: bytes per element + dataFormat, // uint32_t: sDigitizedUS, sDigitizedS, or sAnalog + bitDepth, // uint32_t: number of significant bits per sample + bitAlignment // uint32_t: sBitsAlignedLeft or sBitsAlignedRight + ); + +4. Write the header information (this creates the HDF5 structure and allocates stream objects): ``Monarch3::WriteHeader()`` + +5. Get the pointer(s) to the stream(s) in the file: ``Monarch3::GetStream( [stream number] )`` + +6. Setup to access the data in a stream. You can access either the record for the entire stream with ``M3Stream::GetStreamRecord()``, + or for individual channels with ``M3Stream::GetChannelRecord( [channel number] )``. + If you have only one channel in the stream, the distinction between those is irrelevant. + The record objects have a function ``M3Record::GetData()`` to get the raw byte data array. + There are three ways in which you can interact with the data array: + + * If you want to access the data as an array of bytes (e.g. because either your data is of type ``uint8_t``, or you want to use ``memcpy``), you can use the pointer returned by ``M3Record::GetData()``; + * If you want to access the data as an array of other integer or floating-point data types, you can pass the data pointer from ``M3Record::GetData()`` to an ``M3DataWriter`` object, along with the data type size and data format flag. The type of the values that are passed to the writer is specified as a template argument for ``M3DataWriter``; it doesn't have to match the data type in the data array exactly, but it should be no larger than the data elements, and if the data elements are integer, it should be an integer, and if the data elements are floating-point, it should be floating-point. + * If you want to access the data as an array of complex floating-point data types, you can pass the data pointer from ``M3Record::GetData()`` to an ``M3ComplexDataWriter`` object, along with the data type size and data format flag (you can also specify the element size, but for complex data it should be the default, 2). The type of the values that are returned is specified as a template argument for ``M3ComplexDataWriter``; it should either be ``f4_complex`` or ``f8_complex``, or the equivalent. + +7. For each record, copy the data to the stream data memory using the access method you chose above, and then write to disk with ``M3Stream::WriteRecord( [is new acquisition?] )``. + When a record is from a different acquisition than the previous record, the flag passed to ``WriteRecord`` should be ``true``; otherwise it should be ``false``. + The outcomes from the call are: + + * returns ``true`` if the write was successful; + * throws an ``M3Exception`` if there was an error; + * (should never return ``false``). + +8. When you're finished writing, use ``Monarch3::FinishWriting()`` to close the file. + + +Complete Examples +----------------- + +**Writing a file** + +.. code-block:: cpp + + #include "M3Monarch.hh" + #include "M3DataInterface.hh" + #include + + using namespace monarch3; + + int main() + { + std::string filename = "output.egg"; + + Monarch3* monarch = Monarch3::OpenForWriting( filename ); + + M3Header* hdr = monarch->GetHeader(); + hdr->SetFilename( filename ); + hdr->SetRunDuration( 1000 ); + hdr->SetTimestamp( "2024-01-01T00:00:00" ); + hdr->SetDescription( "Example egg file" ); + + unsigned streamNum = hdr->AddStream( + "my-digitizer", + 200, // acquisition rate (MHz) + 1024, // samples per record + 1, // elements per sample + 1, // bytes per element (uint8) + sDigitizedUS, + 8, // bit depth + sBitsAlignedLeft + ); + + monarch->WriteHeader(); + + M3Stream* stream = monarch->GetStream( streamNum ); + M3Record* record = stream->GetChannelRecord( 0 ); + unsigned recSize = stream->GetChannelRecordSize(); + + M3DataWriter< uint8_t > writer( record->GetData(), 1, sDigitizedUS ); + + // Acquisition 0, first record + for( unsigned i = 0; i < recSize; ++i ) writer.set_at( 42, i ); + stream->WriteRecord( true ); // true = new acquisition + + // Acquisition 0, second record + for( unsigned i = 0; i < recSize; ++i ) writer.set_at( 100, i ); + stream->WriteRecord( false ); // false = continue acquisition + + monarch->FinishWriting(); + delete monarch; + return 0; + } + + +**Reading a file** + +.. code-block:: cpp + + #include "M3Monarch.hh" + #include "M3DataInterface.hh" + #include + + using namespace monarch3; + + int main() + { + Monarch3* monarch = Monarch3::OpenForReading( "output.egg" ); + monarch->ReadHeader(); + + const M3Header* hdr = monarch->GetHeader(); + std::cout << "Egg version: " << hdr->GetEggVersion() << "\n"; + std::cout << "Timestamp: " << hdr->GetTimestamp() << "\n"; + + const M3StreamHeader& streamHdr = hdr->GetStreamHeaders()[0]; + unsigned dataTypeSize = streamHdr.GetDataTypeSize(); + uint32_t dataFormat = streamHdr.GetDataFormat(); + + M3Stream* stream = monarch->GetStream( 0 ); + std::cout << "Stream 0: " << stream->GetNRecordsInFile() << " record(s)\n"; + + const M3Record* record = stream->GetChannelRecord( 0 ); + M3DataReader< uint8_t > reader( record->GetData(), dataTypeSize, dataFormat ); + + while( stream->ReadRecord() ) + { + unsigned recSize = stream->GetChannelRecordSize(); + std::cout << " acq=" << stream->GetAcquisitionId() + << " rec=" << stream->GetRecordCountInAcq() + << " data[0]=" << (int)reader.at( 0 ) << "\n"; + } + + monarch->FinishReading(); + delete monarch; + return 0; + } diff --git a/Documentation/index.rst b/Documentation/index.rst index 83a3241..687f87b 100644 --- a/Documentation/index.rst +++ b/Documentation/index.rst @@ -7,7 +7,7 @@ Contents: :maxdepth: 2 Monarch_versions - UsageMonarch3 + UsageMonarch3Cpp UsageMonarch3Python TestingMonarch3 From 574735f6de7e1ddaeada172249e708c91aeb1752 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Thu, 20 Aug 2026 12:56:37 -0700 Subject: [PATCH 23/30] Add Python API documentation generation to the RTD site --- Documentation/PythonAPI.rst | 15 + Documentation/UsageMonarch3Cpp.rst | 4 +- Documentation/UsageMonarch3Python.rst | 4 +- Documentation/conf.py | 18 +- Documentation/index.rst | 8 +- Documentation/stubs/monarch3.py | 664 ++++++++++++++++++++++++++ 6 files changed, 706 insertions(+), 7 deletions(-) create mode 100644 Documentation/PythonAPI.rst create mode 100644 Documentation/stubs/monarch3.py diff --git a/Documentation/PythonAPI.rst b/Documentation/PythonAPI.rst new file mode 100644 index 0000000..c8f516f --- /dev/null +++ b/Documentation/PythonAPI.rst @@ -0,0 +1,15 @@ +Python API Reference (monarch3) +================================ + +The ``monarch3`` Python module is a pybind11 binding of the Monarch3 C++ +library. It exposes the same read/write workflow as the C++ API using +Pythonic conventions: method and property names use ``snake_case``, raw +data buffers are returned as :class:`numpy.ndarray` views, and files can be +managed with Python's ``with`` statement. + +See :doc:`UsageMonarch3Python` for workflow examples and usage guidance. + +.. automodule:: monarch3 + :members: + :undoc-members: + :show-inheritance: diff --git a/Documentation/UsageMonarch3Cpp.rst b/Documentation/UsageMonarch3Cpp.rst index 9946de3..86930b4 100644 --- a/Documentation/UsageMonarch3Cpp.rst +++ b/Documentation/UsageMonarch3Cpp.rst @@ -1,5 +1,5 @@ -How to use Monarch3 with C++ -============================ +How to use Monarch3 in C++ +========================== Thread safety: Reading and writing records (via ``M3Stream::ReadRecord()`` and ``M3Stream::WriteRecord()``, respectively) are thread-safe except that the HDF5 C library (on which the C++ library is built) is inherently non-thread-safe. Though multi-threaded writing may diff --git a/Documentation/UsageMonarch3Python.rst b/Documentation/UsageMonarch3Python.rst index 0deaa94..43991f4 100644 --- a/Documentation/UsageMonarch3Python.rst +++ b/Documentation/UsageMonarch3Python.rst @@ -1,5 +1,5 @@ -How to use Monarch3 from Python -================================ +How to use Monarch3 in Python +============================= The ``monarch3`` Python module is a pybind11 binding of the Monarch3 C++ library. It exposes the same read/write workflow as the C++ API but with Pythonic conventions: diff --git a/Documentation/conf.py b/Documentation/conf.py index e09a46d..e5018b2 100644 --- a/Documentation/conf.py +++ b/Documentation/conf.py @@ -17,8 +17,9 @@ # * the project, copyright, and author variables # * the arguments used to assign variables htmlhelp_basename, latex_documents, man_pages, and texinfo_documents -#import sys +import sys import os +import importlib.util from subprocess import call, check_output # If extensions (or modules to document with autodoc) are in another directory, @@ -26,6 +27,11 @@ # documentation root, use os.path.abspath to make it absolute, like shown here. #sys.path.insert(0, os.path.abspath('.')) +# Use the pure-Python stub when the compiled monarch3 extension is not available +# (e.g. on ReadTheDocs, which does not compile C++). +if importlib.util.find_spec('monarch3') is None: + sys.path.insert(0, os.path.abspath('stubs')) + # version this_version = 'v?.?.?' try: @@ -50,7 +56,15 @@ # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = [] +extensions = [ + 'sphinx.ext.autodoc', +] + +autodoc_default_options = { + 'members': True, + 'undoc-members': False, + 'show-inheritance': True, +} # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] diff --git a/Documentation/index.rst b/Documentation/index.rst index 687f87b..187bb26 100644 --- a/Documentation/index.rst +++ b/Documentation/index.rst @@ -10,7 +10,13 @@ Contents: UsageMonarch3Cpp UsageMonarch3Python TestingMonarch3 + PythonAPI .. end of toc .. (you must not remove or modify the above comment line, it is required by the API Doc generation) -`Full Doxygen API Reference <_static/index.html>`_ + +API Reference +------------- + +* `C++ API Reference (Doxygen) <_static/index.html>`_ +* :doc:`Python API Reference (monarch3) ` diff --git a/Documentation/stubs/monarch3.py b/Documentation/stubs/monarch3.py new file mode 100644 index 0000000..267a180 --- /dev/null +++ b/Documentation/stubs/monarch3.py @@ -0,0 +1,664 @@ +""" +monarch3 -- Python interface to the Monarch3 egg-file library. + +The ``monarch3`` module is a pybind11 binding of the Monarch3 C++ library. +It exposes the same read/write workflow as the C++ API but with Pythonic +conventions: method and property names use ``snake_case``, raw data buffers +are returned as :class:`numpy.ndarray` views (``uint8`` by default), and +files can be managed with Python's ``with`` statement. + +.. note:: + This stub module is used when the compiled ``monarch3`` extension is not + available (e.g. during documentation builds on ReadTheDocs). It exists + solely to allow :mod:`sphinx.ext.autodoc` to introspect the API surface. + All method bodies are intentionally empty. + +Thread safety follows the same rules as the C++ library: calling +:meth:`M3Stream.read_record` and :meth:`M3Stream.write_record` is +thread-safe (the GIL is released during disk I/O), but all other operations +are not thread-safe. +""" + +# --------------------------------------------------------------------------- +# Constants +# --------------------------------------------------------------------------- + +#: Unsigned integer sample format. +sDigitizedUS: int = 0 + +#: Signed integer sample format. +sDigitizedS: int = 1 + +#: Floating-point (analog) sample format. +sAnalog: int = 2 + +#: Significant bits are aligned to the MSB of the sample word. +sBitsAlignedLeft: int = 0 + +#: Significant bits are aligned to the LSB of the sample word. +sBitsAlignedRight: int = 1 + +#: Channel samples are interleaved within the stream record. +sInterleaved: int = 0 + +#: Each channel occupies its own contiguous block within the stream record. +sSeparate: int = 1 + + +# --------------------------------------------------------------------------- +# Exception +# --------------------------------------------------------------------------- + +class Monarch3Exception(Exception): + """Raised by the monarch3 library on I/O errors and invalid operations.""" + pass + + +# --------------------------------------------------------------------------- +# Header classes +# --------------------------------------------------------------------------- + +class M3StreamHeader: + """Header information for a single data stream.""" + + @property + def number(self) -> int: + """Stream index within the file.""" + ... + + @property + def source(self) -> str: + """Source identifier string (e.g. digitizer name).""" + ... + + @property + def n_channels(self) -> int: + """Number of channels in this stream.""" + ... + + @property + def channels(self): + """List of channel indices belonging to this stream.""" + ... + + @property + def channel_format(self) -> int: + """Channel layout: :data:`sInterleaved` or :data:`sSeparate`.""" + ... + + @property + def acquisition_rate(self) -> int: + """Acquisition rate in MHz.""" + ... + + @property + def record_size(self) -> int: + """Number of samples per record (per channel).""" + ... + + @property + def sample_size(self) -> int: + """Number of elements per sample (1 for real, 2 for complex).""" + ... + + @property + def data_type_size(self) -> int: + """Size in bytes of each sample element.""" + ... + + @property + def data_format(self) -> int: + """Data format: :data:`sDigitizedUS`, :data:`sDigitizedS`, or :data:`sAnalog`.""" + ... + + @property + def bit_depth(self) -> int: + """Number of significant bits per sample.""" + ... + + @property + def bit_alignment(self) -> int: + """Bit alignment: :data:`sBitsAlignedLeft` or :data:`sBitsAlignedRight`.""" + ... + + @property + def n_acquisitions(self) -> int: + """Number of acquisitions recorded in this stream.""" + ... + + @property + def n_records(self) -> int: + """Total number of records across all acquisitions in this stream.""" + ... + + def __repr__(self) -> str: ... + + +class M3ChannelHeader: + """Header information for a single data channel.""" + + @property + def number(self) -> int: + """Channel index within the file.""" + ... + + @property + def source(self) -> str: + """Source identifier string (e.g. digitizer name).""" + ... + + @property + def acquisition_rate(self) -> int: + """Acquisition rate in MHz.""" + ... + + @property + def record_size(self) -> int: + """Number of samples per record.""" + ... + + @property + def sample_size(self) -> int: + """Number of elements per sample (1 for real, 2 for complex).""" + ... + + @property + def data_type_size(self) -> int: + """Size in bytes of each sample element.""" + ... + + @property + def data_format(self) -> int: + """Data format: :data:`sDigitizedUS`, :data:`sDigitizedS`, or :data:`sAnalog`.""" + ... + + @property + def bit_depth(self) -> int: + """Number of significant bits per sample.""" + ... + + @property + def bit_alignment(self) -> int: + """Bit alignment: :data:`sBitsAlignedLeft` or :data:`sBitsAlignedRight`.""" + ... + + @property + def voltage_offset(self) -> float: + """Voltage offset of the channel in volts.""" + ... + + @property + def voltage_range(self) -> float: + """Voltage range of the channel in volts.""" + ... + + @property + def dac_gain(self) -> float: + """DAC gain of the channel.""" + ... + + @property + def frequency_min(self) -> float: + """Minimum frequency in Hz.""" + ... + + @property + def frequency_range(self) -> float: + """Frequency range in Hz.""" + ... + + def __repr__(self) -> str: ... + + +class M3Header: + """Egg file header: run metadata and stream/channel configuration.""" + + # ------------------------------------------------------------------ + # Read-only properties + # ------------------------------------------------------------------ + + @property + def egg_version(self) -> str: + """Egg file format version string.""" + ... + + @property + def n_channels(self) -> int: + """Total number of channels across all streams.""" + ... + + @property + def n_streams(self) -> int: + """Total number of streams in the file.""" + ... + + @property + def stream_headers(self): + """List of :class:`M3StreamHeader` objects, one per stream.""" + ... + + @property + def channel_headers(self): + """List of :class:`M3ChannelHeader` objects, one per channel.""" + ... + + # ------------------------------------------------------------------ + # Read-write properties + # ------------------------------------------------------------------ + + @property + def filename(self) -> str: + """Output filename stored in the header.""" + ... + + @filename.setter + def filename(self, value: str) -> None: ... + + @property + def run_duration(self) -> int: + """Run duration in milliseconds.""" + ... + + @run_duration.setter + def run_duration(self, value: int) -> None: ... + + @property + def timestamp(self) -> str: + """Run timestamp string (ISO 8601 recommended).""" + ... + + @timestamp.setter + def timestamp(self, value: str) -> None: ... + + @property + def description(self) -> str: + """Free-text description of the run.""" + ... + + @description.setter + def description(self, value: str) -> None: ... + + # ------------------------------------------------------------------ + # Methods + # ------------------------------------------------------------------ + + def add_stream(self, + source: str, + acq_rate: int, + rec_size: int, + sample_size: int, + data_type_size: int, + data_format: int, + bit_depth: int, + bit_alignment: int, + chan_vec=None) -> int: + """Add a single-channel stream; returns the stream number. + + :param source: Digitizer or source identifier. + :param acq_rate: Acquisition rate in MHz. + :param rec_size: Number of samples per record. + :param sample_size: Number of elements per sample (1 for real data). + :param data_type_size: Bytes per element (e.g. 2 for ``uint16``). + :param data_format: :data:`sDigitizedUS`, :data:`sDigitizedS`, or :data:`sAnalog`. + :param bit_depth: Number of significant bits per sample. + :param bit_alignment: :data:`sBitsAlignedLeft` or :data:`sBitsAlignedRight`. + :param chan_vec: Optional list of channel indices to assign; ``None`` for automatic. + :returns: Stream number (index used with :meth:`Monarch3.get_stream`). + """ + ... + + def add_stream(self, # noqa: F811 (overload) + source: str, + n_channels: int, + channel_format: int, + acq_rate: int, + rec_size: int, + sample_size: int, + data_type_size: int, + data_format: int, + bit_depth: int, + bit_alignment: int, + chan_vec=None) -> int: + """Add a multi-channel stream; returns the stream number. + + :param source: Digitizer or source identifier. + :param n_channels: Number of channels in the stream. + :param channel_format: :data:`sInterleaved` or :data:`sSeparate`. + :param acq_rate: Acquisition rate in MHz. + :param rec_size: Number of samples per record per channel. + :param sample_size: Number of elements per sample. + :param data_type_size: Bytes per element. + :param data_format: :data:`sDigitizedUS`, :data:`sDigitizedS`, or :data:`sAnalog`. + :param bit_depth: Number of significant bits per sample. + :param bit_alignment: :data:`sBitsAlignedLeft` or :data:`sBitsAlignedRight`. + :param chan_vec: Optional list of channel indices to assign; ``None`` for automatic. + :returns: Stream number (index used with :meth:`Monarch3.get_stream`). + """ + ... + + def __repr__(self) -> str: ... + + +# --------------------------------------------------------------------------- +# Record class +# --------------------------------------------------------------------------- + +class M3Record: + """A single data record: record ID, timestamp, and raw data bytes.""" + + @property + def record_id(self) -> int: + """Record ID (``uint64``).""" + ... + + @property + def time(self) -> int: + """Timestamp in nanoseconds since the start of the run (``uint64``).""" + ... + + def get_data(self, nbytes: int): + """Return a writable :class:`numpy.ndarray` (``uint8``) view of the data buffer. + + This is a zero-copy view: the array does not own the data. + The caller must ensure the record and its parent stream remain alive. + + :param nbytes: Number of bytes to expose; should equal + :attr:`M3Stream.channel_record_n_bytes` (or + :attr:`M3Stream.stream_record_n_bytes` for the stream record). + :returns: ``numpy.ndarray`` of dtype ``uint8``. + Use :meth:`numpy.ndarray.view` to reinterpret as the actual element type. + """ + ... + + +# --------------------------------------------------------------------------- +# Stream class +# --------------------------------------------------------------------------- + +class M3Stream: + """Read/write access for a single data stream.""" + + # ------------------------------------------------------------------ + # State query properties + # ------------------------------------------------------------------ + + @property + def n_channels(self) -> int: + """Number of channels in this stream.""" + ... + + @property + def n_acquisitions(self) -> int: + """Number of acquisitions in this stream (valid after :meth:`read_record` or after writing).""" + ... + + @property + def acquisition_id(self) -> int: + """ID of the most recently accessed acquisition.""" + ... + + @property + def record_count_in_acq(self) -> int: + """Number of records read/written in the current acquisition.""" + ... + + @property + def n_records_in_file(self) -> int: + """Total number of records across all acquisitions in the file.""" + ... + + @property + def n_records_in_acquisition(self) -> int: + """Number of records in the current acquisition.""" + ... + + @property + def data_type_size(self) -> int: + """Size in bytes of each sample element.""" + ... + + @property + def sample_size(self) -> int: + """Number of elements per sample (1 for real, 2 for complex).""" + ... + + @property + def channel_record_size(self) -> int: + """Number of samples in a channel record.""" + ... + + @property + def channel_record_n_bytes(self) -> int: + """Size in bytes of a channel record data buffer.""" + ... + + @property + def stream_record_size(self) -> int: + """Number of samples in the full (potentially interleaved) stream record.""" + ... + + @property + def stream_record_n_bytes(self) -> int: + """Size in bytes of the full stream record data buffer.""" + ... + + @property + def is_interleaved(self) -> bool: + """``True`` if multi-channel data is stored interleaved in the file.""" + ... + + # ------------------------------------------------------------------ + # Record reading + # ------------------------------------------------------------------ + + def read_record(self, offset: int = 0, if_new_acq_start_at_first_rec: bool = True) -> bool: + """Read a record from the file. + + Assuming the last record read was ``[J]``, reads record ``[J+1+offset]``. + + :param offset: + * ``0`` (default): advance to the next record. + * ``-1``: re-read the current record. + * ``< -1``: step backward in the file. + * ``> 0``: skip forward in the file. + :param if_new_acq_start_at_first_rec: If ``True`` (default), when + stepping into a new acquisition the first record of that + acquisition is read regardless of the offset. Set to ``False`` + for backwards-compatible behaviour. + :returns: ``True`` on success; ``False`` when the requested position + is past the end (or before the beginning) of the file. + :raises Monarch3Exception: On I/O error. + """ + ... + + def get_stream_record(self) -> M3Record: + """Return the stream-level record object (all channels interleaved).""" + ... + + def get_channel_record(self, channel: int) -> M3Record: + """Return the record object for the given channel index. + + :param channel: Zero-based channel index. + """ + ... + + # ------------------------------------------------------------------ + # Convenience numpy data access + # ------------------------------------------------------------------ + + def get_stream_data(self): + """Return a writable :class:`numpy.ndarray` (``uint8``) view of the full stream record. + + Call after :meth:`read_record`. + Use :meth:`numpy.ndarray.view` to reinterpret the buffer as the + actual element type (e.g. ``arr.view(np.uint16)``). + + :returns: ``numpy.ndarray`` of dtype ``uint8`` with length + :attr:`stream_record_n_bytes`. + """ + ... + + def get_channel_data(self, channel: int): + """Return a writable :class:`numpy.ndarray` (``uint8``) view of a single channel's data. + + Call after :meth:`read_record` (reading) or before :meth:`write_record` (writing). + Use :meth:`numpy.ndarray.view` to reinterpret the buffer as the + actual element type (e.g. ``arr.view(np.uint16)``). + + :param channel: Zero-based channel index. + :returns: ``numpy.ndarray`` of dtype ``uint8`` with length + :attr:`channel_record_n_bytes`. + """ + ... + + # ------------------------------------------------------------------ + # Record writing + # ------------------------------------------------------------------ + + def write_record(self, is_new_acquisition: bool) -> bool: + """Write the current record contents to the file. + + Fill the data buffer via :meth:`get_channel_data` (or + :meth:`get_stream_data`) before calling this method. + + :param is_new_acquisition: ``True`` to start a new acquisition group; + ``False`` to continue the current acquisition. + :returns: ``True`` on success. + :raises Monarch3Exception: On I/O error. + """ + ... + + +# --------------------------------------------------------------------------- +# Top-level file handle +# --------------------------------------------------------------------------- + +class Monarch3: + """Top-level egg v3 file handle. + + Use :meth:`open_for_reading` or :meth:`open_for_writing` to obtain an + instance. Supports the context manager protocol (``with`` statement), + which automatically calls :meth:`finish_reading` or + :meth:`finish_writing` on exit. + + Example (reading):: + + with monarch3.Monarch3.open_for_reading("data.egg") as m: + m.read_header() + stream = m.get_stream(0) + while stream.read_record(): + arr = stream.get_channel_data(0).view(np.uint16) + ... + + Example (writing):: + + with monarch3.Monarch3.open_for_writing("out.egg") as m: + hdr = m.get_header() + hdr.run_duration = 1000 + hdr.add_stream("src", 200, 4096, 1, 2, monarch3.sDigitizedUS, 14, + monarch3.sBitsAlignedRight) + m.write_header() + stream = m.get_stream(0) + buf = stream.get_channel_data(0).view(np.uint16) + buf[:] = my_samples + stream.write_record(True) + """ + + @staticmethod + def open_for_reading(filename: str) -> "Monarch3": + """Open an existing egg file for reading. + + :param filename: Path to the egg file. + :returns: A :class:`Monarch3` instance in the ``eOpenToRead`` state. + :raises Monarch3Exception: If the file cannot be opened. + """ + ... + + @staticmethod + def open_for_writing(filename: str) -> "Monarch3": + """Create or overwrite an egg file for writing. + + :param filename: Path to the egg file. + :returns: A :class:`Monarch3` instance in the ``eOpenToWrite`` state. + :raises Monarch3Exception: If the file cannot be created. + """ + ... + + def __enter__(self) -> "Monarch3": + """Enter the context manager; returns ``self``.""" + ... + + def __exit__(self, exc_type, exc_val, exc_tb) -> bool: + """Exit the context manager. + + Calls :meth:`finish_reading` or :meth:`finish_writing` as appropriate. + Does not suppress exceptions (always returns ``False``). + """ + ... + + @property + def state(self) -> int: + """Current state of the file handle.""" + ... + + # ------------------------------------------------------------------ + # Reading interface + # ------------------------------------------------------------------ + + def read_header(self) -> None: + """Read header information from the file. + + Must be called after :meth:`open_for_reading` and before accessing + streams or header data. + + :raises Monarch3Exception: On I/O error. + """ + ... + + def get_header(self) -> M3Header: + """Return the file header. + + When called on a file opened for reading, the header is read-only + (modifying it has no effect on the file). When called on a file + opened for writing, the returned object is mutable and should be + configured before calling :meth:`write_header`. + + :returns: The :class:`M3Header` for this file. + """ + ... + + def get_stream(self, stream: int) -> M3Stream: + """Return the stream object for the given stream index. + + :param stream: Zero-based stream index. + :returns: The :class:`M3Stream` for the requested stream. + """ + ... + + def finish_reading(self) -> None: + """Close the file after reading. + + Not needed when using the ``with`` statement. + """ + ... + + # ------------------------------------------------------------------ + # Writing interface + # ------------------------------------------------------------------ + + def write_header(self) -> None: + """Write the header to the file and prepare streams for writing. + + Must be called after configuring the header (via :meth:`get_header`) + and adding all streams (via :meth:`M3Header.add_stream`). + + :raises Monarch3Exception: On I/O error. + """ + ... + + def finish_writing(self) -> None: + """Flush and close the file after writing. + + Not needed when using the ``with`` statement. + """ + ... From d1dde98e05e3615a01a80db4c633d7f931318c62 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Thu, 20 Aug 2026 13:12:23 -0700 Subject: [PATCH 24/30] Build the C++ library and bindings to get the full Python API in the documentation site --- .readthedocs.yaml | 8 ++++++++ Documentation/conf.py | 4 ++-- Documentation/index.rst | 1 - 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 36726bb..4be6fb7 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -15,12 +15,20 @@ build: # rust: "1.64" # golang: "1.19" apt_packages: + - cmake - doxygen - graphviz + - libboost-filesystem-dev + - libhdf5-dev + - pybind11-dev + - rapidjson-dev + - libyaml-cpp-dev - tree jobs: pre_build: - git submodule update --init --recursive + - cmake -S . -B _rtd_build -DMonarch_BUILD_PYTHON=ON -DPBUILDER_PY_INSTALL_IN_SITELIB=ON -DCMAKE_INSTALL_PREFIX=/usr/local + - cmake --build _rtd_build --target install -- -j$(nproc) - python ./Documentation/run_doxygen.py - mkdir -p $READTHEDOCS_OUTPUT/html - mv ./user_doxygen_out/html $READTHEDOCS_OUTPUT/html/_static diff --git a/Documentation/conf.py b/Documentation/conf.py index e5018b2..97cabaa 100644 --- a/Documentation/conf.py +++ b/Documentation/conf.py @@ -27,8 +27,8 @@ # documentation root, use os.path.abspath to make it absolute, like shown here. #sys.path.insert(0, os.path.abspath('.')) -# Use the pure-Python stub when the compiled monarch3 extension is not available -# (e.g. on ReadTheDocs, which does not compile C++). +# Fall back to the pure-Python stub when the compiled monarch3 extension is not +# available (e.g. local doc builds where the C++ library has not been compiled). if importlib.util.find_spec('monarch3') is None: sys.path.insert(0, os.path.abspath('stubs')) diff --git a/Documentation/index.rst b/Documentation/index.rst index 187bb26..d52ee33 100644 --- a/Documentation/index.rst +++ b/Documentation/index.rst @@ -10,7 +10,6 @@ Contents: UsageMonarch3Cpp UsageMonarch3Python TestingMonarch3 - PythonAPI .. end of toc .. (you must not remove or modify the above comment line, it is required by the API Doc generation) From ed607a3098822fa0c4fcdb084c820c90261b3abb Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Thu, 20 Aug 2026 13:14:27 -0700 Subject: [PATCH 25/30] Update OS to make sure software versions are sufficient and to be more future-proof --- .readthedocs.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 4be6fb7..04737b5 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -7,7 +7,7 @@ version: 2 # Set the OS, Python version and other tools you might need build: - os: ubuntu-22.04 + os: ubuntu-lts-latest tools: python: "3.12" # You can also specify other tool versions: From 830e6b01585aa9cf9ac3162f9adf223b50537049 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Thu, 20 Aug 2026 13:18:26 -0700 Subject: [PATCH 26/30] Don't use /usr/local as the install directory --- .readthedocs.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 04737b5..304c8f2 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -27,7 +27,7 @@ build: jobs: pre_build: - git submodule update --init --recursive - - cmake -S . -B _rtd_build -DMonarch_BUILD_PYTHON=ON -DPBUILDER_PY_INSTALL_IN_SITELIB=ON -DCMAKE_INSTALL_PREFIX=/usr/local + - cmake -S . -B _rtd_build -DMonarch_BUILD_PYTHON=ON -DPBUILDER_PY_INSTALL_IN_SITELIB=ON - cmake --build _rtd_build --target install -- -j$(nproc) - python ./Documentation/run_doxygen.py - mkdir -p $READTHEDOCS_OUTPUT/html From 62b468e6cfea5c925f0f75a32eb716da16d31472 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Thu, 20 Aug 2026 13:25:55 -0700 Subject: [PATCH 27/30] [no ci] Minor documentation fix --- Documentation/Monarch_versions.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Documentation/Monarch_versions.rst b/Documentation/Monarch_versions.rst index 6fda233..f889b05 100644 --- a/Documentation/Monarch_versions.rst +++ b/Documentation/Monarch_versions.rst @@ -1,5 +1,5 @@ -EggStandards -============ +Egg Standards +============= .. toctree:: :maxdepth: 3 From b673e69c52862c2e72f4b147043e65e3b06349a1 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Thu, 20 Aug 2026 13:29:45 -0700 Subject: [PATCH 28/30] [no ci] Updated the changelog --- changelog.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/changelog.md b/changelog.md index e42429f..fc5846c 100644 --- a/changelog.md +++ b/changelog.md @@ -27,6 +27,8 @@ Types of changes: Added, Changed, Deprecated, Removed, Fixed, Security - Documentation pages: `UsageMonarch3Python.rst` and `TestingMonarch3.rst` - GitHub Actions workflow (`.github/workflows/run_tests.yaml`) with separate jobs for Monarch3 (including Python validation tests) and Monarch2, plus a Release job +- Python API documentation is built on ReadTheDocs, including building the full C++ library + and Python bindings so that the full API is available. ### Fixed @@ -35,8 +37,10 @@ Types of changes: Added, Changed, Deprecated, Removed, Fixed, Security a `ReadRecord(-2)` call that would step before the start of the file - `M3ReadTest`: Test 3 (stream 2 skip) likewise uses `aIfNewAcqStartAtFirstRec=false` for the initial offset skip +- RTD documentation setup was updated to modern RTD standards ### Changed - HDF5 minimum version raised to 1.10.1; the v1.8 API compatibility workaround has been removed from `CMakeLists.txt` +- C++ use documentation was updated From a59c0de1dbb02cedbfefc29d4c77ddb9537abeb6 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Thu, 20 Aug 2026 13:35:56 -0700 Subject: [PATCH 29/30] [no ci] Updated the README --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index d18afdd..0ecd3ab 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,6 @@ Monarch ======= -[![Codacy Badge](https://api.codacy.com/project/badge/Grade/9495b2fba24c44c8884d78c80873e45d)](https://www.codacy.com/project/Project8/monarch/dashboard?utm_source=github.com&utm_medium=referral&utm_content=project8/monarch&utm_campaign=Badge_Grade_Dashboard) [![Documentation Status](https://readthedocs.org/projects/monarch/badge/?version=stable)](https://monarch.readthedocs.io/en/stable/?badge=stable) [![DOI](https://zenodo.org/badge/2208206.svg)](https://zenodo.org/badge/latestdoi/2208206) @@ -28,7 +27,7 @@ Requirements ------------ - CMake 3.1 or higher -- HDF5 1.8.12 or higher (Monarch3 only) +- HDF5 1.10.1 or higher (Monarch3 only) - Google Protocol Buffers (Monarch2 only) From ad8535ae84bcbb9da06c38b4b95ba7bf4ba394b8 Mon Sep 17 00:00:00 2001 From: Noah Oblath Date: Thu, 20 Aug 2026 13:42:17 -0700 Subject: [PATCH 30/30] [no ci] Updated version and changelog --- CMakeLists.txt | 2 +- changelog.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f280a59..4242b9c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,7 +6,7 @@ cmake_minimum_required (VERSION 3.12) ######### cmake_policy( SET CMP0048 NEW ) # version in project() -project( Monarch VERSION 3.8.7 ) +project( Monarch VERSION 3.9.0 ) list( APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/Scarab/cmake) include( PackageBuilder ) diff --git a/changelog.md b/changelog.md index fc5846c..7f62d51 100644 --- a/changelog.md +++ b/changelog.md @@ -7,7 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 Types of changes: Added, Changed, Deprecated, Removed, Fixed, Security -## [Unreleased] ([3.9.0] - 2026-08-19) +## [3.9.0] - 2026-08-20 ### Added