diff --git a/.github/workflows/run_tests.yaml b/.github/workflows/run_tests.yaml new file mode 100644 index 0000000..3f99136 --- /dev/null +++ b/.github/workflows/run_tests.yaml @@ -0,0 +1,189 @@ +name: Run Tests + +on: + push: + branches: [ 'main', 'develop' ] + tags: ['*'] + pull_request: + + workflow_dispatch: + +jobs: + + Monarch3Tests: + + runs-on: ${{ matrix.os }} + + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04, ubuntu-24.04-arm, ubuntu-22.04, ubuntu-22.04-arm, macos-26, macos-15-intel, macos-15] + + steps: + + - name: Checkout the repo + uses: actions/checkout@v7 + with: + submodules: recursive + + - name: Select CC version + # We require GCC 12 or higher if using GCC + # The default GCC for Ubuntu 22.04 is 11, but 12 is available + if: ${{ matrix.os == 'ubuntu-22.04' || matrix.os == 'ubuntu-22.04-arm' }} + run: | + echo "CC=gcc-12" >> $GITHUB_ENV + echo "CXX=g++-12" >> $GITHUB_ENV + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.x' + + - name: Install Python packages + run: pip install pytest numpy + + - name: Install dependencies -- Mac + # pybind11 is manually installed because we need to use v3.0.0 (as of 8/21/25) + # Fix the /usr/local directory structure + if: startsWith(matrix.os, 'macos') + run: | + brew install \ + hdf5 \ + boost \ + pybind11 \ + rapidjson \ + yaml-cpp + + - name: Install dependencies -- Linux + if: startsWith(matrix.os, 'ubuntu') + run: | + sudo apt-get update + DEBIAN_FRONTEND=noninteractive sudo apt-get install -y \ + libhdf5-dev \ + libboost-all-dev \ + rapidjson-dev \ + libyaml-cpp-dev + git clone https://github.com/pybind/pybind11.git + cd pybind11 + git checkout v3.1.0 + mkdir build + cd build + sudo cmake -DPYBIND11_TEST=FALSE .. + sudo make -j2 install + + - name: Configure + run: | + mkdir build + cd build + cmake .. \ + -DMonarch_BUILD_MONARCH3=TRUE \ + -DMonarch_ENABLE_TESTING=TRUE \ + -DMonarch_BUILD_PYTHON=TRUE + + - name: Build + run: | + cd build + make -j2 install + + - name: Run tests + run: | + cd build + ctest --output-on-failure + +# For debugging +# - name: Setup tmate session +# if: ${{ ! success() }} +# uses: mxschmitt/action-tmate@v3 + + + Monarch2Tests: + + runs-on: ${{ matrix.os }} + + strategy: + fail-fast: false + matrix: + os: [ubuntu-24.04, ubuntu-24.04-arm, ubuntu-22.04, ubuntu-22.04-arm, macos-26, macos-15-intel, macos-15] + + steps: + + - name: Checkout the repo + uses: actions/checkout@v7 + with: + submodules: recursive + + - name: Select CC version + if: ${{ matrix.os == 'ubuntu-22.04' || matrix.os == 'ubuntu-22.04-arm' }} + run: | + echo "CC=gcc-12" >> $GITHUB_ENV + echo "CXX=g++-12" >> $GITHUB_ENV + + - name: Install dependencies -- Mac + if: startsWith(matrix.os, 'macos') + run: | + brew install \ + protobuf \ + boost \ + rapidjson \ + yaml-cpp + + - name: Install dependencies -- Linux + if: startsWith(matrix.os, 'ubuntu') + run: | + sudo apt-get update + DEBIAN_FRONTEND=noninteractive sudo apt-get install -y \ + protobuf-compiler \ + libprotobuf-dev \ + libboost-all-dev \ + rapidjson-dev \ + libyaml-cpp-dev + + - name: Configure + run: | + mkdir build + cd build + cmake .. \ + -DMonarch_BUILD_MONARCH2=TRUE \ + -DMonarch_BUILD_MONARCH3=FALSE \ + -DMonarch_ENABLE_TESTING=TRUE \ + -DScarab_BUILD_PYTHON=FALSE + + - name: Build + run: | + cd build + make -j2 install + +# Tests have not been implemented +# - name: Run tests +# run: | +# cd build +# ctest --output-on-failure + +# For debugging +# - name: Setup tmate session +# if: ${{ ! success() }} +# uses: mxschmitt/action-tmate@v3 + + + Release: + + runs-on: ubuntu-24.04 + + if: ${{ github.event_name == 'push' && contains(github.ref, 'refs/tags/') }} + + needs: [Monarch3Tests, Monarch2Tests] + + steps: + + - name: Checkout the repo + uses: actions/checkout@v7 + with: + submodules: recursive + + - name: Release with a changelog + uses: rasmus-saks/release-a-changelog-action@v1.2.0 + with: + github-token: '${{ secrets.GITHUB_TOKEN }}' + path: 'changelog.md' + title-template: 'Monarch v{version} -- Release Notes' + tag-template: 'v{version}' diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 0000000..304c8f2 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,54 @@ +# .readthedocs.yaml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Set the OS, Python version and other tools you might need +build: + os: ubuntu-lts-latest + tools: + python: "3.12" + # You can also specify other tool versions: + # nodejs: "19" + # rust: "1.64" + # golang: "1.19" + apt_packages: + - cmake + - doxygen + - graphviz + - libboost-filesystem-dev + - libhdf5-dev + - pybind11-dev + - rapidjson-dev + - libyaml-cpp-dev + - tree + jobs: + pre_build: + - git submodule update --init --recursive + - cmake -S . -B _rtd_build -DMonarch_BUILD_PYTHON=ON -DPBUILDER_PY_INSTALL_IN_SITELIB=ON + - cmake --build _rtd_build --target install -- -j$(nproc) + - python ./Documentation/run_doxygen.py + - mkdir -p $READTHEDOCS_OUTPUT/html + - mv ./user_doxygen_out/html $READTHEDOCS_OUTPUT/html/_static + - tree + +# Build documentation in the "Documentation/" directory with Sphinx +sphinx: + configuration: Documentation/conf.py + +submodules: + recursive: true + +# Optionally build your docs in additional formats such as PDF and ePub +# formats: +# - pdf +# - epub + +# Optional but recommended, declare the Python requirements required +# to build your documentation +# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html +python: + install: + - requirements: Documentation/requirements.txt diff --git a/CMakeLists.txt b/CMakeLists.txt index 3352070..f280a59 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -18,6 +18,7 @@ add_definitions( -DEgg_VERSION=${Egg_VERSION} ) # Monarch library options set( Monarch_BUILD_MONARCH2 FALSE CACHE BOOL "Build Monarch2 library (requires Protobuf)" ) set( Monarch_BUILD_MONARCH3 TRUE CACHE BOOL "Build Monarch3 library (requires HDF5)" ) +set( Monarch_BUILD_PYTHON FALSE CACHE BOOL "Build Monarch3 Python bindings (requires pybind11)" ) ######################## @@ -36,82 +37,69 @@ if( Monarch_BUILD_MONARCH3 ) find_package( HDF5 REQUIRED COMPONENTS CXX ) include_directories( BEFORE ${PROJECT_SOURCE_DIR}/Monarch3 ) - # HDF5 doesn't currently define imported targets. Therefore we have to do it ourselves. + if( HDF5_VERSION VERSION_LESS "1.10.1" ) + message( FATAL_ERROR "HDF5 must be v1.10.1 or newer" ) + elseif( HDF5_VERSION VERSION_GREATER_EQUAL "2.0.0" ) + list( APPEND PUBLIC_EXT_LIBS_M3 hdf5::hdf5_cpp ) + else() # 1.10.1 ... <2.0.0 - # Library variables returned by the FindHDF5 module are paths to the actual libraries. - # We'll store all libraries, regardless of platform, in a single list. - set( ALL_HDF5_LIBRARIES ) - if( WIN32 ) + # HDF5 doesn't currently define imported targets. Therefore we have to do it ourselves. - # In testing on 3/24/16 I found that the .dlls were being found, not the .libs. I don't understand why, but this fixes the problem if it occurs. - # HDF5_CXX_LIBRARIES was blank in testing, so I commented out the first line. - #string( REPLACE ".dll" ".lib" HDF5_CXX_LIBRARIES ${HDF5_CXX_LIBRARIES} ) - string( REPLACE ".dll" ".lib" HDF5_C_LIBRARY ${HDF5_C_LIBRARY} ) - string( REPLACE ".dll" ".lib" HDF5_CXX_LIBRARY ${HDF5_CXX_LIBRARY} ) + # Library variables returned by the FindHDF5 module are paths to the actual libraries. + # We'll store all libraries, regardless of platform, in a single list. + set( ALL_HDF5_LIBRARIES ) + if( WIN32 ) - list( APPEND ALL_HDF5_LIBRARIES ${HDF5_C_LIBRARY} ${HDF5_CXX_LIBRARY} ) + # In testing on 3/24/16 I found that the .dlls were being found, not the .libs. I don't understand why, but this fixes the problem if it occurs. + # HDF5_CXX_LIBRARIES was blank in testing, so I commented out the first line. + #string( REPLACE ".dll" ".lib" HDF5_CXX_LIBRARIES ${HDF5_CXX_LIBRARIES} ) + string( REPLACE ".dll" ".lib" HDF5_C_LIBRARY ${HDF5_C_LIBRARY} ) + string( REPLACE ".dll" ".lib" HDF5_CXX_LIBRARY ${HDF5_CXX_LIBRARY} ) - message( STATUS "HDF5_C_LIBRARY: ${HDF5_C_LIBRARY}" ) - message( STATUS "HDF5_CXX_LIBRARY: ${HDF5_CXX_LIBRARY}" ) + list( APPEND ALL_HDF5_LIBRARIES ${HDF5_C_LIBRARY} ${HDF5_CXX_LIBRARY} ) - else() + message( STATUS "HDF5_C_LIBRARY: ${HDF5_C_LIBRARY}" ) + message( STATUS "HDF5_CXX_LIBRARY: ${HDF5_CXX_LIBRARY}" ) - list( APPEND ALL_HDF5_LIBRARIES ${HDF5_CXX_LIBRARIES} ) - message( STATUS "HDF5_CXX_LIBRARIES: ${HDF5_CXX_LIBRARIES}" ) - - endif() - - # Now we create an imported library for the C++ library, and assign all other libraries as dependencies - add_library( HDF5CXXLib SHARED IMPORTED ) - foreach( LIBRARY_PATH IN LISTS ALL_HDF5_LIBRARIES ) - get_filename_component( LIBRARY ${LIBRARY_PATH} NAME_WE ) - string(TOLOWER ${LIBRARY} LIBRARY_LOWER ) # avoid any potential case variation - if( LIBRARY_LOWER STREQUAL "libhdf5_cpp" ) - # this is the C++ library - set_target_properties( HDF5CXXLib PROPERTIES IMPORTED_LOCATION ${LIBRARY_PATH} ) - - # We want the relevant include directories to be propagated as part of the HDF5 library properties - # HDF5_INCLUDE_DIR is deprecated, but is placed here to ensure the directory is picked up if an older version of cmake is used - target_include_directories( HDF5CXXLib INTERFACE ${HDF5_INCLUDE_DIRS} ${HDF5_INCLUDE_DIR} ) else() - # this is a dependency library - add_library( ${LIBRARY} SHARED IMPORTED ) - set_target_properties( ${LIBRARY} PROPERTIES IMPORTED_LOCATION ${LIBRARY_PATH} ) - # We want the dependency libraries to be propagated as part of the HDF5 library properties - target_link_libraries( HDF5CXXLib INTERFACE ${LIBRARY} ) + list( APPEND ALL_HDF5_LIBRARIES ${HDF5_CXX_LIBRARIES} ) + message( STATUS "HDF5_CXX_LIBRARIES: ${HDF5_CXX_LIBRARIES}" ) + endif() - endforeach() - # Add the imported library to the set of external libraries - list( APPEND PUBLIC_EXT_LIBS_M3 HDF5CXXLib ) + # Now we create an imported library for the C++ library, and assign all other libraries as dependencies + add_library( HDF5CXXLib SHARED IMPORTED ) + foreach( LIBRARY_PATH IN LISTS ALL_HDF5_LIBRARIES ) + get_filename_component( LIBRARY ${LIBRARY_PATH} NAME_WE ) + string(TOLOWER ${LIBRARY} LIBRARY_LOWER ) # avoid any potential case variation + if( LIBRARY_LOWER STREQUAL "libhdf5_cpp" ) + # this is the C++ library + set_target_properties( HDF5CXXLib PROPERTIES IMPORTED_LOCATION ${LIBRARY_PATH} ) + + # We want the relevant include directories to be propagated as part of the HDF5 library properties + # HDF5_INCLUDE_DIR is deprecated, but is placed here to ensure the directory is picked up if an older version of cmake is used + target_include_directories( HDF5CXXLib INTERFACE ${HDF5_INCLUDE_DIRS} ${HDF5_INCLUDE_DIR} ) + else() + # this is a dependency library + add_library( ${LIBRARY} SHARED IMPORTED ) + set_target_properties( ${LIBRARY} PROPERTIES IMPORTED_LOCATION ${LIBRARY_PATH} ) + + # We want the dependency libraries to be propagated as part of the HDF5 library properties + target_link_libraries( HDF5CXXLib INTERFACE ${LIBRARY} ) + endif() + endforeach() + + # Add the imported library to the set of external libraries + list( APPEND PUBLIC_EXT_LIBS_M3 HDF5CXXLib ) - # The following is a work-around to provide compatibility with the HDF5 1.8 API and the HDF5 1.10 API - # It should be removed in favor of the 1.10 API once we're able to make 1.10 a requirement - # If you upgrade your version of HDF5, you'll need to clear your CMake cache before rebuilding - # First, test for the version of HDF5 in use - include( CheckCXXSourceCompiles ) - set( CMAKE_REQUIRED_QUIET FALSE ) - set( CMAKE_REQUIRED_LIBRARIES ${HDF5_CXX_LIBRARIES} ) - set( CMAKE_REQUIRED_INCLUDES ${HDF5_INCLUDE_DIRS} ) - check_cxx_source_compiles(" - #include - int main() - { - H5::CommonFG* tester = new H5::H5File(\"new_file.h5\", H5F_ACC_TRUNC); - tester->openGroup(\"my_group\"); - return 0; - } - " HAVE_OLD_HDF5 ) - - # Then set preprocessor macros accordingly - if( ${HAVE_OLD_HDF5} ) - message( STATUS "HDF5 is version 1.10.0 or older" ) - set( HDF5_DEFINITIONS -DHAS_ATTR_IFC=H5::H5Location -DHAS_GRP_IFC=H5::CommonFG) - else( ${HAVE_OLD_HDF5} ) - message( STATUS "HDF5 is version 1.10.1 or newer" ) - set( HDF5_DEFINITIONS -DHAS_ATTR_IFC=H5::H5Object -DHAS_GRP_IFC=H5::H5Object ) - endif( ${HAVE_OLD_HDF5} ) + endif() + + # The following macros were originally introduced to handle compatibility with the HDF5 1.8 API and the HDF5 1.10.1 API + # 1.10 is now the minimum version (as of 8/19/2026) so the workaround for v1.8 has been removed. + # But these definitions still need to be used until the source code is adjusted. + message( STATUS "HDF5 is version 1.10.1 or newer" ) + set( HDF5_DEFINITIONS -DHAS_ATTR_IFC=H5::H5Object -DHAS_GRP_IFC=H5::H5Object ) endif( Monarch_BUILD_MONARCH3 ) @@ -121,23 +109,47 @@ endif( Monarch_BUILD_MONARCH3 ) ########## if( Monarch_BUILD_MONARCH2 ) - find_package( Protobuf REQUIRED ) + # Use config mode so that protobuf's abseil dependency is propagated automatically. + # Modern protobuf (v22 / 4.x+ via brew) depends on abseil internally; the old + # FindProtobuf module mode does not surface that transitive dependency, causing + # undefined absl:: symbols at link time. + find_package( protobuf CONFIG QUIET ) + if( NOT protobuf_FOUND ) + # Fall back to module mode for older protobuf installations that only ship + # a FindProtobuf module and no config file. + find_package( Protobuf REQUIRED ) + endif() include_directories( BEFORE ${PROJECT_SOURCE_DIR}/Monarch2 ${PROJECT_BINARY_DIR}/Monarch2 ${PROJECT_BINARY_DIR} ) - list( APPEND PUBLIC_EXT_LIBS_M2 - protobuf::libprotobuf - protobuf::libprotobuf-lite - protobuf::libprotoc + list( APPEND PUBLIC_EXT_LIBS_M2 + protobuf::libprotobuf + protobuf::libprotobuf-lite ) endif( Monarch_BUILD_MONARCH2 ) +########## +# Python # +########## + +if( Monarch_BUILD_PYTHON ) + find_package( Python3 REQUIRED COMPONENTS Interpreter Development ) + set( PYBIND11_PYTHON_VERSION ${Python3_VERSION} CACHE STRING "" ) + find_package( pybind11 REQUIRED ) + include_directories( ${Python3_INCLUDE_DIRS} ) +endif( Monarch_BUILD_PYTHON ) + + ##################### # Prepare for Build # ##################### pbuilder_prepare_project() +if( Monarch_ENABLE_TESTING ) + enable_testing() +endif( Monarch_ENABLE_TESTING ) + ######## # Scarab diff --git a/Documentation/Monarch_versions.rst b/Documentation/Monarch_versions.rst index 6fda233..f889b05 100644 --- a/Documentation/Monarch_versions.rst +++ b/Documentation/Monarch_versions.rst @@ -1,5 +1,5 @@ -EggStandards -============ +Egg Standards +============= .. toctree:: :maxdepth: 3 diff --git a/Documentation/PythonAPI.rst b/Documentation/PythonAPI.rst new file mode 100644 index 0000000..c8f516f --- /dev/null +++ b/Documentation/PythonAPI.rst @@ -0,0 +1,15 @@ +Python API Reference (monarch3) +================================ + +The ``monarch3`` Python module is a pybind11 binding of the Monarch3 C++ +library. It exposes the same read/write workflow as the C++ API using +Pythonic conventions: method and property names use ``snake_case``, raw +data buffers are returned as :class:`numpy.ndarray` views, and files can be +managed with Python's ``with`` statement. + +See :doc:`UsageMonarch3Python` for workflow examples and usage guidance. + +.. automodule:: monarch3 + :members: + :undoc-members: + :show-inheritance: diff --git a/Documentation/TestingMonarch3.rst b/Documentation/TestingMonarch3.rst new file mode 100644 index 0000000..00bb7ab --- /dev/null +++ b/Documentation/TestingMonarch3.rst @@ -0,0 +1,137 @@ +Testing Monarch3 +================ + +Overview +-------- + +Monarch3 includes two categories of tests: + +**C++ test executables** (``M3WriteTest``, ``M3ReadTest``, ``M3WriteSpeedTest``, +``M3MultithreadingTest``) are built when ``Monarch_ENABLE_TESTING`` is ON. +They exercise the C++ library directly and do not require Python. + +**Python validation tests** live in ``Monarch3/Tests/Validation/`` and require both +``Monarch_ENABLE_TESTING`` and ``Monarch_BUILD_PYTHON`` to be ON. They use +`pytest `_ as the test runner and verify cross-language +compatibility between the C++ library and the Python binding. The validation tests +are registered with CTest so they run automatically alongside the C++ tests. + + +Building for Testing +-------------------- + +To build all tests — C++ executables and Python validation — configure CMake with:: + + cmake \ + -DMonarch_BUILD_MONARCH3=ON \ + -DMonarch_ENABLE_TESTING=ON \ + -DMonarch_BUILD_PYTHON=ON \ + /path/to/monarch/source + + make + +.. note:: + ``Monarch_BUILD_PYTHON`` requires `pybind11 `_ + and Python 3 development headers (``python3-dev`` or equivalent). + ``numpy`` and ``pytest`` must also be available in the active Python environment. + +To build only the C++ test executables (no Python binding or validation tests):: + + cmake -DMonarch_BUILD_MONARCH3=ON -DMonarch_ENABLE_TESTING=ON /path/to/monarch/source + make + + +Running Tests +------------- + +**Via CTest** (recommended; runs everything registered in the build):: + + cd build + ctest --output-on-failure + +This runs the C++ test executables and, if the Python binding was built, the pytest +validation suite as a single CTest entry named ``monarch3_validation``. + +**Running the Python validation tests directly** (after building and installing):: + + cd build + source bin/add_lib_python_path.sh # makes monarch3.so importable + pytest Monarch3/Tests/Validation/ -v + +Run a specific test file:: + + pytest Monarch3/Tests/Validation/test_cpp_write_python_read.py -v + +Run a specific test function:: + + pytest Monarch3/Tests/Validation/test_cpp_write_python_read.py::test_stream0_data -v + + +Validation +---------- + +The validation tests verify cross-language compatibility: that egg v3 files written by +C++ are correctly read by Python, and that files written by Python are correctly read by +C++. All three test modules use the same gold-standard 4-stream data structure defined +in the ``M3WriteTest`` C++ executable. + +Gold-Standard Data +~~~~~~~~~~~~~~~~~~ + +Every validation test is written against a common set of known values: + ++--------+----------------------------+---------+-----------+------+------+----------+---------------------+ +| Stream | Source | Channels| Format | Rate | Rec | Type | Record values | ++========+============================+=========+===========+======+======+==========+=====================+ +| 0 | "1-channel device" | 1 | — | 500 | 10 | uint8 | rec0=1, rec1=10 | ++--------+----------------------------+---------+-----------+------+------+----------+---------------------+ +| 1 | "2-channel device" | 2 | interleaved| 250 | 5 | uint16 | rec0=(1,2), | +| | | | | | | | rec1=(1000,2000), | +| | | | | | | | rec2=(10000,20000) | ++--------+----------------------------+---------+-----------+------+------+----------+---------------------+ +| 2 | "3-channel device" | 3 | separate | 100 | 5 | uint8 | rec0=(1,2,3), | +| | | | | | | | rec1=(10,20,30) | ++--------+----------------------------+---------+-----------+------+------+----------+---------------------+ +| 3 | "Floating-point device" | 1 | — | 100 | 10 | float32 | rec0=π, rec1=e | ++--------+----------------------------+---------+-----------+------+------+----------+---------------------+ + +Header fields: ``run_duration=8675309``, ``timestamp="Stardate 33515"``, +``description="Bigger on the inside"``. + +Stream 1 has two acquisitions: acquisition 0 contains record 0; acquisition 1 contains +records 1 and 2. Stream 3 has two acquisitions, one record each. + +The shared constants are defined in ``conftest.py`` and imported by all test modules. + +Test Scenarios +~~~~~~~~~~~~~~ + +**C++ write → Python read** (``test_cpp_write_python_read.py``) + +The fixture ``cpp_written_egg`` invokes the ``M3WriteTest`` executable to produce a +known-good egg file in a temporary directory. The tests then open that file with the +Python binding and verify: + +- Header fields (``run_duration``, ``timestamp``, ``description``, ``n_streams``) +- Per-stream structure (channel count, acquisition count, record count, record size) +- Sample values for each stream and channel, including float comparison with + ``numpy.isclose`` +- Offset-based record navigation for stream 1: forward skips, rewinds, and requests + past the end or before the beginning of the file (mirrors the checks in + ``M3ReadTest.cc``) + +**Python write → C++ read** (``test_python_write_cpp_read.py``) + +The helper function ``write_standard_egg(path)`` uses the Python binding to write the +same 4-stream structure as ``M3WriteTest``. The test then runs the ``M3ReadTest`` +executable against that file and asserts that it exits with return code 0. Since +``M3ReadTest`` checks record counts, acquisition counts, channel counts, and data values +internally, a clean exit confirms byte-level compatibility. + +**Python write → Python read** (``test_python_write_python_read.py``) + +Uses ``write_standard_egg`` (imported from ``test_python_write_cpp_read.py``) to produce +a file, then reads it back entirely with the Python binding. This isolates the binding +from the C++ executables and verifies that the write and read paths are internally +consistent. The same structure and value checks as the C++ write → Python read scenario +are applied. diff --git a/Documentation/UsageMonarch3.rst b/Documentation/UsageMonarch3.rst deleted file mode 100644 index 7e5a969..0000000 --- a/Documentation/UsageMonarch3.rst +++ /dev/null @@ -1,71 +0,0 @@ -How to use Monarch3 -=================== - -Thread safety: Reading and writing records (via M3Stream::ReadRecord() and M3Stream::WriteRecord(), respectively) are thread-safe -except that the HDF5 C library (on which the C++ library is built) is inherently non-thread-safe. Though multi-threaded writing may -work even most of the time, it is inherently unstable. -All other operations in Monarch (besides writing and reading records) are are explicitly not thread-safe. - -Reading Egg3 Files ------------------- - -1. Open the file: ``Monarch3::OpenForReading( [filename] )`` -2. Access the header information: ``Monarch3::ReadHeader()`` -3. Get the pointer to the header, and use as needed: ``Monarch3::GetHeader()`` -4. Get the pointer(s) to the stream(s) in the file: ``Monarch3::GetStream( [stream number] )`` -5. Setup to access the data in a stream. You can acces either the record for the entire stream with ``Monarch3::GetStreamRecord()``, - or for individual channels with ``Monarch3::GetChannelRecord( [record number] )``. - If you have only one channel in the stream, the distinction between those is irrelevant. - The record objects have a function ``M3Record::GetData()`` to get the data array. - There are two ways in which you can interact with the data array: - - * If you want to access the data as an array of bytes (e.g. because either your data is of type ``uint8_t``, or you want to use ``memcpy``), you can use the pointer returned by ``M3Record::GetData()``; - * If you want to access the data as an array of other integer or floating-point data types, you can pass the data pointer from ``M3Record::GetData()`` to an ``M3DataReader`` object, along with the data type size and data format flag. The type of the values that are returned is specified as a template argument for ``M3DataReader``; it doesn't have to match the data type in the data array exactly, but it should have at least as many bytes as the data elements, and if the data elements are integer, it should be an integer, and if the data elements are floating-point, it should be floating-point. - * If you want to access the data as an array of complex floating-point data types, you can pass the data pointer from ``M3Record::GetData()`` to an ``M3ComplexDataReader`` object, along with the data type size and data format flag (you can also specify the element size, but for complex data it should be the default, 2). The type of the values that are returned is specified as a template argument for ``M3ComplexDataReader``; it should either be f4_complex or f8_complex, or the equivalent. - -6. When moving from record to record in the file, the memory used for the data stays the same, but it gets updated with new values. - To move to a new record use the ``M3Stream::ReadRecord( [offset] )`` function. The offset parameter allows you to move forward and - backward within the file. If the last record read was ``[J]`` (``= -1`` for a just-opened file), ReadRecord will access the ``[J+1+offset]`` record. - This means that the offset parameter has the following meanings: - - * if ``offset == 0`` (default), the next record will be accessed; - * if ``offset == -1``, the current record will be reread; - * ``offset < -1`` will go backwards in the file; - * ``offset > 0`` will skip forward in the file. - - The outcomes from the call are: - - * returns true if the move was successful; - * returns false if the move was unsuccessful because it goes past the end (or beginning) of the file; - * throws an M3Exception if there was an error. - -7. When you're finished reading, use ``Monarch3::FinishReading()`` to close the file. - - -Writing Egg3 Files ------------------- - -1. Open the file: ``Monarch3::OpenForWriting( [filename] )`` -2. Get the pointer to the header, and use as needed: ``Monarch3::GetHeader()`` -3. Fill in the header information and setup the streams. For the latter, use the AddStream functions to add streams with one or multiple channels. -4. Write the header information: ``Monarch3::WriteHeader()`` -5. Get the pointer(s) to the stream(s) in the file: ``Monarch3::GetStream( [stream number] )`` -6. Setup to access the data in a stream. You can acces either the record for the entire stream with ``Monarch3::GetStreamRecord()``, - or for individual channels with ``Monarch3::GetChannelRecord( [record number] )``. - If you have only one channel in the stream, the distinction between those is irrelevant. - The record objects have a function ``M3Record::GetData()`` to get the data array. - There are two ways in which you can interact with the data array: - - * If you want to access the data as an array of bytes (e.g. because either your data is of type ``uint8_t``, or you want to use ``memcpy``), you can use the pointer returned by ``M3Record::GetData()``; - * If you want to access the data as an array of other integer or floating-point data types, you can pass the data pointer from ``M3Record::GetData()`` to an ``M3DataWriter`` object, along with the data type size and data format flag. The type of the values that are passed to the writer is specified as a template argument for ``M3DataWriter``; it doesn't have to match the data type in the data array exactly, but it should be no larger than the data elements, and if the data elements are integer, it should be an integer, and if the data elements are floating-point, it should be floating-point. - * If you want to access the data as an array of complex floating-point data types, you can pass the data pointer from ``M3Record::GetData()`` to an ``M3ComplexDataWriter`` object, along with the data type size and data format flag (you can also specify the element size, but for complex data it should be the default, 2). The type of the values that are returned is specified as a template argument for ``M3ComplexDataWriter``; it should either be ``f4_complex`` or ``f8_complex``, or the equivalent. - -7. For each record, copy the data to the stream data memory using the access method you chose above, and then write to disk with ``M3Stream::WriteRecord( [is new acquisition?] )``. - When a record is from a different acquisition than the previous record, the flag passed to ``WriteRecord`` should be ``true``; otherwise it should be ``false``. - The outcomes from the call are: - - * returns true if the write was successful; - * throws an M3Exception if there was an error; - * (should never return false). - -8. When you're finished writing, use ``Monarch3::FinishWriting()`` to close the file. diff --git a/Documentation/UsageMonarch3Cpp.rst b/Documentation/UsageMonarch3Cpp.rst new file mode 100644 index 0000000..86930b4 --- /dev/null +++ b/Documentation/UsageMonarch3Cpp.rst @@ -0,0 +1,298 @@ +How to use Monarch3 in C++ +========================== + +Thread safety: Reading and writing records (via ``M3Stream::ReadRecord()`` and ``M3Stream::WriteRecord()``, respectively) are thread-safe +except that the HDF5 C library (on which the C++ library is built) is inherently non-thread-safe. Though multi-threaded writing may +work even most of the time, it is inherently unstable. +All other operations in Monarch (besides writing and reading records) are explicitly not thread-safe. + + +Constants +--------- + +The following constants are defined in the ``monarch3`` namespace (``M3Constants.hh``): + +**Data format** + +* ``sDigitizedUS`` -- unsigned integer samples +* ``sDigitizedS`` -- signed integer samples +* ``sAnalog`` -- floating-point samples + +**Bit alignment** + +* ``sBitsAlignedLeft`` -- significant bits are aligned to the MSB of the sample word +* ``sBitsAlignedRight`` -- significant bits are aligned to the LSB of the sample word + +**Channel format (multi-channel streams)** + +* ``sInterleaved`` -- channel samples are interleaved within the stream record +* ``sSeparate`` -- each channel has its own contiguous block within the stream record + + +Reading Egg3 Files +------------------ + +1. Open the file: ``Monarch3::OpenForReading( [filename] )`` + +2. Read the header: ``Monarch3::ReadHeader()`` + +3. Get the pointer to the header and inspect it: ``Monarch3::GetHeader()`` + + Key fields on ``M3Header``: + + * ``GetEggVersion()`` / ``SetEggVersion()`` + * ``GetFilename()`` + * ``GetRunDuration()`` -- run duration in milliseconds + * ``GetTimestamp()`` + * ``GetDescription()`` + + Stream and channel metadata are available through header vectors: + + .. code-block:: cpp + + const M3Header* hdr = monarch->GetHeader(); + + // Stream-level metadata + const M3StreamHeader& streamHdr = hdr->GetStreamHeaders()[0]; + streamHdr.GetNChannels(); + streamHdr.GetAcquisitionRate(); + streamHdr.GetRecordSize(); + streamHdr.GetDataTypeSize(); + streamHdr.GetDataFormat(); + + // Channel-level metadata + const M3ChannelHeader& chanHdr = hdr->GetChannelHeaders()[0]; + chanHdr.GetVoltageOffset(); + chanHdr.GetVoltageRange(); + +4. Get the pointer(s) to the stream(s) in the file: ``Monarch3::GetStream( [stream number] )`` + + The total number of records in the stream is available via ``M3Stream::GetNRecordsInFile()``. + +5. Setup to access the data in a stream. You can access either the record for the entire stream with ``M3Stream::GetStreamRecord()``, + or for individual channels with ``M3Stream::GetChannelRecord( [channel number] )``. + If you have only one channel in the stream, the distinction between those is irrelevant. + The record objects have a function ``M3Record::GetData()`` to get the raw byte data array. + There are three ways in which you can interact with the data array: + + * If you want to access the data as an array of bytes (e.g. because either your data is of type ``uint8_t``, or you want to use ``memcpy``), you can use the pointer returned by ``M3Record::GetData()``; + * If you want to access the data as an array of other integer or floating-point data types, you can pass the data pointer from ``M3Record::GetData()`` to an ``M3DataReader`` object, along with the data type size and data format flag. The type of the values that are returned is specified as a template argument for ``M3DataReader``; it doesn't have to match the data type in the data array exactly, but it should have at least as many bytes as the data elements, and if the data elements are integer, it should be an integer, and if the data elements are floating-point, it should be floating-point. + * If you want to access the data as an array of complex floating-point data types, you can pass the data pointer from ``M3Record::GetData()`` to an ``M3ComplexDataReader`` object, along with the data type size and data format flag (you can also specify the element size, but for complex data it should be the default, 2). The type of the values that are returned is specified as a template argument for ``M3ComplexDataReader``; it should either be ``f4_complex`` or ``f8_complex``, or the equivalent. + +6. When moving from record to record in the file, the memory used for the data stays the same, but it gets updated with new values. + To move to a new record use the ``M3Stream::ReadRecord( [offset] )`` function. The offset parameter allows you to move forward and + backward within the file. If the last record read was ``[J]`` (``= -1`` for a just-opened file), ``ReadRecord`` will access the ``[J+1+offset]`` record. + This means that the offset parameter has the following meanings: + + * if ``offset == 0`` (default), the next record will be accessed; + * if ``offset == -1``, the current record will be reread; + * ``offset < -1`` will go backwards in the file; + * ``offset > 0`` will skip forward in the file. + + The outcomes from the call are: + + * returns ``true`` if the move was successful; + * returns ``false`` if the move was unsuccessful because it goes past the end (or beginning) of the file; + * throws an ``M3Exception`` if there was an error. + + After each successful ``ReadRecord``, the current acquisition and record position are available via: + + * ``M3Stream::GetAcquisitionId()`` -- index of the current acquisition + * ``M3Stream::GetRecordCountInAcq()`` -- record index within the current acquisition + + A typical read loop: + + .. code-block:: cpp + + M3Stream* stream = monarch->GetStream( 0 ); + const M3Record* record = stream->GetChannelRecord( 0 ); + + // Data type from the stream header + unsigned dataTypeSize = stream->GetDataTypeSize(); + uint32_t dataFormat = hdr->GetStreamHeaders()[0].GetDataFormat(); + + M3DataReader< uint16_t > reader( record->GetData(), dataTypeSize, dataFormat ); + + while( stream->ReadRecord() ) + { + unsigned recSize = stream->GetChannelRecordSize(); + for( unsigned i = 0; i < recSize; ++i ) + { + uint16_t sample = reader.at( i ); + // process sample ... + } + } + +7. When you're finished reading, use ``Monarch3::FinishReading()`` to close the file. + + +Writing Egg3 Files +------------------ + +1. Open the file: ``Monarch3::OpenForWriting( [filename] )`` + +2. Get the pointer to the header and configure it: ``Monarch3::GetHeader()`` + + .. code-block:: cpp + + M3Header* hdr = monarch->GetHeader(); + hdr->SetFilename( filename ); + hdr->SetRunDuration( 1000 ); // milliseconds + hdr->SetTimestamp( "2024-01-01T00:00:00" ); + hdr->SetDescription( "My data" ); + +3. Add streams using the ``AddStream`` functions. Both overloads return the stream number, + which is used to address the stream after the header is written. + + For a single-channel stream:: + + unsigned streamNum = hdr->AddStream( + source, // std::string: digitizer label + acqRate, // uint32_t: acquisition rate (MHz) + recSize, // uint32_t: samples per record + sampleSize, // uint32_t: elements per sample (1 for real, 2 for complex) + dataTypeSize, // uint32_t: bytes per element + dataFormat, // uint32_t: sDigitizedUS, sDigitizedS, or sAnalog + bitDepth, // uint32_t: number of significant bits per sample + bitAlignment // uint32_t: sBitsAlignedLeft or sBitsAlignedRight + ); + + For a multi-channel stream:: + + unsigned streamNum = hdr->AddStream( + source, // std::string: digitizer label + nChannels, // uint32_t: number of channels + channelFormat, // uint32_t: sInterleaved or sSeparate + acqRate, // uint32_t: acquisition rate (MHz) + recSize, // uint32_t: samples per record per channel + sampleSize, // uint32_t: elements per sample + dataTypeSize, // uint32_t: bytes per element + dataFormat, // uint32_t: sDigitizedUS, sDigitizedS, or sAnalog + bitDepth, // uint32_t: number of significant bits per sample + bitAlignment // uint32_t: sBitsAlignedLeft or sBitsAlignedRight + ); + +4. Write the header information (this creates the HDF5 structure and allocates stream objects): ``Monarch3::WriteHeader()`` + +5. Get the pointer(s) to the stream(s) in the file: ``Monarch3::GetStream( [stream number] )`` + +6. Setup to access the data in a stream. You can access either the record for the entire stream with ``M3Stream::GetStreamRecord()``, + or for individual channels with ``M3Stream::GetChannelRecord( [channel number] )``. + If you have only one channel in the stream, the distinction between those is irrelevant. + The record objects have a function ``M3Record::GetData()`` to get the raw byte data array. + There are three ways in which you can interact with the data array: + + * If you want to access the data as an array of bytes (e.g. because either your data is of type ``uint8_t``, or you want to use ``memcpy``), you can use the pointer returned by ``M3Record::GetData()``; + * If you want to access the data as an array of other integer or floating-point data types, you can pass the data pointer from ``M3Record::GetData()`` to an ``M3DataWriter`` object, along with the data type size and data format flag. The type of the values that are passed to the writer is specified as a template argument for ``M3DataWriter``; it doesn't have to match the data type in the data array exactly, but it should be no larger than the data elements, and if the data elements are integer, it should be an integer, and if the data elements are floating-point, it should be floating-point. + * If you want to access the data as an array of complex floating-point data types, you can pass the data pointer from ``M3Record::GetData()`` to an ``M3ComplexDataWriter`` object, along with the data type size and data format flag (you can also specify the element size, but for complex data it should be the default, 2). The type of the values that are returned is specified as a template argument for ``M3ComplexDataWriter``; it should either be ``f4_complex`` or ``f8_complex``, or the equivalent. + +7. For each record, copy the data to the stream data memory using the access method you chose above, and then write to disk with ``M3Stream::WriteRecord( [is new acquisition?] )``. + When a record is from a different acquisition than the previous record, the flag passed to ``WriteRecord`` should be ``true``; otherwise it should be ``false``. + The outcomes from the call are: + + * returns ``true`` if the write was successful; + * throws an ``M3Exception`` if there was an error; + * (should never return ``false``). + +8. When you're finished writing, use ``Monarch3::FinishWriting()`` to close the file. + + +Complete Examples +----------------- + +**Writing a file** + +.. code-block:: cpp + + #include "M3Monarch.hh" + #include "M3DataInterface.hh" + #include + + using namespace monarch3; + + int main() + { + std::string filename = "output.egg"; + + Monarch3* monarch = Monarch3::OpenForWriting( filename ); + + M3Header* hdr = monarch->GetHeader(); + hdr->SetFilename( filename ); + hdr->SetRunDuration( 1000 ); + hdr->SetTimestamp( "2024-01-01T00:00:00" ); + hdr->SetDescription( "Example egg file" ); + + unsigned streamNum = hdr->AddStream( + "my-digitizer", + 200, // acquisition rate (MHz) + 1024, // samples per record + 1, // elements per sample + 1, // bytes per element (uint8) + sDigitizedUS, + 8, // bit depth + sBitsAlignedLeft + ); + + monarch->WriteHeader(); + + M3Stream* stream = monarch->GetStream( streamNum ); + M3Record* record = stream->GetChannelRecord( 0 ); + unsigned recSize = stream->GetChannelRecordSize(); + + M3DataWriter< uint8_t > writer( record->GetData(), 1, sDigitizedUS ); + + // Acquisition 0, first record + for( unsigned i = 0; i < recSize; ++i ) writer.set_at( 42, i ); + stream->WriteRecord( true ); // true = new acquisition + + // Acquisition 0, second record + for( unsigned i = 0; i < recSize; ++i ) writer.set_at( 100, i ); + stream->WriteRecord( false ); // false = continue acquisition + + monarch->FinishWriting(); + delete monarch; + return 0; + } + + +**Reading a file** + +.. code-block:: cpp + + #include "M3Monarch.hh" + #include "M3DataInterface.hh" + #include + + using namespace monarch3; + + int main() + { + Monarch3* monarch = Monarch3::OpenForReading( "output.egg" ); + monarch->ReadHeader(); + + const M3Header* hdr = monarch->GetHeader(); + std::cout << "Egg version: " << hdr->GetEggVersion() << "\n"; + std::cout << "Timestamp: " << hdr->GetTimestamp() << "\n"; + + const M3StreamHeader& streamHdr = hdr->GetStreamHeaders()[0]; + unsigned dataTypeSize = streamHdr.GetDataTypeSize(); + uint32_t dataFormat = streamHdr.GetDataFormat(); + + M3Stream* stream = monarch->GetStream( 0 ); + std::cout << "Stream 0: " << stream->GetNRecordsInFile() << " record(s)\n"; + + const M3Record* record = stream->GetChannelRecord( 0 ); + M3DataReader< uint8_t > reader( record->GetData(), dataTypeSize, dataFormat ); + + while( stream->ReadRecord() ) + { + unsigned recSize = stream->GetChannelRecordSize(); + std::cout << " acq=" << stream->GetAcquisitionId() + << " rec=" << stream->GetRecordCountInAcq() + << " data[0]=" << (int)reader.at( 0 ) << "\n"; + } + + monarch->FinishReading(); + delete monarch; + return 0; + } diff --git a/Documentation/UsageMonarch3Python.rst b/Documentation/UsageMonarch3Python.rst new file mode 100644 index 0000000..43991f4 --- /dev/null +++ b/Documentation/UsageMonarch3Python.rst @@ -0,0 +1,243 @@ +How to use Monarch3 in Python +============================= + +The ``monarch3`` Python module is a pybind11 binding of the Monarch3 C++ library. +It exposes the same read/write workflow as the C++ API but with Pythonic conventions: +method and property names use ``snake_case``, raw data buffers are returned as +``numpy.ndarray`` views (``uint8`` by default), and files can be managed with Python's +``with`` statement. + +.. note:: + Record data is returned as a ``numpy.ndarray`` of ``uint8`` bytes. + Use ``array.view(dtype)`` to reinterpret the buffer as the appropriate element type + (e.g. ``np.uint16``, ``np.float32``). The element type and size are available from + the stream header's ``data_type_size`` and ``data_format`` properties. + +Thread safety follows the same rules as the C++ library: calling ``read_record()`` and +``write_record()`` is thread-safe (the GIL is released during disk I/O), but all other +operations are not thread-safe. + + +Constants +--------- + +The following constants are provided by the ``monarch3`` module: + +**Data format** + +* ``sDigitizedUS`` -- unsigned integer samples +* ``sDigitizedS`` -- signed integer samples +* ``sAnalog`` -- floating-point samples + +**Bit alignment** + +* ``sBitsAlignedLeft`` -- significant bits are aligned to the MSB of the sample word +* ``sBitsAlignedRight`` -- significant bits are aligned to the LSB of the sample word + +**Channel format (multi-channel streams)** + +* ``sInterleaved`` -- channel samples are interleaved within the stream record +* ``sSeparate`` -- each channel has its own contiguous block within the stream record + + +Reading Egg3 Files +------------------ + +1. Open the file:: + + m = monarch3.Monarch3.open_for_reading( filename ) + + Or use the context manager (recommended; closes the file automatically):: + + with monarch3.Monarch3.open_for_reading( filename ) as m: + ... + +2. Read the header:: + + m.read_header() + +3. Inspect the header:: + + hdr = m.get_header() + print( hdr.egg_version ) + print( hdr.run_duration ) + print( hdr.timestamp ) + print( hdr.description ) + + Stream and channel metadata are available through header lists:: + + stream_hdr = hdr.stream_headers[0] + print( stream_hdr.n_channels, stream_hdr.acquisition_rate, stream_hdr.record_size ) + print( stream_hdr.data_type_size, stream_hdr.data_format ) + + channel_hdr = hdr.channel_headers[0] + print( channel_hdr.voltage_offset, channel_hdr.voltage_range ) + +4. Get a stream:: + + stream = m.get_stream( stream_index ) + n_records = stream.n_records_in_file + +5. Access data record by record. + Use ``get_channel_data( channel )`` to obtain a ``numpy.ndarray`` of ``uint8`` bytes + for a single channel, or ``get_stream_data()`` for the full (potentially interleaved) + stream buffer:: + + while stream.read_record(): + arr = stream.get_channel_data( 0 ) + # reinterpret bytes as the actual element type, e.g.: + samples = arr.view( np.uint16 ) + print( samples ) + + The ``offset`` parameter to ``read_record`` controls navigation within the file + (same semantics as the C++ API): + + * ``offset == 0`` (default): advance to the next record + * ``offset == -1``: re-read the current record + * ``offset < -1``: step backward + * ``offset > 0``: skip forward + + ``read_record()`` returns ``True`` on success and ``False`` when the requested + position is past the end (or before the beginning) of the file. + +6. Close the file (not needed when using the ``with`` statement):: + + m.finish_reading() + + +Writing Egg3 Files +------------------ + +1. Open the file:: + + m = monarch3.Monarch3.open_for_writing( filename ) + + Or use the context manager:: + + with monarch3.Monarch3.open_for_writing( filename ) as m: + ... + +2. Configure the header:: + + hdr = m.get_header() + hdr.filename = filename + hdr.run_duration = 1000 # milliseconds + hdr.timestamp = "2024-01-01T00:00:00" + hdr.description = "My data" + +3. Add streams. + For a single-channel stream:: + + stream_num = hdr.add_stream( + source = "my-digitizer", + acq_rate = 200, # MHz + rec_size = 4096, # samples per record + sample_size = 1, # elements per sample + data_type_size = 2, # bytes per element + data_format = monarch3.sDigitizedUS, + bit_depth = 14, + bit_alignment = monarch3.sBitsAlignedRight, + ) + + For a multi-channel stream:: + + stream_num = hdr.add_stream( + source = "my-2ch-digitizer", + n_channels = 2, + channel_format = monarch3.sInterleaved, + acq_rate = 200, + rec_size = 4096, + sample_size = 1, + data_type_size = 2, + data_format = monarch3.sDigitizedUS, + bit_depth = 14, + bit_alignment = monarch3.sBitsAlignedRight, + ) + +4. Write the header (this creates the HDF5 structure and allocates stream objects):: + + m.write_header() + +5. Get the stream object:: + + stream = m.get_stream( stream_num ) + +6. For each record, fill the data buffer and write to disk:: + + # get a writable uint8 view; reinterpret as the target element type + buf = stream.get_channel_data( 0 ).view( np.uint16 ) + buf[:] = my_samples # numpy array of uint16 + + stream.write_record( is_new_acquisition=True ) # True = start new acquisition + stream.write_record( is_new_acquisition=False ) # False = continue acquisition + +7. Close the file (not needed when using the ``with`` statement):: + + m.finish_writing() + + +Complete Examples +----------------- + +**Writing a file** + +.. code-block:: python + + import monarch3 + import numpy as np + + filename = "output.egg" + + with monarch3.Monarch3.open_for_writing( filename ) as m: + hdr = m.get_header() + hdr.filename = filename + hdr.run_duration = 1000 + hdr.timestamp = "2024-01-01T00:00:00" + hdr.description = "Example egg file" + + stream_num = hdr.add_stream( + source = "my-digitizer", + acq_rate = 200, + rec_size = 1024, + sample_size = 1, + data_type_size = 1, + data_format = monarch3.sDigitizedUS, + bit_depth = 8, + bit_alignment = monarch3.sBitsAlignedLeft, + ) + + m.write_header() + + stream = m.get_stream( stream_num ) + + # Acquisition 0, two records + buf = stream.get_channel_data( 0 ) + buf[:] = 42 + stream.write_record( True ) + + buf[:] = 100 + stream.write_record( False ) + + +**Reading a file** + +.. code-block:: python + + import monarch3 + import numpy as np + + with monarch3.Monarch3.open_for_reading( "output.egg" ) as m: + m.read_header() + hdr = m.get_header() + print( "Egg version:", hdr.egg_version ) + print( "Timestamp: ", hdr.timestamp ) + + stream_hdr = hdr.stream_headers[0] + dtype = np.uint8 # choose based on stream_hdr.data_type_size and data_format + + stream = m.get_stream( 0 ) + print( f"Stream 0: {stream.n_records_in_file} record(s)" ) + + while stream.read_record(): + arr = stream.get_channel_data( 0 ).view( dtype ) + print( f" acq={stream.acquisition_id} rec={stream.record_count_in_acq} data={arr}" ) diff --git a/Documentation/conf.py b/Documentation/conf.py index b0a3fec..97cabaa 100644 --- a/Documentation/conf.py +++ b/Documentation/conf.py @@ -17,8 +17,9 @@ # * the project, copyright, and author variables # * the arguments used to assign variables htmlhelp_basename, latex_documents, man_pages, and texinfo_documents -#import sys +import sys import os +import importlib.util from subprocess import call, check_output # If extensions (or modules to document with autodoc) are in another directory, @@ -26,32 +27,18 @@ # documentation root, use os.path.abspath to make it absolute, like shown here. #sys.path.insert(0, os.path.abspath('.')) +# Fall back to the pure-Python stub when the compiled monarch3 extension is not +# available (e.g. local doc builds where the C++ library has not been compiled). +if importlib.util.find_spec('monarch3') is None: + sys.path.insert(0, os.path.abspath('stubs')) + # version this_version = 'v?.?.?' try: - this_version = check_output(['git', 'describe', '--abbrev=0', '--tags']) + this_version = check_output(['git', 'describe', '--abbrev=0', '--tags']).decode('utf-8').strip() except: pass -# environment variables used by Doxygen -os.environ['PROJECT_NAME'] = 'Monarch' -os.environ['PROJECT_NUMBER'] = this_version -os.environ['PROJECT_BRIEF_DESC'] = 'Project 8 Data File Format Library' -# located in your documentation directory, or give the relative path from the documentation directory -os.environ['PROJECT_LOGO'] = '' - -# directories in which doxygen should look for source files; if you have a `doxfiles` directory in your documentation, that should go here; string with space-separated directories -os.environ['DOXYGEN_INPUT'] = 'DoxFiles ../Monarch3 ../Monarch2' -# directories within DOXYGEN_INPUT that you want to exclude from doxygen (e.g. if there's a submodule included that you don't want to index); string with space-separated directories -os.environ['DOXYGEN_EXCLUDE'] = '' -# directories outside of DOXYGEN_INPUT that you want the C preprocessor to look in for macro definitions (e.g. if there's a submodule not included that has relevant macros); string with space-separated directories -os.environ['PREPROC_INCLUDE_PATH'] = '../Scarab/library/utility ../Scarab/library/logger' - -# Doxygen -call(['doxygen', '../Scarab/documentation/cpp/Doxyfile']) -call(['mv', './user_doxygen_out/html', './_static']) - - on_rtd = os.environ.get('READTHEDOCS', None) == 'True' if on_rtd: html_theme = 'default' @@ -69,7 +56,15 @@ # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = [] +extensions = [ + 'sphinx.ext.autodoc', +] + +autodoc_default_options = { + 'members': True, + 'undoc-members': False, + 'show-inheritance': True, +} # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] @@ -87,7 +82,7 @@ # General information about the project. project = u'Monarch' -copyright = u'2018, Monarch Authors' +copyright = u'2026, Monarch Authors' author = u'Project 8 Collaboration' # The version info for the project you're documenting, acts as replacement for diff --git a/Documentation/index.rst b/Documentation/index.rst index 4121b74..d52ee33 100644 --- a/Documentation/index.rst +++ b/Documentation/index.rst @@ -7,8 +7,15 @@ Contents: :maxdepth: 2 Monarch_versions - UsageMonarch3 + UsageMonarch3Cpp + UsageMonarch3Python + TestingMonarch3 .. end of toc .. (you must not remove or modify the above comment line, it is required by the API Doc generation) -`Full Doxygen API Reference <_static/index.html>`_ + +API Reference +------------- + +* `C++ API Reference (Doxygen) <_static/index.html>`_ +* :doc:`Python API Reference (monarch3) ` diff --git a/Documentation/requirements.txt b/Documentation/requirements.txt new file mode 100644 index 0000000..3ef203c --- /dev/null +++ b/Documentation/requirements.txt @@ -0,0 +1,2 @@ +sphinx +furo diff --git a/Documentation/run_doxygen.py b/Documentation/run_doxygen.py new file mode 100644 index 0000000..cdccecc --- /dev/null +++ b/Documentation/run_doxygen.py @@ -0,0 +1,28 @@ +# This script sets environment variables that are used by Doxygen + +import os +from subprocess import call, check_output + +# version +this_version = 'v?.?.?' +try: + this_version = check_output(['git', 'describe', '--abbrev=0', '--tags']).decode('utf-8').strip() +except: + pass + +# environment variables used by Doxygen +os.environ['PROJECT_NAME'] = 'Monarch' +os.environ['PROJECT_NUMBER'] = this_version +os.environ['PROJECT_BRIEF_DESC'] = 'Library for reading and writing egg files' +# located in your documentation directory, or give the relative path from the documentation directory +#os.environ['PROJECT_LOGO'] = './documentation/images/Logo_55x55.png' + +# directories in which doxygen should look for source files; if you have a `doxfiles` directory in your documentation, that should go here; string with space-separated directories +os.environ['DOXYGEN_INPUT'] = './Documentation/DoxFiles ./Monarch2 ./Monarch3' +# directories within DOXYGEN_INPUT that you want to exclude from doxygen (e.g. if there's a submodule included that you don't want to index); string with space-separated directories +os.environ['DOXYGEN_EXCLUDE'] = '' +# directories outside of DOXYGEN_INPUT that you want the C preprocessor to look in for macro definitions (e.g. if there's a submodule not included that has relevant macros); string with space-separated directories +os.environ['PREPROC_INCLUDE_PATH'] = './Scarab/library/utility ./Scarab/library/logger' + +# Doxygen +call(['doxygen', './Scarab/documentation/Doxyfile']) diff --git a/Documentation/stubs/monarch3.py b/Documentation/stubs/monarch3.py new file mode 100644 index 0000000..267a180 --- /dev/null +++ b/Documentation/stubs/monarch3.py @@ -0,0 +1,664 @@ +""" +monarch3 -- Python interface to the Monarch3 egg-file library. + +The ``monarch3`` module is a pybind11 binding of the Monarch3 C++ library. +It exposes the same read/write workflow as the C++ API but with Pythonic +conventions: method and property names use ``snake_case``, raw data buffers +are returned as :class:`numpy.ndarray` views (``uint8`` by default), and +files can be managed with Python's ``with`` statement. + +.. note:: + This stub module is used when the compiled ``monarch3`` extension is not + available (e.g. during documentation builds on ReadTheDocs). It exists + solely to allow :mod:`sphinx.ext.autodoc` to introspect the API surface. + All method bodies are intentionally empty. + +Thread safety follows the same rules as the C++ library: calling +:meth:`M3Stream.read_record` and :meth:`M3Stream.write_record` is +thread-safe (the GIL is released during disk I/O), but all other operations +are not thread-safe. +""" + +# --------------------------------------------------------------------------- +# Constants +# --------------------------------------------------------------------------- + +#: Unsigned integer sample format. +sDigitizedUS: int = 0 + +#: Signed integer sample format. +sDigitizedS: int = 1 + +#: Floating-point (analog) sample format. +sAnalog: int = 2 + +#: Significant bits are aligned to the MSB of the sample word. +sBitsAlignedLeft: int = 0 + +#: Significant bits are aligned to the LSB of the sample word. +sBitsAlignedRight: int = 1 + +#: Channel samples are interleaved within the stream record. +sInterleaved: int = 0 + +#: Each channel occupies its own contiguous block within the stream record. +sSeparate: int = 1 + + +# --------------------------------------------------------------------------- +# Exception +# --------------------------------------------------------------------------- + +class Monarch3Exception(Exception): + """Raised by the monarch3 library on I/O errors and invalid operations.""" + pass + + +# --------------------------------------------------------------------------- +# Header classes +# --------------------------------------------------------------------------- + +class M3StreamHeader: + """Header information for a single data stream.""" + + @property + def number(self) -> int: + """Stream index within the file.""" + ... + + @property + def source(self) -> str: + """Source identifier string (e.g. digitizer name).""" + ... + + @property + def n_channels(self) -> int: + """Number of channels in this stream.""" + ... + + @property + def channels(self): + """List of channel indices belonging to this stream.""" + ... + + @property + def channel_format(self) -> int: + """Channel layout: :data:`sInterleaved` or :data:`sSeparate`.""" + ... + + @property + def acquisition_rate(self) -> int: + """Acquisition rate in MHz.""" + ... + + @property + def record_size(self) -> int: + """Number of samples per record (per channel).""" + ... + + @property + def sample_size(self) -> int: + """Number of elements per sample (1 for real, 2 for complex).""" + ... + + @property + def data_type_size(self) -> int: + """Size in bytes of each sample element.""" + ... + + @property + def data_format(self) -> int: + """Data format: :data:`sDigitizedUS`, :data:`sDigitizedS`, or :data:`sAnalog`.""" + ... + + @property + def bit_depth(self) -> int: + """Number of significant bits per sample.""" + ... + + @property + def bit_alignment(self) -> int: + """Bit alignment: :data:`sBitsAlignedLeft` or :data:`sBitsAlignedRight`.""" + ... + + @property + def n_acquisitions(self) -> int: + """Number of acquisitions recorded in this stream.""" + ... + + @property + def n_records(self) -> int: + """Total number of records across all acquisitions in this stream.""" + ... + + def __repr__(self) -> str: ... + + +class M3ChannelHeader: + """Header information for a single data channel.""" + + @property + def number(self) -> int: + """Channel index within the file.""" + ... + + @property + def source(self) -> str: + """Source identifier string (e.g. digitizer name).""" + ... + + @property + def acquisition_rate(self) -> int: + """Acquisition rate in MHz.""" + ... + + @property + def record_size(self) -> int: + """Number of samples per record.""" + ... + + @property + def sample_size(self) -> int: + """Number of elements per sample (1 for real, 2 for complex).""" + ... + + @property + def data_type_size(self) -> int: + """Size in bytes of each sample element.""" + ... + + @property + def data_format(self) -> int: + """Data format: :data:`sDigitizedUS`, :data:`sDigitizedS`, or :data:`sAnalog`.""" + ... + + @property + def bit_depth(self) -> int: + """Number of significant bits per sample.""" + ... + + @property + def bit_alignment(self) -> int: + """Bit alignment: :data:`sBitsAlignedLeft` or :data:`sBitsAlignedRight`.""" + ... + + @property + def voltage_offset(self) -> float: + """Voltage offset of the channel in volts.""" + ... + + @property + def voltage_range(self) -> float: + """Voltage range of the channel in volts.""" + ... + + @property + def dac_gain(self) -> float: + """DAC gain of the channel.""" + ... + + @property + def frequency_min(self) -> float: + """Minimum frequency in Hz.""" + ... + + @property + def frequency_range(self) -> float: + """Frequency range in Hz.""" + ... + + def __repr__(self) -> str: ... + + +class M3Header: + """Egg file header: run metadata and stream/channel configuration.""" + + # ------------------------------------------------------------------ + # Read-only properties + # ------------------------------------------------------------------ + + @property + def egg_version(self) -> str: + """Egg file format version string.""" + ... + + @property + def n_channels(self) -> int: + """Total number of channels across all streams.""" + ... + + @property + def n_streams(self) -> int: + """Total number of streams in the file.""" + ... + + @property + def stream_headers(self): + """List of :class:`M3StreamHeader` objects, one per stream.""" + ... + + @property + def channel_headers(self): + """List of :class:`M3ChannelHeader` objects, one per channel.""" + ... + + # ------------------------------------------------------------------ + # Read-write properties + # ------------------------------------------------------------------ + + @property + def filename(self) -> str: + """Output filename stored in the header.""" + ... + + @filename.setter + def filename(self, value: str) -> None: ... + + @property + def run_duration(self) -> int: + """Run duration in milliseconds.""" + ... + + @run_duration.setter + def run_duration(self, value: int) -> None: ... + + @property + def timestamp(self) -> str: + """Run timestamp string (ISO 8601 recommended).""" + ... + + @timestamp.setter + def timestamp(self, value: str) -> None: ... + + @property + def description(self) -> str: + """Free-text description of the run.""" + ... + + @description.setter + def description(self, value: str) -> None: ... + + # ------------------------------------------------------------------ + # Methods + # ------------------------------------------------------------------ + + def add_stream(self, + source: str, + acq_rate: int, + rec_size: int, + sample_size: int, + data_type_size: int, + data_format: int, + bit_depth: int, + bit_alignment: int, + chan_vec=None) -> int: + """Add a single-channel stream; returns the stream number. + + :param source: Digitizer or source identifier. + :param acq_rate: Acquisition rate in MHz. + :param rec_size: Number of samples per record. + :param sample_size: Number of elements per sample (1 for real data). + :param data_type_size: Bytes per element (e.g. 2 for ``uint16``). + :param data_format: :data:`sDigitizedUS`, :data:`sDigitizedS`, or :data:`sAnalog`. + :param bit_depth: Number of significant bits per sample. + :param bit_alignment: :data:`sBitsAlignedLeft` or :data:`sBitsAlignedRight`. + :param chan_vec: Optional list of channel indices to assign; ``None`` for automatic. + :returns: Stream number (index used with :meth:`Monarch3.get_stream`). + """ + ... + + def add_stream(self, # noqa: F811 (overload) + source: str, + n_channels: int, + channel_format: int, + acq_rate: int, + rec_size: int, + sample_size: int, + data_type_size: int, + data_format: int, + bit_depth: int, + bit_alignment: int, + chan_vec=None) -> int: + """Add a multi-channel stream; returns the stream number. + + :param source: Digitizer or source identifier. + :param n_channels: Number of channels in the stream. + :param channel_format: :data:`sInterleaved` or :data:`sSeparate`. + :param acq_rate: Acquisition rate in MHz. + :param rec_size: Number of samples per record per channel. + :param sample_size: Number of elements per sample. + :param data_type_size: Bytes per element. + :param data_format: :data:`sDigitizedUS`, :data:`sDigitizedS`, or :data:`sAnalog`. + :param bit_depth: Number of significant bits per sample. + :param bit_alignment: :data:`sBitsAlignedLeft` or :data:`sBitsAlignedRight`. + :param chan_vec: Optional list of channel indices to assign; ``None`` for automatic. + :returns: Stream number (index used with :meth:`Monarch3.get_stream`). + """ + ... + + def __repr__(self) -> str: ... + + +# --------------------------------------------------------------------------- +# Record class +# --------------------------------------------------------------------------- + +class M3Record: + """A single data record: record ID, timestamp, and raw data bytes.""" + + @property + def record_id(self) -> int: + """Record ID (``uint64``).""" + ... + + @property + def time(self) -> int: + """Timestamp in nanoseconds since the start of the run (``uint64``).""" + ... + + def get_data(self, nbytes: int): + """Return a writable :class:`numpy.ndarray` (``uint8``) view of the data buffer. + + This is a zero-copy view: the array does not own the data. + The caller must ensure the record and its parent stream remain alive. + + :param nbytes: Number of bytes to expose; should equal + :attr:`M3Stream.channel_record_n_bytes` (or + :attr:`M3Stream.stream_record_n_bytes` for the stream record). + :returns: ``numpy.ndarray`` of dtype ``uint8``. + Use :meth:`numpy.ndarray.view` to reinterpret as the actual element type. + """ + ... + + +# --------------------------------------------------------------------------- +# Stream class +# --------------------------------------------------------------------------- + +class M3Stream: + """Read/write access for a single data stream.""" + + # ------------------------------------------------------------------ + # State query properties + # ------------------------------------------------------------------ + + @property + def n_channels(self) -> int: + """Number of channels in this stream.""" + ... + + @property + def n_acquisitions(self) -> int: + """Number of acquisitions in this stream (valid after :meth:`read_record` or after writing).""" + ... + + @property + def acquisition_id(self) -> int: + """ID of the most recently accessed acquisition.""" + ... + + @property + def record_count_in_acq(self) -> int: + """Number of records read/written in the current acquisition.""" + ... + + @property + def n_records_in_file(self) -> int: + """Total number of records across all acquisitions in the file.""" + ... + + @property + def n_records_in_acquisition(self) -> int: + """Number of records in the current acquisition.""" + ... + + @property + def data_type_size(self) -> int: + """Size in bytes of each sample element.""" + ... + + @property + def sample_size(self) -> int: + """Number of elements per sample (1 for real, 2 for complex).""" + ... + + @property + def channel_record_size(self) -> int: + """Number of samples in a channel record.""" + ... + + @property + def channel_record_n_bytes(self) -> int: + """Size in bytes of a channel record data buffer.""" + ... + + @property + def stream_record_size(self) -> int: + """Number of samples in the full (potentially interleaved) stream record.""" + ... + + @property + def stream_record_n_bytes(self) -> int: + """Size in bytes of the full stream record data buffer.""" + ... + + @property + def is_interleaved(self) -> bool: + """``True`` if multi-channel data is stored interleaved in the file.""" + ... + + # ------------------------------------------------------------------ + # Record reading + # ------------------------------------------------------------------ + + def read_record(self, offset: int = 0, if_new_acq_start_at_first_rec: bool = True) -> bool: + """Read a record from the file. + + Assuming the last record read was ``[J]``, reads record ``[J+1+offset]``. + + :param offset: + * ``0`` (default): advance to the next record. + * ``-1``: re-read the current record. + * ``< -1``: step backward in the file. + * ``> 0``: skip forward in the file. + :param if_new_acq_start_at_first_rec: If ``True`` (default), when + stepping into a new acquisition the first record of that + acquisition is read regardless of the offset. Set to ``False`` + for backwards-compatible behaviour. + :returns: ``True`` on success; ``False`` when the requested position + is past the end (or before the beginning) of the file. + :raises Monarch3Exception: On I/O error. + """ + ... + + def get_stream_record(self) -> M3Record: + """Return the stream-level record object (all channels interleaved).""" + ... + + def get_channel_record(self, channel: int) -> M3Record: + """Return the record object for the given channel index. + + :param channel: Zero-based channel index. + """ + ... + + # ------------------------------------------------------------------ + # Convenience numpy data access + # ------------------------------------------------------------------ + + def get_stream_data(self): + """Return a writable :class:`numpy.ndarray` (``uint8``) view of the full stream record. + + Call after :meth:`read_record`. + Use :meth:`numpy.ndarray.view` to reinterpret the buffer as the + actual element type (e.g. ``arr.view(np.uint16)``). + + :returns: ``numpy.ndarray`` of dtype ``uint8`` with length + :attr:`stream_record_n_bytes`. + """ + ... + + def get_channel_data(self, channel: int): + """Return a writable :class:`numpy.ndarray` (``uint8``) view of a single channel's data. + + Call after :meth:`read_record` (reading) or before :meth:`write_record` (writing). + Use :meth:`numpy.ndarray.view` to reinterpret the buffer as the + actual element type (e.g. ``arr.view(np.uint16)``). + + :param channel: Zero-based channel index. + :returns: ``numpy.ndarray`` of dtype ``uint8`` with length + :attr:`channel_record_n_bytes`. + """ + ... + + # ------------------------------------------------------------------ + # Record writing + # ------------------------------------------------------------------ + + def write_record(self, is_new_acquisition: bool) -> bool: + """Write the current record contents to the file. + + Fill the data buffer via :meth:`get_channel_data` (or + :meth:`get_stream_data`) before calling this method. + + :param is_new_acquisition: ``True`` to start a new acquisition group; + ``False`` to continue the current acquisition. + :returns: ``True`` on success. + :raises Monarch3Exception: On I/O error. + """ + ... + + +# --------------------------------------------------------------------------- +# Top-level file handle +# --------------------------------------------------------------------------- + +class Monarch3: + """Top-level egg v3 file handle. + + Use :meth:`open_for_reading` or :meth:`open_for_writing` to obtain an + instance. Supports the context manager protocol (``with`` statement), + which automatically calls :meth:`finish_reading` or + :meth:`finish_writing` on exit. + + Example (reading):: + + with monarch3.Monarch3.open_for_reading("data.egg") as m: + m.read_header() + stream = m.get_stream(0) + while stream.read_record(): + arr = stream.get_channel_data(0).view(np.uint16) + ... + + Example (writing):: + + with monarch3.Monarch3.open_for_writing("out.egg") as m: + hdr = m.get_header() + hdr.run_duration = 1000 + hdr.add_stream("src", 200, 4096, 1, 2, monarch3.sDigitizedUS, 14, + monarch3.sBitsAlignedRight) + m.write_header() + stream = m.get_stream(0) + buf = stream.get_channel_data(0).view(np.uint16) + buf[:] = my_samples + stream.write_record(True) + """ + + @staticmethod + def open_for_reading(filename: str) -> "Monarch3": + """Open an existing egg file for reading. + + :param filename: Path to the egg file. + :returns: A :class:`Monarch3` instance in the ``eOpenToRead`` state. + :raises Monarch3Exception: If the file cannot be opened. + """ + ... + + @staticmethod + def open_for_writing(filename: str) -> "Monarch3": + """Create or overwrite an egg file for writing. + + :param filename: Path to the egg file. + :returns: A :class:`Monarch3` instance in the ``eOpenToWrite`` state. + :raises Monarch3Exception: If the file cannot be created. + """ + ... + + def __enter__(self) -> "Monarch3": + """Enter the context manager; returns ``self``.""" + ... + + def __exit__(self, exc_type, exc_val, exc_tb) -> bool: + """Exit the context manager. + + Calls :meth:`finish_reading` or :meth:`finish_writing` as appropriate. + Does not suppress exceptions (always returns ``False``). + """ + ... + + @property + def state(self) -> int: + """Current state of the file handle.""" + ... + + # ------------------------------------------------------------------ + # Reading interface + # ------------------------------------------------------------------ + + def read_header(self) -> None: + """Read header information from the file. + + Must be called after :meth:`open_for_reading` and before accessing + streams or header data. + + :raises Monarch3Exception: On I/O error. + """ + ... + + def get_header(self) -> M3Header: + """Return the file header. + + When called on a file opened for reading, the header is read-only + (modifying it has no effect on the file). When called on a file + opened for writing, the returned object is mutable and should be + configured before calling :meth:`write_header`. + + :returns: The :class:`M3Header` for this file. + """ + ... + + def get_stream(self, stream: int) -> M3Stream: + """Return the stream object for the given stream index. + + :param stream: Zero-based stream index. + :returns: The :class:`M3Stream` for the requested stream. + """ + ... + + def finish_reading(self) -> None: + """Close the file after reading. + + Not needed when using the ``with`` statement. + """ + ... + + # ------------------------------------------------------------------ + # Writing interface + # ------------------------------------------------------------------ + + def write_header(self) -> None: + """Write the header to the file and prepare streams for writing. + + Must be called after configuring the header (via :meth:`get_header`) + and adding all streams (via :meth:`M3Header.add_stream`). + + :raises Monarch3Exception: On I/O error. + """ + ... + + def finish_writing(self) -> None: + """Flush and close the file after writing. + + Not needed when using the ``with`` statement. + """ + ... diff --git a/Monarch2/CMakeLists.txt b/Monarch2/CMakeLists.txt index 0bd2c98..e341655 100644 --- a/Monarch2/CMakeLists.txt +++ b/Monarch2/CMakeLists.txt @@ -7,15 +7,23 @@ set( PROTO_FILES Protobuf/MonarchHeader.proto ) -protobuf_generate_cpp( Monarch_Protobuf_Sources Monarch_Protobuf_Headers ${PROTO_FILES} ) - +protobuf_generate( + LANGUAGE CPP + OUT_VAR Monarch_Protobuf_Output + PROTOS ${PROTO_FILES} +) +message( STATUS "Generating protobuf source files: ${Monarch_Protobuf_Output}" ) #################### # monarch2 library # #################### +list( GET Monarch_Protobuf_Output 0 Monarch_Protobuf_Headers ) +list( GET Monarch_Protobuf_Output 1 Monarch_Protobuf_Sources ) +include_directories( ${CMAKE_CURRENT_BINARY_DIR}/Protobuf ) + set( MONARCH2_HEADERFILES - ${Monarch_Protobuf_Sources} + ${Monarch_Protobuf_Headers} M2Exception.hh M2Header.hh M2IO.hh diff --git a/Monarch2/M2Header.cc b/Monarch2/M2Header.cc index e421d9a..3069281 100644 --- a/Monarch2/M2Header.cc +++ b/Monarch2/M2Header.cc @@ -267,24 +267,24 @@ TimeType M2Header::GetRecordTime0() const { return fProtobufHeader->voltagerange(); } -} -std::ostream& operator<<( std::ostream& out, const monarch2::M2Header& hdr ) -{ - out << "Monarch Header Content: " << "\n"; - out << "\tFilename: " << hdr.GetFilename() << "\n"; - out << "\tAcquisition Mode (# channels): " << hdr.GetAcquisitionMode() << "\n"; - out << "\tAcquisition Rate: " << hdr.GetAcquisitionRate() << " MHz\n"; - out << "\tRun Duration: " << hdr.GetRunDuration() << " ms\n"; - out << "\tRecord Size: " << hdr.GetRecordSize() << "\n"; - out << "\tTimestamp: " << hdr.GetTimestamp() << "\n"; - out << "\tDescription: " << hdr.GetDescription() << "\n"; - out << "\tRun Type: " << hdr.GetRunType() << "\n"; - out << "\tRun Source: " << hdr.GetRunSource() << "\n"; - out << "\tFormat Mode: " << hdr.GetFormatMode() << "\n"; - out << "\tData Type Size: " << hdr.GetDataTypeSize() << " bytes\n"; - out << "\tBit Depth: " << hdr.GetBitDepth() << " bits\n"; - out << "\tVoltage Min: " << hdr.GetVoltageMin() << " V\n"; - out << "\tVoltage Range: " << hdr.GetVoltageRange() << " V\n"; - return out; -} + std::ostream& operator<<( std::ostream& out, const M2Header& hdr ) + { + out << "Monarch Header Content: " << "\n"; + out << "\tFilename: " << hdr.GetFilename() << "\n"; + out << "\tAcquisition Mode (# channels): " << hdr.GetAcquisitionMode() << "\n"; + out << "\tAcquisition Rate: " << hdr.GetAcquisitionRate() << " MHz\n"; + out << "\tRun Duration: " << hdr.GetRunDuration() << " ms\n"; + out << "\tRecord Size: " << hdr.GetRecordSize() << "\n"; + out << "\tTimestamp: " << hdr.GetTimestamp() << "\n"; + out << "\tDescription: " << hdr.GetDescription() << "\n"; + out << "\tRun Type: " << hdr.GetRunType() << "\n"; + out << "\tRun Source: " << hdr.GetRunSource() << "\n"; + out << "\tFormat Mode: " << hdr.GetFormatMode() << "\n"; + out << "\tData Type Size: " << hdr.GetDataTypeSize() << " bytes\n"; + out << "\tBit Depth: " << hdr.GetBitDepth() << " bits\n"; + out << "\tVoltage Min: " << hdr.GetVoltageMin() << " V\n"; + out << "\tVoltage Range: " << hdr.GetVoltageRange() << " V\n"; + return out; + } +} // namespace monarch2 diff --git a/Monarch2/M2Header.hh b/Monarch2/M2Header.hh index ac9fce4..be3dba3 100644 --- a/Monarch2/M2Header.hh +++ b/Monarch2/M2Header.hh @@ -91,9 +91,9 @@ namespace monarch2 }; -} + // Pretty printing method + std::ostream& operator<<( std::ostream& out, const M2Header& hdr ); -// Pretty printing method -std::ostream& operator<<( std::ostream& out, const monarch2::M2Header& hdr ); +} #endif diff --git a/Monarch3/CMakeLists.txt b/Monarch3/CMakeLists.txt index bed9c3e..c535d58 100644 --- a/Monarch3/CMakeLists.txt +++ b/Monarch3/CMakeLists.txt @@ -48,3 +48,13 @@ pbuilder_install_headers( ${MONARCH3_HEADERFILES} ) # Executables add_subdirectory( Executables ) + +# Python bindings +if( Monarch_BUILD_PYTHON ) + add_subdirectory( python ) +endif( Monarch_BUILD_PYTHON ) + +# Tests +if( Monarch_ENABLE_TESTING ) + add_subdirectory( Tests ) +endif( Monarch_ENABLE_TESTING ) diff --git a/Monarch3/Executables/M3ReadTest.cc b/Monarch3/Executables/M3ReadTest.cc index 443be5e..5fb7ee2 100644 --- a/Monarch3/Executables/M3ReadTest.cc +++ b/Monarch3/Executables/M3ReadTest.cc @@ -112,35 +112,35 @@ int main( const int argc, const char** argv ) return RETURN_ERROR; } - LINFO( mlog, "Skip to the third record, crossing to the next acquisition (record 2; acquisition 1)" ); - if( ! ReadRecordCheck( tStream1, 1, tStrHeader1.GetDataFormat() ) ) + LINFO( mlog, "Skip to the second record in acquisition 1 (record 2 in file), crossing to the next acquisition" ); + if( ! tStream1->ReadRecord( 1, false ) ) { - LERROR( mlog, "Failed read record check" ); + LERROR( mlog, "Failed to read record" ); return RETURN_ERROR; } - LINFO( mlog, "Reread the third record (record 2; acquisition 1)" ); + LINFO( mlog, "Reread the current record (record 2 in file; record 1 in acquisition 1)" ); if( ! ReadRecordCheck( tStream1, -1, tStrHeader1.GetDataFormat() ) ) { LERROR( mlog, "Failed read record check" ); return RETURN_ERROR; } - LINFO( mlog, "Go backwards to the second record (record 1; acquisition 1)" ); + LINFO( mlog, "Go backwards to the first record in acquisition 1 (record 1 in file)" ); if( ! ReadRecordCheck( tStream1, -2, tStrHeader1.GetDataFormat() ) ) { LERROR( mlog, "Failed read record check" ); return RETURN_ERROR; } - LINFO( mlog, "Go backwards to the first record (record 1; acquisition 0)" ); + LINFO( mlog, "Go backwards to acquisition 0 (record 0 in file)" ); if( ! ReadRecordCheck( tStream1, -2, tStrHeader1.GetDataFormat() ) ) { LERROR( mlog, "Failed read record check" ); return RETURN_ERROR; } - LINFO( mlog, "Reread the first record (record 1; acquisition 0)" ); + LINFO( mlog, "Reread the first record (record 0 in file; acquisition 0)" ); if( ! ReadRecordCheck( tStream1, -1, tStrHeader1.GetDataFormat() ) ) { LERROR( mlog, "Failed read record check" ); @@ -177,9 +177,9 @@ int main( const int argc, const char** argv ) } LINFO( mlog, "Skipping immediately to the second record (record 1)" ); - if( ! ReadRecordCheck( tStream2, 1, tStrHeader2.GetDataFormat() ) ) + if( ! tStream2->ReadRecord( 1, false ) ) { - LERROR( mlog, "Failed read record check" ); + LERROR( mlog, "Failed to read record" ); return RETURN_ERROR; } diff --git a/Monarch3/Tests/CMakeLists.txt b/Monarch3/Tests/CMakeLists.txt new file mode 100644 index 0000000..3b90ebd --- /dev/null +++ b/Monarch3/Tests/CMakeLists.txt @@ -0,0 +1 @@ +add_subdirectory( Validation ) diff --git a/Monarch3/Tests/Validation/CMakeLists.txt b/Monarch3/Tests/Validation/CMakeLists.txt new file mode 100644 index 0000000..81df3d5 --- /dev/null +++ b/Monarch3/Tests/Validation/CMakeLists.txt @@ -0,0 +1,37 @@ +# CMakeLists.txt for Monarch3/Tests/Validation +# Author: N.S. Oblath + +if( Monarch_BUILD_PYTHON ) + + message( STATUS "Configuring Monarch3 validation tests" ) + + set( VALIDATION_TEST_FILES + conftest.py + test_cpp_write_python_read.py + test_python_write_cpp_read.py + test_python_write_python_read.py + ) + + # Copy Python test files to build tree so pytest can find the built monarch3 module + foreach( tfile ${VALIDATION_TEST_FILES} ) + configure_file( ${tfile} ${CMAKE_CURRENT_BINARY_DIR}/${tfile} COPYONLY ) + endforeach() + + # Register validation tests with CTest + # pytest is invoked in the build-tree copy so it can find monarch3.so and the executables + add_test( + NAME monarch3_validation + COMMAND ${Python3_EXECUTABLE} -m pytest ${CMAKE_CURRENT_BINARY_DIR} -v + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + ) + + # Make the built monarch3 module and test executables available at test time + set_tests_properties( monarch3_validation PROPERTIES + ENVIRONMENT "PYTHONPATH=${LIB_INSTALL_DIR}:$ENV{PYTHONPATH};PATH=${BIN_INSTALL_DIR}:$ENV{PATH}" + ) + +else( Monarch_BUILD_PYTHON ) + + message( STATUS "Skipping Monarch3 Python validation tests (Monarch_BUILD_PYTHON is OFF)" ) + +endif( Monarch_BUILD_PYTHON ) diff --git a/Monarch3/Tests/Validation/conftest.py b/Monarch3/Tests/Validation/conftest.py new file mode 100644 index 0000000..3056ead --- /dev/null +++ b/Monarch3/Tests/Validation/conftest.py @@ -0,0 +1,78 @@ +# conftest.py +# Shared fixtures and constants for Monarch3 validation tests. +# All data constants match the exact values written by the M3WriteTest C++ executable. + +import pytest +import subprocess + +# ---- Gold-standard data constants from M3WriteTest ---- + +# Header +HEADER_RUN_DURATION = 8675309 +HEADER_TIMESTAMP = "Stardate 33515" +HEADER_DESCRIPTION = "Bigger on the inside" + +# Stream 0: 1 channel, uint8, 10 samples/record, 1 acquisition, 2 records +STREAM0_N_CHANNELS = 1 +STREAM0_N_ACQUISITIONS = 1 +STREAM0_N_RECORDS = 2 +STREAM0_REC_SIZE = 10 +STREAM0_VALUES = [1, 10] # one uniform value per record + +# Stream 1: 2 channels, uint16, interleaved, 5 samples/record, 2 acquisitions, 3 records +# acq 0: rec 0 (WriteRecord true) +# acq 1: rec 1 (WriteRecord true), rec 2 (WriteRecord false) +STREAM1_N_CHANNELS = 2 +STREAM1_N_ACQUISITIONS = 2 +STREAM1_N_RECORDS = 3 +STREAM1_REC_SIZE = 5 +STREAM1_VALUES = [ + (1, 2), # rec 0: ch0=1, ch1=2 + (1000, 2000), # rec 1: ch0=1000, ch1=2000 + (10000, 20000), # rec 2: ch0=10000, ch1=20000 +] + +# Stream 2: 3 channels, uint8, separate, 5 samples/record, 1 acquisition, 2 records +STREAM2_N_CHANNELS = 3 +STREAM2_N_ACQUISITIONS = 1 +STREAM2_N_RECORDS = 2 +STREAM2_REC_SIZE = 5 +STREAM2_VALUES = [ + (1, 2, 3), # rec 0: ch0=1, ch1=2, ch2=3 + (10, 20, 30), # rec 1: ch0=10, ch1=20, ch2=30 +] + +# Stream 3: 1 channel, float32, 10 samples/record, 2 acquisitions, 2 records (one each) +STREAM3_N_CHANNELS = 1 +STREAM3_N_ACQUISITIONS = 2 +STREAM3_N_RECORDS = 2 +STREAM3_REC_SIZE = 10 +STREAM3_VALUES = [3.1415926535898, 2.71828182846] # pi, e + +# ---- Fixtures ---- + +@pytest.fixture +def tmp_egg(tmp_path): + """Provide a temporary .egg file path, cleaned up automatically after the test.""" + return str(tmp_path / "test.egg") + + +@pytest.fixture(scope="session") +def cpp_written_egg(tmp_path_factory): + """Run M3WriteTest once per session to produce the gold-standard egg file. + + Session scope avoids spawning multiple concurrent M3WriteTest processes (which + can be killed by macOS resource throttling when pytest runs tests back-to-back). + """ + path = str(tmp_path_factory.mktemp("cpp_written") / "cpp_written.egg") + result = subprocess.run( + ["M3WriteTest", path], + capture_output=True, + text=True, + ) + assert result.returncode == 0, ( + f"M3WriteTest failed with return code {result.returncode}:\n" + f"stdout: {result.stdout}\n" + f"stderr: {result.stderr}" + ) + return path diff --git a/Monarch3/Tests/Validation/test_cpp_write_python_read.py b/Monarch3/Tests/Validation/test_cpp_write_python_read.py new file mode 100644 index 0000000..6fcce6a --- /dev/null +++ b/Monarch3/Tests/Validation/test_cpp_write_python_read.py @@ -0,0 +1,183 @@ +# test_cpp_write_python_read.py +# Validates that the Python binding can correctly read a file produced by the C++ +# M3WriteTest executable. The structure and values checked here mirror M3ReadTest.cc. + +import numpy as np +import pytest +import monarch3 + +from conftest import ( + HEADER_RUN_DURATION, HEADER_TIMESTAMP, HEADER_DESCRIPTION, + STREAM0_N_CHANNELS, STREAM0_N_ACQUISITIONS, STREAM0_N_RECORDS, STREAM0_REC_SIZE, STREAM0_VALUES, + STREAM1_N_CHANNELS, STREAM1_N_ACQUISITIONS, STREAM1_N_RECORDS, STREAM1_REC_SIZE, STREAM1_VALUES, + STREAM2_N_CHANNELS, STREAM2_N_ACQUISITIONS, STREAM2_N_RECORDS, STREAM2_REC_SIZE, STREAM2_VALUES, + STREAM3_N_CHANNELS, STREAM3_N_ACQUISITIONS, STREAM3_N_RECORDS, STREAM3_REC_SIZE, STREAM3_VALUES, +) + + +# ---- Header ---- + +def test_header(cpp_written_egg): + with monarch3.Monarch3.open_for_reading(cpp_written_egg) as m: + m.read_header() + hdr = m.get_header() + assert hdr.run_duration == HEADER_RUN_DURATION + assert hdr.timestamp == HEADER_TIMESTAMP + assert hdr.description == HEADER_DESCRIPTION + assert hdr.n_streams == 4 + assert hdr.n_channels == 7 # 1 + 2 + 3 + 1 + + +# ---- Stream 0: single-channel uint8 ---- + +def test_stream0_structure(cpp_written_egg): + with monarch3.Monarch3.open_for_reading(cpp_written_egg) as m: + m.read_header() + stream = m.get_stream(0) + assert stream.n_channels == STREAM0_N_CHANNELS + assert stream.n_acquisitions == STREAM0_N_ACQUISITIONS + assert stream.n_records_in_file == STREAM0_N_RECORDS + assert stream.channel_record_size == STREAM0_REC_SIZE + + +def test_stream0_data(cpp_written_egg): + with monarch3.Monarch3.open_for_reading(cpp_written_egg) as m: + m.read_header() + stream = m.get_stream(0) + for expected_val in STREAM0_VALUES: + assert stream.read_record(), "Unexpected end of stream 0" + arr = stream.get_channel_data(0).view(np.uint8) + assert np.all(arr == expected_val), \ + f"Stream 0: expected all {expected_val}, got {arr}" + assert not stream.read_record(), "Stream 0 should be exhausted" + + +# ---- Stream 1: two-channel uint16, interleaved, offset navigation ---- + +def test_stream1_structure(cpp_written_egg): + with monarch3.Monarch3.open_for_reading(cpp_written_egg) as m: + m.read_header() + stream = m.get_stream(1) + assert stream.n_channels == STREAM1_N_CHANNELS + assert stream.n_acquisitions == STREAM1_N_ACQUISITIONS + assert stream.n_records_in_file == STREAM1_N_RECORDS + assert stream.channel_record_size == STREAM1_REC_SIZE + + +def test_stream1_data_sequential(cpp_written_egg): + with monarch3.Monarch3.open_for_reading(cpp_written_egg) as m: + m.read_header() + stream = m.get_stream(1) + for v0, v1 in STREAM1_VALUES: + assert stream.read_record(), "Unexpected end of stream 1" + ch0 = stream.get_channel_data(0).view(np.uint16) + ch1 = stream.get_channel_data(1).view(np.uint16) + assert np.all(ch0 == v0), f"Stream 1 ch0: expected {v0}, got {ch0}" + assert np.all(ch1 == v1), f"Stream 1 ch1: expected {v1}, got {ch1}" + + +def test_stream1_offset_navigation(cpp_written_egg): + """Verify seeking with non-zero offsets (mirrors M3ReadTest Test 2). + + Key: ReadRecord uses aIfNewAcqStartAtFirstRec=True by default, which snaps to the + first record of any new acquisition entered. Pass False to land exactly on the + offset-targeted record when crossing an acquisition boundary. + """ + with monarch3.Monarch3.open_for_reading(cpp_written_egg) as m: + m.read_header() + stream = m.get_stream(1) + + # Read record 0 (offset=0 from start); acq 0, rec 0 + assert stream.read_record(0), "Expected record 0" + ch0 = stream.get_channel_data(0).view(np.uint16) + assert np.all(ch0 == STREAM1_VALUES[0][0]), f"Expected {STREAM1_VALUES[0][0]}, got {ch0}" + + # Skip forward with offset=1 crossing to acquisition 1. + # aIfNewAcqStartAtFirstRec=False: land on the exact target (file rec 2, acq 1 rec 1, + # values 10000/20000) rather than snapping back to the start of acq 1. + assert stream.read_record(1, False), "Expected record 2" + ch0 = stream.get_channel_data(0).view(np.uint16) + assert np.all(ch0 == STREAM1_VALUES[2][0]), f"Expected {STREAM1_VALUES[2][0]}, got {ch0}" + + # Reread record 2 (offset=-1) + assert stream.read_record(-1), "Expected reread of record 2" + ch0 = stream.get_channel_data(0).view(np.uint16) + assert np.all(ch0 == STREAM1_VALUES[2][0]), f"Expected {STREAM1_VALUES[2][0]}, got {ch0}" + + # Step back to record 1 (offset=-2) + assert stream.read_record(-2), "Expected record 1" + ch0 = stream.get_channel_data(0).view(np.uint16) + assert np.all(ch0 == STREAM1_VALUES[1][0]), f"Expected {STREAM1_VALUES[1][0]}, got {ch0}" + + # Request past end of file + assert not stream.read_record(5), "Expected False for out-of-bounds forward seek" + + # Request before beginning of file + assert not stream.read_record(-5), "Expected False for out-of-bounds backward seek" + + +# ---- Stream 2: three-channel uint8, separate ---- + +def test_stream2_structure(cpp_written_egg): + with monarch3.Monarch3.open_for_reading(cpp_written_egg) as m: + m.read_header() + stream = m.get_stream(2) + assert stream.n_channels == STREAM2_N_CHANNELS + assert stream.n_acquisitions == STREAM2_N_ACQUISITIONS + assert stream.n_records_in_file == STREAM2_N_RECORDS + assert stream.channel_record_size == STREAM2_REC_SIZE + + +def test_stream2_data(cpp_written_egg): + with monarch3.Monarch3.open_for_reading(cpp_written_egg) as m: + m.read_header() + stream = m.get_stream(2) + for v0, v1, v2 in STREAM2_VALUES: + assert stream.read_record(), "Unexpected end of stream 2" + assert np.all(stream.get_channel_data(0).view(np.uint8) == v0) + assert np.all(stream.get_channel_data(1).view(np.uint8) == v1) + assert np.all(stream.get_channel_data(2).view(np.uint8) == v2) + + +def test_stream2_skip_to_second_record(cpp_written_egg): + """Skip directly to record 1 using offset=1 (mirrors M3ReadTest Test 3). + + Stream 2 has only 1 acquisition, so aIfNewAcqStartAtFirstRec does not matter here; + the snap-to-first-in-acq only fires when entering a *new* acquisition. However, + the very first ReadRecord call always treats the stream as entering a new acquisition, + so we must pass False to land on the exact offset target (rec 1) rather than rec 0. + """ + with monarch3.Monarch3.open_for_reading(cpp_written_egg) as m: + m.read_header() + stream = m.get_stream(2) + assert stream.read_record(1, False), "Expected record 1 via skip" + v0, v1, v2 = STREAM2_VALUES[1] + assert np.all(stream.get_channel_data(0).view(np.uint8) == v0) + assert np.all(stream.get_channel_data(1).view(np.uint8) == v1) + assert np.all(stream.get_channel_data(2).view(np.uint8) == v2) + + # Verify that a backward seek past the beginning returns False + assert not stream.read_record(-3), "Expected False for out-of-bounds backward seek" + + +# ---- Stream 3: single-channel float32 ---- + +def test_stream3_structure(cpp_written_egg): + with monarch3.Monarch3.open_for_reading(cpp_written_egg) as m: + m.read_header() + stream = m.get_stream(3) + assert stream.n_channels == STREAM3_N_CHANNELS + assert stream.n_acquisitions == STREAM3_N_ACQUISITIONS + assert stream.n_records_in_file == STREAM3_N_RECORDS + assert stream.channel_record_size == STREAM3_REC_SIZE + + +def test_stream3_data(cpp_written_egg): + with monarch3.Monarch3.open_for_reading(cpp_written_egg) as m: + m.read_header() + stream = m.get_stream(3) + for expected_val in STREAM3_VALUES: + assert stream.read_record(), "Unexpected end of stream 3" + arr = stream.get_channel_data(0).view(np.float32) + assert np.all(np.isclose(arr, expected_val, rtol=1e-6)), \ + f"Stream 3: expected ~{expected_val}, got {arr}" diff --git a/Monarch3/Tests/Validation/test_python_write_cpp_read.py b/Monarch3/Tests/Validation/test_python_write_cpp_read.py new file mode 100644 index 0000000..1243a29 --- /dev/null +++ b/Monarch3/Tests/Validation/test_python_write_cpp_read.py @@ -0,0 +1,100 @@ +# test_python_write_cpp_read.py +# Validates that a file written by the Python binding is correctly read by the C++ +# M3ReadTest executable. The Python write reproduces the same 4-stream structure as +# M3WriteTest so that M3ReadTest's built-in checks pass without modification. + +import numpy as np +import subprocess +import monarch3 + +from conftest import ( + HEADER_RUN_DURATION, HEADER_TIMESTAMP, HEADER_DESCRIPTION, + STREAM0_REC_SIZE, STREAM0_VALUES, + STREAM1_REC_SIZE, STREAM1_VALUES, + STREAM2_REC_SIZE, STREAM2_VALUES, + STREAM3_REC_SIZE, STREAM3_VALUES, +) + + +def write_standard_egg(path): + """Write the same 4-stream structure as M3WriteTest using the Python binding. + + This is also imported by test_python_write_python_read.py to avoid + duplicating the write logic. + """ + with monarch3.Monarch3.open_for_writing(path) as m: + hdr = m.get_header() + hdr.filename = path + hdr.run_duration = HEADER_RUN_DURATION + hdr.timestamp = HEADER_TIMESTAMP + hdr.description = HEADER_DESCRIPTION + + s0 = hdr.add_stream( + "1-channel device", + 500, STREAM0_REC_SIZE, 1, 1, + monarch3.sDigitizedUS, 8, monarch3.sBitsAlignedLeft, + ) + s1 = hdr.add_stream( + "2-channel device", + 2, monarch3.sInterleaved, + 250, STREAM1_REC_SIZE, 1, 2, + monarch3.sDigitizedUS, 16, monarch3.sBitsAlignedLeft, + ) + s2 = hdr.add_stream( + "3-channel device", + 3, monarch3.sSeparate, + 100, STREAM2_REC_SIZE, 1, 1, + monarch3.sDigitizedUS, 8, monarch3.sBitsAlignedLeft, + ) + s3 = hdr.add_stream( + "Floating-point device", + 100, STREAM3_REC_SIZE, 1, 4, + monarch3.sAnalog, 8, monarch3.sBitsAlignedLeft, + ) + m.write_header() + + # Stream 0: 2 records in 1 acquisition + stream = m.get_stream(s0) + for i, val in enumerate(STREAM0_VALUES): + stream.get_channel_data(0)[:] = val + stream.write_record(i == 0) + + # Stream 1: 3 records in 2 acquisitions + # acq 0: rec 0 (is_new=True) + # acq 1: rec 1 (is_new=True), rec 2 (is_new=False) + stream = m.get_stream(s1) + for i, (v0, v1) in enumerate(STREAM1_VALUES): + stream.get_channel_data(0).view(np.uint16)[:] = v0 + stream.get_channel_data(1).view(np.uint16)[:] = v1 + is_new = (i == 0 or i == 1) + stream.write_record(is_new) + + # Stream 2: 2 records in 1 acquisition + stream = m.get_stream(s2) + for i, (v0, v1, v2) in enumerate(STREAM2_VALUES): + stream.get_channel_data(0)[:] = v0 + stream.get_channel_data(1)[:] = v1 + stream.get_channel_data(2)[:] = v2 + stream.write_record(i == 0) + + # Stream 3: 2 records, each in its own acquisition + stream = m.get_stream(s3) + for val in STREAM3_VALUES: + stream.get_channel_data(0).view(np.float32)[:] = val + stream.write_record(True) + + +def test_python_write_cpp_read(tmp_egg): + """Write the standard 4-stream file from Python, then validate with M3ReadTest.""" + write_standard_egg(tmp_egg) + + result = subprocess.run( + ["M3ReadTest", tmp_egg], + capture_output=True, + text=True, + ) + assert result.returncode == 0, ( + f"M3ReadTest failed on Python-written file (return code {result.returncode}):\n" + f"stdout:\n{result.stdout}\n" + f"stderr:\n{result.stderr}" + ) diff --git a/Monarch3/Tests/Validation/test_python_write_python_read.py b/Monarch3/Tests/Validation/test_python_write_python_read.py new file mode 100644 index 0000000..d7ac312 --- /dev/null +++ b/Monarch3/Tests/Validation/test_python_write_python_read.py @@ -0,0 +1,91 @@ +# test_python_write_python_read.py +# Pure Python round-trip: write with the Python binding, read back with the Python +# binding. Isolates the binding from the C++ executables. + +import numpy as np +import monarch3 + +from conftest import ( + HEADER_RUN_DURATION, HEADER_DESCRIPTION, + STREAM0_N_CHANNELS, STREAM0_N_ACQUISITIONS, STREAM0_N_RECORDS, STREAM0_REC_SIZE, STREAM0_VALUES, + STREAM1_N_CHANNELS, STREAM1_N_ACQUISITIONS, STREAM1_N_RECORDS, STREAM1_REC_SIZE, STREAM1_VALUES, + STREAM2_N_CHANNELS, STREAM2_N_ACQUISITIONS, STREAM2_N_RECORDS, STREAM2_REC_SIZE, STREAM2_VALUES, + STREAM3_N_CHANNELS, STREAM3_N_ACQUISITIONS, STREAM3_N_RECORDS, STREAM3_REC_SIZE, STREAM3_VALUES, +) +from test_python_write_cpp_read import write_standard_egg + + +def test_header_round_trip(tmp_egg): + write_standard_egg(tmp_egg) + with monarch3.Monarch3.open_for_reading(tmp_egg) as m: + m.read_header() + hdr = m.get_header() + assert hdr.run_duration == HEADER_RUN_DURATION + assert hdr.description == HEADER_DESCRIPTION + assert hdr.n_streams == 4 + + +def test_stream0_round_trip(tmp_egg): + write_standard_egg(tmp_egg) + with monarch3.Monarch3.open_for_reading(tmp_egg) as m: + m.read_header() + stream = m.get_stream(0) + assert stream.n_channels == STREAM0_N_CHANNELS + assert stream.n_acquisitions == STREAM0_N_ACQUISITIONS + assert stream.n_records_in_file == STREAM0_N_RECORDS + assert stream.channel_record_size == STREAM0_REC_SIZE + for expected_val in STREAM0_VALUES: + assert stream.read_record(), "Unexpected end of stream 0" + arr = stream.get_channel_data(0).view(np.uint8) + assert np.all(arr == expected_val), \ + f"Stream 0: expected all {expected_val}, got {arr}" + assert not stream.read_record(), "Stream 0 should be exhausted" + + +def test_stream1_round_trip(tmp_egg): + write_standard_egg(tmp_egg) + with monarch3.Monarch3.open_for_reading(tmp_egg) as m: + m.read_header() + stream = m.get_stream(1) + assert stream.n_channels == STREAM1_N_CHANNELS + assert stream.n_acquisitions == STREAM1_N_ACQUISITIONS + assert stream.n_records_in_file == STREAM1_N_RECORDS + assert stream.channel_record_size == STREAM1_REC_SIZE + for v0, v1 in STREAM1_VALUES: + assert stream.read_record(), "Unexpected end of stream 1" + ch0 = stream.get_channel_data(0).view(np.uint16) + ch1 = stream.get_channel_data(1).view(np.uint16) + assert np.all(ch0 == v0), f"Stream 1 ch0: expected {v0}, got {ch0}" + assert np.all(ch1 == v1), f"Stream 1 ch1: expected {v1}, got {ch1}" + + +def test_stream2_round_trip(tmp_egg): + write_standard_egg(tmp_egg) + with monarch3.Monarch3.open_for_reading(tmp_egg) as m: + m.read_header() + stream = m.get_stream(2) + assert stream.n_channels == STREAM2_N_CHANNELS + assert stream.n_acquisitions == STREAM2_N_ACQUISITIONS + assert stream.n_records_in_file == STREAM2_N_RECORDS + assert stream.channel_record_size == STREAM2_REC_SIZE + for v0, v1, v2 in STREAM2_VALUES: + assert stream.read_record(), "Unexpected end of stream 2" + assert np.all(stream.get_channel_data(0).view(np.uint8) == v0) + assert np.all(stream.get_channel_data(1).view(np.uint8) == v1) + assert np.all(stream.get_channel_data(2).view(np.uint8) == v2) + + +def test_stream3_round_trip(tmp_egg): + write_standard_egg(tmp_egg) + with monarch3.Monarch3.open_for_reading(tmp_egg) as m: + m.read_header() + stream = m.get_stream(3) + assert stream.n_channels == STREAM3_N_CHANNELS + assert stream.n_acquisitions == STREAM3_N_ACQUISITIONS + assert stream.n_records_in_file == STREAM3_N_RECORDS + assert stream.channel_record_size == STREAM3_REC_SIZE + for expected_val in STREAM3_VALUES: + assert stream.read_record(), "Unexpected end of stream 3" + arr = stream.get_channel_data(0).view(np.float32) + assert np.all(np.isclose(arr, expected_val, rtol=1e-6)), \ + f"Stream 3: expected ~{expected_val}, got {arr}" diff --git a/Monarch3/python/CMakeLists.txt b/Monarch3/python/CMakeLists.txt new file mode 100644 index 0000000..554900e --- /dev/null +++ b/Monarch3/python/CMakeLists.txt @@ -0,0 +1,37 @@ +# CMakeLists.txt for Monarch3/python +# Author: N.S. Oblath + +message( "Building Monarch3 Python bindings" ) + +set( dir ${CMAKE_CURRENT_SOURCE_DIR} ) + +set( PYBINDING_HEADERFILES + ${dir}/monarch3_binding_helpers.hh + ${dir}/m3constants_pybind.hh + ${dir}/m3header_pybind.hh + ${dir}/m3record_pybind.hh + ${dir}/m3stream_pybind.hh + ${dir}/m3monarch_pybind.hh +) + +set( PYBINDING_SOURCEFILES + ${dir}/monarch3_namespace_pybind.cc +) + +set( PYBINDING_PROJECT_LIBRARIES Monarch3 ) + +pbuilder_add_pybind11_module( + MODULE_NAME monarch3 + SOURCE_FILES ${PYBINDING_SOURCEFILES} + PROJECT_LIBRARIES ${PYBINDING_PROJECT_LIBRARIES} +) + +pbuilder_install_headers( ${PYBINDING_HEADERFILES} ) + +if( NOT DEFINED PBUILDER_PY_INSTALL_IN_SITELIB ) + message( STATUS "Installing add_lib_python_path.sh since python modules will not be installed in the site-package directory" ) + configure_file( add_lib_python_path.sh.in add_lib_python_path.sh @ONLY ) + install( FILES ${CMAKE_CURRENT_BINARY_DIR}/add_lib_python_path.sh DESTINATION ${BIN_INSTALL_DIR} ) +endif( NOT DEFINED PBUILDER_PY_INSTALL_IN_SITELIB ) + +message( "Done with Monarch3 Python bindings" ) diff --git a/Monarch3/python/add_lib_python_path.sh.in b/Monarch3/python/add_lib_python_path.sh.in new file mode 100644 index 0000000..68aff4b --- /dev/null +++ b/Monarch3/python/add_lib_python_path.sh.in @@ -0,0 +1,3 @@ +# Adds the lib install directory to the python path + +export PYTHONPATH=@LIB_INSTALL_DIR@:$PYTHONPATH diff --git a/Monarch3/python/m3constants_pybind.hh b/Monarch3/python/m3constants_pybind.hh new file mode 100644 index 0000000..566fb1d --- /dev/null +++ b/Monarch3/python/m3constants_pybind.hh @@ -0,0 +1,44 @@ +#ifndef M3CONSTANTS_PYBIND_HH_ +#define M3CONSTANTS_PYBIND_HH_ + +#include "M3Constants.hh" + +#include "pybind11/pybind11.h" + +namespace monarch3_pybind +{ + + std::list< std::string > export_constants( pybind11::module& mod ) + { + std::list< std::string > all_items; + + // Data format constants + all_items.push_back( "sDigitizedUS" ); + mod.attr( "sDigitizedUS" ) = monarch3::sDigitizedUS; + + all_items.push_back( "sDigitizedS" ); + mod.attr( "sDigitizedS" ) = monarch3::sDigitizedS; + + all_items.push_back( "sAnalog" ); + mod.attr( "sAnalog" ) = monarch3::sAnalog; + + // Bit alignment constants + all_items.push_back( "sBitsAlignedLeft" ); + mod.attr( "sBitsAlignedLeft" ) = monarch3::sBitsAlignedLeft; + + all_items.push_back( "sBitsAlignedRight" ); + mod.attr( "sBitsAlignedRight" ) = monarch3::sBitsAlignedRight; + + // Channel format constants + all_items.push_back( "sInterleaved" ); + mod.attr( "sInterleaved" ) = monarch3::sInterleaved; + + all_items.push_back( "sSeparate" ); + mod.attr( "sSeparate" ) = monarch3::sSeparate; + + return all_items; + } + +} /* namespace monarch3_pybind */ + +#endif /* M3CONSTANTS_PYBIND_HH_ */ diff --git a/Monarch3/python/m3header_pybind.hh b/Monarch3/python/m3header_pybind.hh new file mode 100644 index 0000000..d881675 --- /dev/null +++ b/Monarch3/python/m3header_pybind.hh @@ -0,0 +1,166 @@ +#ifndef M3HEADER_PYBIND_HH_ +#define M3HEADER_PYBIND_HH_ + +#include "M3Header.hh" + +#include "pybind11/pybind11.h" +#include "pybind11/stl.h" + +#include + +namespace monarch3_pybind +{ + + std::list< std::string > export_header( pybind11::module& mod ) + { + std::list< std::string > all_items; + + // M3StreamHeader + all_items.push_back( "M3StreamHeader" ); + pybind11::class_< monarch3::M3StreamHeader >( mod, "M3StreamHeader", + "Header information for a single data stream" ) + .def_property_readonly( "number", &monarch3::M3StreamHeader::GetNumber ) + .def_property_readonly( "source", []( const monarch3::M3StreamHeader& h ) { return h.Source(); } ) + .def_property_readonly( "n_channels", &monarch3::M3StreamHeader::GetNChannels ) + .def_property_readonly( "channels", []( const monarch3::M3StreamHeader& h ) { return h.Channels(); } ) + .def_property_readonly( "channel_format", &monarch3::M3StreamHeader::GetChannelFormat ) + .def_property_readonly( "acquisition_rate",&monarch3::M3StreamHeader::GetAcquisitionRate ) + .def_property_readonly( "record_size", &monarch3::M3StreamHeader::GetRecordSize ) + .def_property_readonly( "sample_size", &monarch3::M3StreamHeader::GetSampleSize ) + .def_property_readonly( "data_type_size", &monarch3::M3StreamHeader::GetDataTypeSize ) + .def_property_readonly( "data_format", &monarch3::M3StreamHeader::GetDataFormat ) + .def_property_readonly( "bit_depth", &monarch3::M3StreamHeader::GetBitDepth ) + .def_property_readonly( "bit_alignment", &monarch3::M3StreamHeader::GetBitAlignment ) + .def_property_readonly( "n_acquisitions", &monarch3::M3StreamHeader::GetNAcquisitions ) + .def_property_readonly( "n_records", &monarch3::M3StreamHeader::GetNRecords ) + .def( "__repr__", []( const monarch3::M3StreamHeader& h ) { + std::ostringstream out; + out << h; + return out.str(); + } ) + ; + + // M3ChannelHeader + all_items.push_back( "M3ChannelHeader" ); + pybind11::class_< monarch3::M3ChannelHeader >( mod, "M3ChannelHeader", + "Header information for a single data channel" ) + .def_property_readonly( "number", &monarch3::M3ChannelHeader::GetNumber ) + .def_property_readonly( "source", []( const monarch3::M3ChannelHeader& h ) { return h.Source(); } ) + .def_property_readonly( "acquisition_rate",&monarch3::M3ChannelHeader::GetAcquisitionRate ) + .def_property_readonly( "record_size", &monarch3::M3ChannelHeader::GetRecordSize ) + .def_property_readonly( "sample_size", &monarch3::M3ChannelHeader::GetSampleSize ) + .def_property_readonly( "data_type_size", &monarch3::M3ChannelHeader::GetDataTypeSize ) + .def_property_readonly( "data_format", &monarch3::M3ChannelHeader::GetDataFormat ) + .def_property_readonly( "bit_depth", &monarch3::M3ChannelHeader::GetBitDepth ) + .def_property_readonly( "bit_alignment", &monarch3::M3ChannelHeader::GetBitAlignment ) + .def_property_readonly( "voltage_offset", &monarch3::M3ChannelHeader::GetVoltageOffset ) + .def_property_readonly( "voltage_range", &monarch3::M3ChannelHeader::GetVoltageRange ) + .def_property_readonly( "dac_gain", &monarch3::M3ChannelHeader::GetDACGain ) + .def_property_readonly( "frequency_min", &monarch3::M3ChannelHeader::GetFrequencyMin ) + .def_property_readonly( "frequency_range", &monarch3::M3ChannelHeader::GetFrequencyRange ) + .def( "__repr__", []( const monarch3::M3ChannelHeader& h ) { + std::ostringstream out; + out << h; + return out.str(); + } ) + ; + + // M3Header + all_items.push_back( "M3Header" ); + pybind11::class_< monarch3::M3Header >( mod, "M3Header", + "Egg file header: run metadata and stream/channel configuration" ) + // Read-only properties + .def_property_readonly( "egg_version", []( const monarch3::M3Header& h ) { return h.EggVersion(); } ) + .def_property_readonly( "n_channels", &monarch3::M3Header::GetNChannels ) + .def_property_readonly( "n_streams", &monarch3::M3Header::GetNStreams ) + .def_property_readonly( "stream_headers", + []( const monarch3::M3Header& h ) { return h.StreamHeaders(); } ) + .def_property_readonly( "channel_headers", + []( const monarch3::M3Header& h ) { return h.ChannelHeaders(); } ) + // Read-write properties (ref-accessor pattern) + .def_property( "filename", + []( const monarch3::M3Header& h ) { return h.Filename(); }, + []( monarch3::M3Header& h, const std::string& v ) { h.Filename() = v; } ) + .def_property( "run_duration", + &monarch3::M3Header::GetRunDuration, + &monarch3::M3Header::SetRunDuration ) + .def_property( "timestamp", + []( const monarch3::M3Header& h ) { return h.Timestamp(); }, + []( monarch3::M3Header& h, const std::string& v ) { h.Timestamp() = v; } ) + .def_property( "description", + []( const monarch3::M3Header& h ) { return h.Description(); }, + []( monarch3::M3Header& h, const std::string& v ) { h.Description() = v; } ) + // AddStream overloads — lambda wrappers are needed because pybind11 cannot + // implicitly convert Python None to a raw std::vector* pointer. + .def( "add_stream", + []( monarch3::M3Header& h, + const std::string& source, + uint32_t acq_rate, uint32_t rec_size, uint32_t sample_size, + uint32_t data_type_size, uint32_t data_format, + uint32_t bit_depth, uint32_t bit_alignment, + pybind11::object chan_vec_py ) -> unsigned + { + std::vector< unsigned > buf; + std::vector< unsigned >* ptr = nullptr; + if( ! chan_vec_py.is_none() ) + { + buf = chan_vec_py.cast< std::vector< unsigned > >(); + ptr = &buf; + } + return h.AddStream( source, acq_rate, rec_size, sample_size, + data_type_size, data_format, bit_depth, bit_alignment, ptr ); + }, + pybind11::arg( "source" ), + pybind11::arg( "acq_rate" ), + pybind11::arg( "rec_size" ), + pybind11::arg( "sample_size" ), + pybind11::arg( "data_type_size" ), + pybind11::arg( "data_format" ), + pybind11::arg( "bit_depth" ), + pybind11::arg( "bit_alignment" ), + pybind11::arg( "chan_vec" ) = pybind11::none(), + "Add a single-channel stream; returns the stream number" ) + .def( "add_stream", + []( monarch3::M3Header& h, + const std::string& source, + uint32_t n_channels, uint32_t channel_format, + uint32_t acq_rate, uint32_t rec_size, uint32_t sample_size, + uint32_t data_type_size, uint32_t data_format, + uint32_t bit_depth, uint32_t bit_alignment, + pybind11::object chan_vec_py ) -> unsigned + { + std::vector< unsigned > buf; + std::vector< unsigned >* ptr = nullptr; + if( ! chan_vec_py.is_none() ) + { + buf = chan_vec_py.cast< std::vector< unsigned > >(); + ptr = &buf; + } + return h.AddStream( source, n_channels, channel_format, acq_rate, rec_size, + sample_size, data_type_size, data_format, bit_depth, bit_alignment, ptr ); + }, + pybind11::arg( "source" ), + pybind11::arg( "n_channels" ), + pybind11::arg( "channel_format" ), + pybind11::arg( "acq_rate" ), + pybind11::arg( "rec_size" ), + pybind11::arg( "sample_size" ), + pybind11::arg( "data_type_size" ), + pybind11::arg( "data_format" ), + pybind11::arg( "bit_depth" ), + pybind11::arg( "bit_alignment" ), + pybind11::arg( "chan_vec" ) = pybind11::none(), + "Add a multi-channel stream; returns the stream number" ) + .def( "__repr__", []( const monarch3::M3Header& h ) { + std::ostringstream out; + out << h; + return out.str(); + } ) + ; + + return all_items; + } + +} /* namespace monarch3_pybind */ + +#endif /* M3HEADER_PYBIND_HH_ */ diff --git a/Monarch3/python/m3monarch_pybind.hh b/Monarch3/python/m3monarch_pybind.hh new file mode 100644 index 0000000..20eadc5 --- /dev/null +++ b/Monarch3/python/m3monarch_pybind.hh @@ -0,0 +1,146 @@ +#ifndef M3MONARCH_PYBIND_HH_ +#define M3MONARCH_PYBIND_HH_ + +#include "M3Monarch.hh" + +#include "monarch3_binding_helpers.hh" + +#include "pybind11/pybind11.h" + +#include + +namespace monarch3_pybind +{ + + // Custom deleter: call FinishReading/FinishWriting before deleting so HDF5 file is + // closed properly even when Python's garbage collector destroys the object. + struct Monarch3Deleter + { + void operator()( monarch3::Monarch3* m ) const + { + if( m == nullptr ) return; + auto state = m->GetState(); + if( state == monarch3::Monarch3::eReadyToRead || state == monarch3::Monarch3::eOpenToRead ) + { + try { m->FinishReading(); } catch(...) {} + } + else if( state == monarch3::Monarch3::eReadyToWrite || state == monarch3::Monarch3::eOpenToWrite ) + { + try { m->FinishWriting(); } catch(...) {} + } + delete m; + } + }; + + // Convenience alias + using Monarch3Ptr = std::unique_ptr< monarch3::Monarch3, Monarch3Deleter >; + + + std::list< std::string > export_monarch( pybind11::module& mod ) + { + std::list< std::string > all_items; + + all_items.push_back( "Monarch3" ); + pybind11::class_< monarch3::Monarch3, Monarch3Ptr >( mod, "Monarch3", + "Top-level egg v3 file handle.\n\n" + "Use open_for_reading() or open_for_writing() to obtain an instance.\n" + "Supports the context manager protocol ('with' statement)." ) + + // ---- Factory methods ---- + .def_static( "open_for_reading", + []( const std::string& filename ) -> Monarch3Ptr { + return Monarch3Ptr( const_cast(monarch3::Monarch3::OpenForReading( filename )) ); + }, + pybind11::arg( "filename" ), + "Open an existing egg file for reading.\n" + "Returns a Monarch3 instance in the eOpenToRead state.", + MONARCH3_BIND_CALL_GUARD_STREAMS ) + .def_static( "open_for_writing", + []( const std::string& filename ) -> Monarch3Ptr { + return Monarch3Ptr( monarch3::Monarch3::OpenForWriting( filename ) ); + }, + pybind11::arg( "filename" ), + "Create or overwrite an egg file for writing.\n" + "Returns a Monarch3 instance in the eOpenToWrite state.", + MONARCH3_BIND_CALL_GUARD_STREAMS ) + + // ---- Context manager ---- + .def( "__enter__", + []( monarch3::Monarch3* self ) { return self; }, + pybind11::return_value_policy::reference, + "Enter context manager; returns self." ) + .def( "__exit__", + []( monarch3::Monarch3* self, + pybind11::object /*exc_type*/, + pybind11::object /*exc_val*/, + pybind11::object /*exc_tb*/ ) + { + auto state = self->GetState(); + if( state == monarch3::Monarch3::eReadyToRead || state == monarch3::Monarch3::eOpenToRead ) + { + self->FinishReading(); + } + else if( state == monarch3::Monarch3::eReadyToWrite || state == monarch3::Monarch3::eOpenToWrite ) + { + self->FinishWriting(); + } + return false; // do not suppress exceptions + }, + "Exit context manager; calls finish_reading() or finish_writing() as appropriate." ) + + // ---- State ---- + .def_property_readonly( "state", + &monarch3::Monarch3::GetState, + "Current state of the file handle" ) + + // ---- Reading ---- + .def( "read_header", + &monarch3::Monarch3::ReadHeader, + "Read header information from the file.\n" + "Must be called after open_for_reading() before accessing data.", + MONARCH3_BIND_CALL_GUARD_STREAMS ) + .def( "get_header", + ( const monarch3::M3Header* ( monarch3::Monarch3::* )() const ) + &monarch3::Monarch3::GetHeader, + pybind11::return_value_policy::reference_internal, + "Return the file header (read access)" ) + .def( "get_stream", + ( const monarch3::M3Stream* ( monarch3::Monarch3::* )( unsigned ) const ) + &monarch3::Monarch3::GetStream, + pybind11::arg( "stream" ), + pybind11::return_value_policy::reference_internal, + "Return the stream object for the given stream index (read access)" ) + .def( "finish_reading", + &monarch3::Monarch3::FinishReading, + "Close the file after reading.", + MONARCH3_BIND_CALL_GUARD_STREAMS ) + + // ---- Writing ---- + .def( "write_header", + &monarch3::Monarch3::WriteHeader, + "Write the header to file and prepare streams for writing.\n" + "Must be called after configuring the header and adding streams.", + MONARCH3_BIND_CALL_GUARD_STREAMS ) + .def( "get_header", + ( monarch3::M3Header* ( monarch3::Monarch3::* )() ) + &monarch3::Monarch3::GetHeader, + pybind11::return_value_policy::reference_internal, + "Return the file header (write access)" ) + .def( "get_stream", + ( monarch3::M3Stream* ( monarch3::Monarch3::* )( unsigned ) ) + &monarch3::Monarch3::GetStream, + pybind11::arg( "stream" ), + pybind11::return_value_policy::reference_internal, + "Return the stream object for the given stream index (write access)" ) + .def( "finish_writing", + &monarch3::Monarch3::FinishWriting, + "Flush and close the file after writing.", + MONARCH3_BIND_CALL_GUARD_STREAMS ) + ; + + return all_items; + } + +} /* namespace monarch3_pybind */ + +#endif /* M3MONARCH_PYBIND_HH_ */ diff --git a/Monarch3/python/m3record_pybind.hh b/Monarch3/python/m3record_pybind.hh new file mode 100644 index 0000000..59a111d --- /dev/null +++ b/Monarch3/python/m3record_pybind.hh @@ -0,0 +1,47 @@ +#ifndef M3RECORD_PYBIND_HH_ +#define M3RECORD_PYBIND_HH_ + +#include "M3Record.hh" + +#include "pybind11/pybind11.h" +#include "pybind11/numpy.h" + +namespace monarch3_pybind +{ + + std::list< std::string > export_record( pybind11::module& mod ) + { + std::list< std::string > all_items; + + all_items.push_back( "M3Record" ); + pybind11::class_< monarch3::M3Record >( mod, "M3Record", + "A single data record: ID, timestamp (ns), and raw data bytes" ) + .def_property_readonly( "record_id", + &monarch3::M3Record::GetRecordId, + "Record ID (uint64)" ) + .def_property_readonly( "time", + &monarch3::M3Record::GetTime, + "Timestamp in nanoseconds since the start of the run (uint64)" ) + .def( "get_data", + []( const monarch3::M3Record& r, unsigned nbytes ) { + // Zero-copy view: the numpy array does not own the data. + // The caller must ensure the record (and its parent stream) stays alive. + return pybind11::array_t< monarch3::byte_type >( + { static_cast< pybind11::ssize_t >( nbytes ) }, + { sizeof( monarch3::byte_type ) }, + r.GetData(), + pybind11::cast( r ) // keep-alive base object + ); + }, + pybind11::arg( "nbytes" ), + "Return a writable numpy uint8 view of the data buffer.\n" + "nbytes should equal stream.channel_record_n_bytes (or stream_record_n_bytes).\n" + "Use numpy.ndarray.view(dtype) to reinterpret as a different element type." ) + ; + + return all_items; + } + +} /* namespace monarch3_pybind */ + +#endif /* M3RECORD_PYBIND_HH_ */ diff --git a/Monarch3/python/m3stream_pybind.hh b/Monarch3/python/m3stream_pybind.hh new file mode 100644 index 0000000..e53cbc8 --- /dev/null +++ b/Monarch3/python/m3stream_pybind.hh @@ -0,0 +1,138 @@ +#ifndef M3STREAM_PYBIND_HH_ +#define M3STREAM_PYBIND_HH_ + +#include "M3Stream.hh" + +#include "monarch3_binding_helpers.hh" + +#include "pybind11/pybind11.h" +#include "pybind11/numpy.h" + +namespace monarch3_pybind +{ + + std::list< std::string > export_stream( pybind11::module& mod ) + { + std::list< std::string > all_items; + + all_items.push_back( "M3Stream" ); + pybind11::class_< monarch3::M3Stream >( mod, "M3Stream", + "Read/write access for a single data stream" ) + + // ---- State query properties ---- + .def_property_readonly( "n_channels", + &monarch3::M3Stream::GetNChannels, + "Number of channels in this stream" ) + .def_property_readonly( "n_acquisitions", + &monarch3::M3Stream::GetNAcquisitions, + "Number of acquisitions in this stream (valid after ReadRecord or after writing)" ) + .def_property_readonly( "acquisition_id", + &monarch3::M3Stream::GetAcquisitionId, + "ID of the most recently accessed acquisition" ) + .def_property_readonly( "record_count_in_acq", + &monarch3::M3Stream::GetRecordCountInAcq, + "Number of records read/written in the current acquisition" ) + .def_property_readonly( "n_records_in_file", + &monarch3::M3Stream::GetNRecordsInFile, + "Total number of records across all acquisitions in the file" ) + .def_property_readonly( "n_records_in_acquisition", + &monarch3::M3Stream::GetNRecordsInAcquisition, + "Number of records in the current acquisition" ) + .def_property_readonly( "data_type_size", + &monarch3::M3Stream::GetDataTypeSize, + "Size in bytes of each sample element" ) + .def_property_readonly( "sample_size", + &monarch3::M3Stream::GetSampleSize, + "Number of elements per sample (1 for real, 2 for complex)" ) + .def_property_readonly( "channel_record_size", + &monarch3::M3Stream::GetChannelRecordSize, + "Number of samples in a channel record" ) + .def_property_readonly( "channel_record_n_bytes", + &monarch3::M3Stream::GetChannelRecordNBytes, + "Size in bytes of a channel record data buffer" ) + .def_property_readonly( "stream_record_size", + &monarch3::M3Stream::GetStreamRecordSize, + "Number of samples in the full (interleaved) stream record" ) + .def_property_readonly( "stream_record_n_bytes", + &monarch3::M3Stream::GetStreamRecordNBytes, + "Size in bytes of the full stream record data buffer" ) + .def_property_readonly( "is_interleaved", + &monarch3::M3Stream::GetIsInterleaved, + "True if multi-channel data is stored interleaved in the file" ) + + // ---- Record access: reading ---- + .def( "read_record", + &monarch3::M3Stream::ReadRecord, + pybind11::arg( "offset" ) = 0, + pybind11::arg( "if_new_acq_start_at_first_rec" ) = true, + "Read a record from the file.\n\n" + "If the last record read was [J], reads record [J+1+offset].\n" + "offset=0 (default): next record; offset=-1: reread current;\n" + "offset<-1: step backward; offset>0: skip forward.\n" + "Returns True on success, False when past end/start of file.\n" + "Raises Monarch3Exception on error.", + MONARCH3_BIND_CALL_GUARD_STREAMS_AND_GIL ) + .def( "get_stream_record", + ( monarch3::M3Record* ( monarch3::M3Stream::* )() ) + &monarch3::M3Stream::GetStreamRecord, + pybind11::return_value_policy::reference_internal, + "Return the stream-level record object (all channels interleaved)" ) + .def( "get_channel_record", + ( monarch3::M3Record* ( monarch3::M3Stream::* )( unsigned ) ) + &monarch3::M3Stream::GetChannelRecord, + pybind11::arg( "channel" ), + pybind11::return_value_policy::reference_internal, + "Return the record object for the given channel index" ) + + // ---- Convenience: typed numpy data access ---- + .def( "get_stream_data", + []( monarch3::M3Stream& s ) { + monarch3::M3Record* r = s.GetStreamRecord(); + unsigned nbytes = s.GetStreamRecordNBytes(); + return pybind11::array_t< monarch3::byte_type >( + { static_cast< pybind11::ssize_t >( nbytes ) }, + { sizeof( monarch3::byte_type ) }, + r->GetData(), + pybind11::cast( s ) + ); + }, + pybind11::return_value_policy::reference_internal, + "Return a writable numpy uint8 view of the full (interleaved) stream record.\n" + "Call after read_record(). Use .view(dtype) to reinterpret elements." ) + .def( "get_channel_data", + []( monarch3::M3Stream& s, unsigned channel ) { + monarch3::M3Record* r = s.GetChannelRecord( channel ); + if( r == nullptr ) + { + throw monarch3::M3Exception() << "Channel " << channel << " does not exist in this stream"; + } + unsigned nbytes = s.GetChannelRecordNBytes(); + return pybind11::array_t< monarch3::byte_type >( + { static_cast< pybind11::ssize_t >( nbytes ) }, + { sizeof( monarch3::byte_type ) }, + r->GetData(), + pybind11::cast( s ) + ); + }, + pybind11::arg( "channel" ), + pybind11::return_value_policy::reference_internal, + "Return a writable numpy uint8 view of the data buffer for the given channel.\n" + "Call after read_record(). Use .view(dtype) to reinterpret elements." ) + + // ---- Record access: writing ---- + .def( "write_record", + &monarch3::M3Stream::WriteRecord, + pybind11::arg( "is_new_acquisition" ), + "Write the current record contents to the file.\n\n" + "is_new_acquisition=True starts a new acquisition group;\n" + "is_new_acquisition=False continues the current acquisition.\n" + "Returns True on success. Raises Monarch3Exception on error.", + MONARCH3_BIND_CALL_GUARD_STREAMS_AND_GIL ) + ; + + return all_items; + } + +} /* namespace monarch3_pybind */ + +#endif /* M3STREAM_PYBIND_HH_ */ diff --git a/Monarch3/python/monarch3_binding_helpers.hh b/Monarch3/python/monarch3_binding_helpers.hh new file mode 100644 index 0000000..9d4fb67 --- /dev/null +++ b/Monarch3/python/monarch3_binding_helpers.hh @@ -0,0 +1,15 @@ +#ifndef MONARCH3_PYBIND_BINDING_HELPERS_HH_ +#define MONARCH3_PYBIND_BINDING_HELPERS_HH_ + +#include "pybind11/iostream.h" + +#define MONARCH3_BIND_CALL_GUARD_STREAMS \ + pybind11::call_guard< pybind11::scoped_ostream_redirect, pybind11::scoped_estream_redirect >() + +#define MONARCH3_BIND_CALL_GUARD_GIL \ + pybind11::call_guard< pybind11::gil_scoped_release >() + +#define MONARCH3_BIND_CALL_GUARD_STREAMS_AND_GIL \ + pybind11::call_guard< pybind11::scoped_ostream_redirect, pybind11::scoped_estream_redirect, pybind11::gil_scoped_release >() + +#endif /* MONARCH3_PYBIND_BINDING_HELPERS_HH_ */ diff --git a/Monarch3/python/monarch3_namespace_pybind.cc b/Monarch3/python/monarch3_namespace_pybind.cc new file mode 100644 index 0000000..60624db --- /dev/null +++ b/Monarch3/python/monarch3_namespace_pybind.cc @@ -0,0 +1,45 @@ +/* + * monarch3_namespace_pybind.cc + * + * Pybind11 module entry point for the monarch3 Python binding. + */ + +#include "pybind11/pybind11.h" +#include "pybind11/stl.h" +#include "pybind11/numpy.h" + +#include "M3Exception.hh" + +#include "m3constants_pybind.hh" +#include "m3header_pybind.hh" +#include "m3record_pybind.hh" +#include "m3stream_pybind.hh" +#include "m3monarch_pybind.hh" + +PYBIND11_MODULE( monarch3, mod ) +{ + mod.doc() = "Python bindings for the Monarch3 egg v3 file library"; + + std::list< std::string > all_members; + + // Exception + pybind11::register_exception< monarch3::M3Exception >( mod, "Monarch3Exception" ); + all_members.push_back( "Monarch3Exception" ); + + // Constants + all_members.splice( all_members.end(), monarch3_pybind::export_constants( mod ) ); + + // Header classes + all_members.splice( all_members.end(), monarch3_pybind::export_header( mod ) ); + + // Record class + all_members.splice( all_members.end(), monarch3_pybind::export_record( mod ) ); + + // Stream class + all_members.splice( all_members.end(), monarch3_pybind::export_stream( mod ) ); + + // Top-level Monarch3 class + all_members.splice( all_members.end(), monarch3_pybind::export_monarch( mod ) ); + + mod.attr( "__all__" ) = all_members; +} diff --git a/README.md b/README.md index d18afdd..0ecd3ab 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,6 @@ Monarch ======= -[![Codacy Badge](https://api.codacy.com/project/badge/Grade/9495b2fba24c44c8884d78c80873e45d)](https://www.codacy.com/project/Project8/monarch/dashboard?utm_source=github.com&utm_medium=referral&utm_content=project8/monarch&utm_campaign=Badge_Grade_Dashboard) [![Documentation Status](https://readthedocs.org/projects/monarch/badge/?version=stable)](https://monarch.readthedocs.io/en/stable/?badge=stable) [![DOI](https://zenodo.org/badge/2208206.svg)](https://zenodo.org/badge/latestdoi/2208206) @@ -28,7 +27,7 @@ Requirements ------------ - CMake 3.1 or higher -- HDF5 1.8.12 or higher (Monarch3 only) +- HDF5 1.10.1 or higher (Monarch3 only) - Google Protocol Buffers (Monarch2 only) diff --git a/changelog.md b/changelog.md new file mode 100644 index 0000000..fc5846c --- /dev/null +++ b/changelog.md @@ -0,0 +1,46 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +Types of changes: Added, Changed, Deprecated, Removed, Fixed, Security + +## [Unreleased] ([3.9.0] - 2026-08-19) + +### Added + +- Python interface for Monarch3 via pybind11 (`Monarch3/python/`), exposing `M3Header`, + `M3StreamHeader`, `M3ChannelHeader`, `M3Stream`, `M3Record`, and `Monarch3` to Python + as the `monarch3` module; record data is returned as zero-copy numpy `uint8` arrays +- Context manager (`with` statement) support for `Monarch3` objects; `FinishReading()` + or `FinishWriting()` is called automatically on exit +- `Monarch_BUILD_PYTHON` CMake option to enable the Python binding (requires pybind11 and + Python 3 development headers) +- Validation test suite (`Monarch3/Tests/Validation/`) using pytest and CTest: + - C++ write → Python read (`test_cpp_write_python_read.py`) + - Python write → C++ read (`test_python_write_cpp_read.py`) + - Python write → Python read (`test_python_write_python_read.py`) +- `Monarch_ENABLE_TESTING` CMake option now activates `enable_testing()` and the CTest + validation suite in addition to the existing C++ test executables +- Documentation pages: `UsageMonarch3Python.rst` and `TestingMonarch3.rst` +- GitHub Actions workflow (`.github/workflows/run_tests.yaml`) with separate jobs for + Monarch3 (including Python validation tests) and Monarch2, plus a Release job +- Python API documentation is built on ReadTheDocs, including building the full C++ library + and Python bindings so that the full API is available. + +### Fixed + +- `M3ReadTest`: offset-navigation test (Test 2) now passes `aIfNewAcqStartAtFirstRec=false` + when crossing an acquisition boundary to land on the exact target record, and corrects + a `ReadRecord(-2)` call that would step before the start of the file +- `M3ReadTest`: Test 3 (stream 2 skip) likewise uses `aIfNewAcqStartAtFirstRec=false` for + the initial offset skip +- RTD documentation setup was updated to modern RTD standards + +### Changed + +- HDF5 minimum version raised to 1.10.1; the v1.8 API compatibility workaround has been + removed from `CMakeLists.txt` +- C++ use documentation was updated