diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index ddb2ae0..888d102 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -19,24 +19,20 @@ jobs: name: Build BlazingMQ as a dependency runs-on: ubuntu-24.04 outputs: - blazingmq_sha: ${{ steps.get-sha.outputs.blazingmq_sha }} + deps_cache_key: ${{ steps.cache-key.outputs.deps_cache_key }} steps: - uses: actions/checkout@v4 - - name: Checkout BlazingMQ - run: git clone --depth 1 https://github.com/bloomberg/blazingmq - - - name: Get latest BlazingMQ commit SHA - id: get-sha - working-directory: blazingmq - run: echo "blazingmq_sha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT + - name: Compute dependency cache key + id: cache-key + run: echo "deps_cache_key=deps-${{ hashFiles('bin/clone-dependencies.sh') }}" >> $GITHUB_OUTPUT - name: Try to get cached BlazingMQ build artifacts id: cache-restore uses: actions/cache/restore@v4 with: path: blazingmq_artifacts.tar.gz - key: ${{ steps.get-sha.outputs.blazingmq_sha }} + key: ${{ steps.cache-key.outputs.deps_cache_key }} - name: Set up dependencies if: steps.cache-restore.outputs.cache-hit != 'true' @@ -52,19 +48,22 @@ jobs: bison \ libfl-dev \ libbenchmark-dev \ + libgtest-dev \ + libgmock-dev \ libz-dev - name: Create install directory for BlazingMQ and its dependencies if: steps.cache-restore.outputs.cache-hit != 'true' run: mkdir -p blazingmq_artifacts - - name: Skip building bison and google-benchmark + - name: Skip building bison, google-benchmark, and googletest if: steps.cache-restore.outputs.cache-hit != 'true' run: | - mkdir -p thirdparty/{bison,google-benchmark} - mkdir -p build/{bison,google-benchmark} + mkdir -p thirdparty/{bison,google-benchmark,googletest} + mkdir -p build/{bison,google-benchmark,googletest} touch thirdparty/bison/.complete touch build/google-benchmark/.complete + touch build/googletest/.complete - name: Build and install BlazingMQ and its dependencies if: steps.cache-restore.outputs.cache-hit != 'true' @@ -82,7 +81,7 @@ jobs: uses: actions/cache/save@v4 with: path: blazingmq_artifacts.tar.gz - key: ${{ steps.get-sha.outputs.blazingmq_sha }} + key: ${{ steps.cache-key.outputs.deps_cache_key }} linux-check: name: Test on Linux @@ -104,7 +103,7 @@ jobs: uses: actions/cache/restore@v4 with: path: blazingmq_artifacts.tar.gz - key: ${{ needs.blazingmq-dependency.outputs.blazingmq_sha }} + key: ${{ needs.blazingmq-dependency.outputs.deps_cache_key }} - name: Restore cached BlazingMQ build artifacts run: tar xzf blazingmq_artifacts.tar.gz - name: Set up Python ${{ matrix.python-version }} @@ -131,6 +130,8 @@ jobs: bison \ libfl-dev \ libbenchmark-dev \ + libgtest-dev \ + libgmock-dev \ libz-dev - name: Log in to GHCR uses: docker/login-action@v3 @@ -147,6 +148,11 @@ jobs: /usr/local/bin/bmqbrkr /broker-config timeout 30 bash -c 'until nc -z localhost 30114; do sleep 1; done' \ || (docker logs bmqbrkr; exit 1) + - name: Setup core dumps + run: | + sudo mkdir -p /cores + sudo chmod 777 /cores + echo "/cores/%e.%p.%s.%t" | sudo tee /proc/sys/kernel/core_pattern - name: Build and run tests env: BMQ_BROKER_URI: tcp://localhost:30114 @@ -154,7 +160,22 @@ jobs: PYTHON: ./venv/bin/python PKG_CONFIG_PATH: /usr/lib/x86_64-linux-gnu/pkgconfig:/opt/bb/lib64/pkgconfig:./blazingmq_artifacts/lib64/pkgconfig run: | + ulimit -c unlimited make test-build && make test-install && make check + - name: Print backtrace from core dump + if: failure() + run: | + for core in /cores/*; do + if [ -f "$core" ]; then + echo "=== Backtrace from $core ===" + gdb -batch \ + -ex "set print frame-arguments all" \ + -ex "bt full" \ + -ex "echo \n=== All threads ===\n" \ + -ex "thread apply all bt" \ + ./venv/bin/python "$core" + fi + done - name: Stop BlazingMQ broker if: always() run: docker rm -f bmqbrkr @@ -170,7 +191,7 @@ jobs: uses: actions/cache/restore@v4 with: path: blazingmq_artifacts.tar.gz - key: ${{ needs.blazingmq-dependency.outputs.blazingmq_sha }} + key: ${{ needs.blazingmq-dependency.outputs.deps_cache_key }} - name: Restore cached BlazingMQ build artifacts run: tar xzf blazingmq_artifacts.tar.gz - name: Set up Python 3.9 @@ -218,7 +239,7 @@ jobs: uses: actions/cache/restore@v4 with: path: blazingmq_artifacts.tar.gz - key: ${{ needs.blazingmq-dependency.outputs.blazingmq_sha }} + key: ${{ needs.blazingmq-dependency.outputs.deps_cache_key }} - name: Restore cached BlazingMQ build artifacts run: tar xzf blazingmq_artifacts.tar.gz - name: Set up Python diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 532a118..082d734 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -2,6 +2,9 @@ name: Wheels on: push: + pull_request: + branches: + - main release: types: - published @@ -85,11 +88,10 @@ jobs: run: | echo 0 | sudo tee /proc/sys/kernel/yama/ptrace_scope - name: Build wheels - uses: pypa/cibuildwheel@v2.19.2 + uses: pypa/cibuildwheel@v3.4.1 env: CIBW_ARCHS_LINUX: ${{ matrix.cibw_arch }} CIBW_BUILD: ${{ matrix.cibw_python }}-* - CIBW_PRERELEASE_PYTHONS: True CIBW_TEST_COMMAND: python3 -m pytest {project}/tests/unit CIBW_TEST_REQUIRES: pytest mock pkgconfig - uses: actions/upload-artifact@v4 @@ -130,12 +132,11 @@ jobs: run: | echo "CFLAGS=-target arm64-apple-macos14" >> $GITHUB_ENV - name: Build wheels - uses: pypa/cibuildwheel@v2.19.2 + uses: pypa/cibuildwheel@v3.4.1 env: CIBW_ARCHS_MACOS: ${{ matrix.cibw_arch }} CIBW_BUILD: ${{ matrix.cibw_python }}-* CIBW_BUILD_VERBOSITY: 1 - CIBW_PRERELEASE_PYTHONS: True CIBW_TEST_COMMAND: python3 -m pytest {project}/tests/unit CIBW_TEST_REQUIRES: pytest mock pkgconfig MACOSX_DEPLOYMENT_TARGET: "14.0" diff --git a/bin/build-macos-universal.sh b/bin/build-macos-universal.sh index 298b642..8ce42e6 100755 --- a/bin/build-macos-universal.sh +++ b/bin/build-macos-universal.sh @@ -16,6 +16,7 @@ brew install \ bison \ flex \ google-benchmark \ + googletest \ cmake PREREQUISITES @@ -68,6 +69,7 @@ if [ ! -e "${DIR_BUILD}/ntf-core/.complete" ]; then --output "${DIR_BUILD}/ntf-core" \ --ufid opt_64_pic_cpp17 \ --generator "Ninja" \ + --with-zlib \ --without-lz4 \ --without-zstd \ --without-warnings-as-errors \ diff --git a/bin/build-manylinux.sh b/bin/build-manylinux.sh index 90d841f..32a6b79 100755 --- a/bin/build-manylinux.sh +++ b/bin/build-manylinux.sh @@ -46,6 +46,9 @@ source ./bin/clone-dependencies.sh if [ ! -d "${DIR_THIRDPARTY}/google-benchmark" ]; then git clone --depth 1 https://github.com/google/benchmark.git "${DIR_THIRDPARTY}/google-benchmark" fi +if [ ! -d "${DIR_THIRDPARTY}/googletest" ]; then + git clone --depth 1 https://github.com/google/googletest.git "${DIR_THIRDPARTY}/googletest" +fi if [ ! -d "${DIR_THIRDPARTY}/bison" ]; then mkdir -p "${DIR_THIRDPARTY}/bison" curl https://ftp.gnu.org/gnu/bison/bison-3.8.2.tar.xz | tar -Jx -C "${DIR_THIRDPARTY}/bison" --strip-components 1 @@ -72,6 +75,20 @@ if [ ! -e "${DIR_THIRDPARTY}/bison/.complete" ]; then touch "${DIR_THIRDPARTY}/bison/.complete" fi +if [ ! -e "${DIR_BUILD}/openssl/.complete" ]; then + if [ ! -d "${DIR_THIRDPARTY}/openssl" ]; then + mkdir -p "${DIR_THIRDPARTY}/openssl" + curl -sL https://www.openssl.org/source/openssl-1.1.1w.tar.gz | tar -xz -C "${DIR_THIRDPARTY}/openssl" --strip-components 1 + fi + pushd "${DIR_THIRDPARTY}/openssl" + ./config --prefix="${DIR_INSTALL}" --openssldir="${DIR_INSTALL}/ssl" --libdir=lib64 no-shared -fPIC + make -j 16 + make install_sw + popd + mkdir -p "${DIR_BUILD}/openssl" + touch "${DIR_BUILD}/openssl/.complete" +fi + if [ ! -e "${DIR_BUILD}/google-benchmark/.complete" ]; then pushd "${DIR_THIRDPARTY}/google-benchmark" cmake -E make_directory "${DIR_BUILD}/google-benchmark" @@ -82,6 +99,15 @@ if [ ! -e "${DIR_BUILD}/google-benchmark/.complete" ]; then touch "${DIR_BUILD}/google-benchmark/.complete" fi +if [ ! -e "${DIR_BUILD}/googletest/.complete" ]; then + pushd "${DIR_THIRDPARTY}/googletest" + cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX="${DIR_INSTALL}" -DCMAKE_INSTALL_LIBDIR="lib64" -S . -B "${DIR_BUILD}/googletest" + cmake --build "${DIR_BUILD}/googletest" --config Release -j16 + cmake --build "${DIR_BUILD}/googletest" --config Release --target install + popd + touch "${DIR_BUILD}/googletest/.complete" +fi + # Build and install BDE # Refer to https://bloomberg.github.io/bde/library_information/build.html PATH="${DIR_THIRDPARTY}/bde-tools/bin:$PATH" @@ -100,10 +126,12 @@ fi if [ ! -e "${DIR_BUILD}/ntf-core/.complete" ]; then # Build and install NTF pushd "${DIR_THIRDPARTY}/ntf-core" + OPENSSL_ROOT_DIR="${DIR_INSTALL}" \ ./configure --prefix "${DIR_INSTALL}" \ --output "${DIR_BUILD}/ntf-core" \ --ufid opt_64_pic_cpp17 \ --generator "Ninja" \ + --without-zlib \ --without-lz4 \ --without-zstd \ --without-warnings-as-errors \ diff --git a/bin/clone-dependencies.sh b/bin/clone-dependencies.sh index b961855..10260a8 100755 --- a/bin/clone-dependencies.sh +++ b/bin/clone-dependencies.sh @@ -12,40 +12,36 @@ set -u # These are the release tags for each of the dependencies we manually clone. # Update these to update the version of each dependency we build against. -BDE_TOOLS_TAG=4.32.0.0 -BDE_TAG=4.32.0.0 -NTF_CORE_TAG=2.6.6 -BLAZINGMQ_TAG=BMQBRKR_0.94.8 +BDE_TOOLS_TAG=4.38.0.0 +BDE_TAG=4.38.0.0 +NTF_CORE_TAG=2.6.12 +BLAZINGMQ_TAG=v0.95.14 if [ ! -d "${DIR_THIRDPARTY}/bde-tools" ]; then git clone \ - --depth 1 \ - --branch ${BDE_TOOLS_TAG} \ https://github.com/bloomberg/bde-tools \ "${DIR_THIRDPARTY}/bde-tools" + git -C "${DIR_THIRDPARTY}/bde-tools" checkout ${BDE_TOOLS_TAG} fi if [ ! -d "${DIR_THIRDPARTY}/bde" ]; then git clone \ - --depth 1 \ - --branch ${BDE_TAG} \ https://github.com/bloomberg/bde.git \ "${DIR_THIRDPARTY}/bde" + git -C "${DIR_THIRDPARTY}/bde" checkout ${BDE_TAG} fi if [ ! -d "${DIR_THIRDPARTY}/ntf-core" ]; then git clone \ - --depth 1 \ - --branch ${NTF_CORE_TAG} \ https://github.com/bloomberg/ntf-core.git \ "${DIR_THIRDPARTY}/ntf-core" + git -C "${DIR_THIRDPARTY}/ntf-core" checkout ${NTF_CORE_TAG} fi if [ ! -d "${DIR_THIRDPARTY}/blazingmq" ]; then git clone \ - --depth 1 \ - --branch ${BLAZINGMQ_TAG} \ https://github.com/bloomberg/blazingmq.git \ "${DIR_THIRDPARTY}/blazingmq" + git -C "${DIR_THIRDPARTY}/blazingmq" checkout ${BLAZINGMQ_TAG} fi diff --git a/pyproject.toml b/pyproject.toml index a36f890..8aa0f90 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,6 +4,7 @@ requires = ['setuptools>=39.2.0', 'Cython>=0.28.4, <3; python_version < "3.8"', 'Cython >= 3, <4; python_version >= "3.8"', 'wheel>=0.31.0', + 'packaging>=24.2', 'pkgconfig>1.5.0'] build-backend = "setuptools.build_meta" @@ -53,8 +54,7 @@ directory = "news" [tool.cibuildwheel] build = ["cp39-*", "cp310-*", "cp311-*", "cp312-*"] skip = "*-musllinux_*" -manylinux-x86_64-image = "manylinux2014" -manylinux-i686-image = "manylinux2014" +manylinux-x86_64-image = "manylinux_2_28" [tool.cibuildwheel.linux.environment] PKG_CONFIG_PATH="/usr/local/lib64/pkgconfig:/usr/lib64/pkgconfig" @@ -64,7 +64,7 @@ PKG_CONFIG_PATH="install/lib/pkgconfig" [tool.cibuildwheel.linux] before-all = [ - "yum install -y gdb curl pkgconfig ninja-build flex flex-devel zlib-devel openssl-devel", + "dnf install -y gdb curl pkgconfig ninja-build flex flex-devel zlib-devel zlib-static perl-core", "DIR_INSTALL=/usr/local bash bin/build-manylinux.sh" ] diff --git a/src/cpp/pybmq_session.cpp b/src/cpp/pybmq_session.cpp index 80c8947..acab799 100644 --- a/src/cpp/pybmq_session.cpp +++ b/src/cpp/pybmq_session.cpp @@ -500,7 +500,9 @@ Session::post( bmqa::Message& message = builder.startMessage(); - message.setDataRef(payload, payload_length); + if (payload_length > 0) { + message.setDataRef(payload, payload_length); + } if (properties != Py_None) { message.setPropertiesRef(&c_properties);