diff --git a/.github/actions/setup-boost-186/action.yml b/.github/actions/setup-boost-186/action.yml new file mode 100644 index 00000000..a90ce8b6 --- /dev/null +++ b/.github/actions/setup-boost-186/action.yml @@ -0,0 +1,33 @@ +name: Setup Boost 1.86 +description: Restores or builds the Boost 1.86 toolchain required by libbitcoin-system. +runs: + using: composite + steps: + - name: Restore Boost 1.86 cache + id: boost-cache + uses: actions/cache/restore@v4 + with: + path: ${{ runner.tool_cache }}/bitcoinfuzz/boost-1.86.0 + key: boost-1.86.0-${{ runner.os }}-clang-${{ hashFiles('.github/actions/setup-boost-186/**') }} + + - name: Build Boost 1.86 + shell: bash + env: + BOOST_PREFIX: ${{ runner.tool_cache }}/bitcoinfuzz/boost-1.86.0 + BOOST_VERSION: 1.86.0 + BOOST_URL: https://github.com/boostorg/boost/releases/download/boost-1.86.0/boost-1.86.0-cmake.tar.gz + BOOST_LIBRARIES: iostreams;locale;program_options;thread;url;test;regex;filesystem;system;date_time;chrono;atomic;random;container;charconv;json + CC: /usr/bin/clang + CXX: /usr/bin/clang++ + run: bash "${{ github.action_path }}/install-boost.sh" + + - name: Save Boost 1.86 cache + if: ${{ steps.boost-cache.outputs.cache-hit != 'true' }} + uses: actions/cache/save@v4 + with: + path: ${{ runner.tool_cache }}/bitcoinfuzz/boost-1.86.0 + key: boost-1.86.0-${{ runner.os }}-clang-${{ hashFiles('.github/actions/setup-boost-186/**') }} + + - name: Export BOOST_ROOT + shell: bash + run: echo "BOOST_ROOT=${{ runner.tool_cache }}/bitcoinfuzz/boost-1.86.0" >> "$GITHUB_ENV" diff --git a/.github/actions/setup-boost-186/install-boost.sh b/.github/actions/setup-boost-186/install-boost.sh new file mode 100755 index 00000000..d081d493 --- /dev/null +++ b/.github/actions/setup-boost-186/install-boost.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +set -euo pipefail + +boost_config="${BOOST_PREFIX}/lib/cmake/Boost-${BOOST_VERSION}/BoostConfig.cmake" +if [[ -f "${boost_config}" ]]; then + exit 0 +fi + +mkdir -p "$(dirname "${BOOST_PREFIX}")" + +workdir="$(mktemp -d "${RUNNER_TEMP:-/tmp}/boost-${BOOST_VERSION}.XXXXXX")" +archive="${workdir}/boost.tar.gz" +srcdir="${workdir}/boost-${BOOST_VERSION}" +stage_prefix="${BOOST_PREFIX}.new" + +cleanup() { + rm -rf "${workdir}" "${stage_prefix}" +} + +trap cleanup EXIT + +rm -rf "${BOOST_PREFIX}" "${stage_prefix}" + +curl --fail --show-error --location \ + --retry 5 \ + --retry-all-errors \ + --retry-delay 2 \ + --output "${archive}" \ + "${BOOST_URL}" + +tar -tzf "${archive}" >/dev/null +tar -xzf "${archive}" -C "${workdir}" + +if [[ ! -d "${srcdir}" ]]; then + echo "Expected Boost source directory ${srcdir} was not created" >&2 + exit 1 +fi + +cmake -S "${srcdir}" -B "${srcdir}/build" \ + -DCMAKE_C_COMPILER="${CC}" \ + -DCMAKE_CXX_COMPILER="${CXX}" \ + -DCMAKE_INSTALL_PREFIX="${stage_prefix}" \ + -DBUILD_SHARED_LIBS=OFF \ + -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ + -DBOOST_INCLUDE_LIBRARIES="${BOOST_LIBRARIES}" + +cmake --build "${srcdir}/build" --parallel "$(nproc)" +cmake --install "${srcdir}/build" + +mv "${stage_prefix}" "${BOOST_PREFIX}" diff --git a/.github/actions/setup-test-env/action.yml b/.github/actions/setup-test-env/action.yml index da17073b..9d31d657 100644 --- a/.github/actions/setup-test-env/action.yml +++ b/.github/actions/setup-test-env/action.yml @@ -1,50 +1,33 @@ name: Setup Test Environment -description: Sets up the environment for a test job, including repo checkout, cache restore, and macOS-specific setup. +description: Checks out bitcoinfuzz at the commit the build job used, downloads the build artifacts and installs the runtime deps. inputs: - os: - description: Operating system - required: true - github-sha: - description: GitHub SHA - required: true - github-run-id: - description: GitHub Run ID + bitcoinfuzz-ref: + description: bitcoinfuzz commit SHA the build job compiled required: true runs: using: "composite" steps: - - name: Restore build artifacts - uses: actions/cache/restore@v4 + - name: Checkout bitcoinfuzz repo + uses: actions/checkout@v4 + with: + repository: bitcoinfuzz/bitcoinfuzz + ref: ${{ inputs.bitcoinfuzz-ref }} + path: bitcoinfuzz + + - name: Download build artifacts + uses: actions/download-artifact@v8 with: - path: | - bitcoinfuzz - build.env - key: bitcoinfuzz-build-${{ inputs.os }}-${{ inputs.github-sha }}-${{ inputs.github-run-id }} + name: bitcoinfuzz-build + path: bitcoinfuzz + - name: Setup environment shell: bash run: | sudo apt-get update sudo apt-get install -y lowdown libsodium-dev + # libbitcoin-system requires CMake >= 3.30; Ubuntu 24.04 ships 3.28.3. + # pip provides a binary cmake that takes precedence on PATH. + python3 -m pip install --upgrade --break-system-packages 'cmake>=3.30' - name: Install Boost 1.86 (required for libbitcoin-system targets) - shell: bash - run: | - # Cache hit short-circuits this; Boost is installed to /opt/boost-1.86 - # so the bitcoinfuzz link step can resolve libbitcoin-system's Boost refs. - if [ ! -f /opt/boost-1.86/lib/cmake/Boost-1.86.0/BoostConfig.cmake ]; then - python3 -m pip install --upgrade --break-system-packages 'cmake>=3.30' - curl -sSL -o /tmp/boost.tar.gz \ - https://github.com/boostorg/boost/releases/download/boost-1.86.0/boost-1.86.0-cmake.tar.gz - tar -xzf /tmp/boost.tar.gz -C /tmp - cd /tmp/boost-1.86.0 - cmake -B build \ - -DCMAKE_C_COMPILER=/usr/bin/clang \ - -DCMAKE_CXX_COMPILER=/usr/bin/clang++ \ - -DCMAKE_INSTALL_PREFIX=/opt/boost-1.86 \ - -DBUILD_SHARED_LIBS=OFF \ - -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ - -DBOOST_INCLUDE_LIBRARIES="iostreams;locale;program_options;thread;url;test;regex;filesystem;system;date_time;chrono;atomic;random;container;charconv" - cmake --build build --parallel "$(nproc)" - sudo cmake --install build - fi - echo "BOOST_ROOT=/opt/boost-1.86" >> $GITHUB_ENV + uses: ./.github/actions/setup-boost-186 diff --git a/.github/scripts/run-fuzz.sh b/.github/scripts/run-fuzz.sh index 19c8bdc0..c5ddfa70 100755 --- a/.github/scripts/run-fuzz.sh +++ b/.github/scripts/run-fuzz.sh @@ -7,6 +7,10 @@ CXXFLAGS="$3" ASAN_OPTIONS="${4:-}" CXX="${5}" +# Remove the fixed arguments in order to forward +# any extra arguments to libFuzzer +shift 5 + if [ -d "./${CORPUS_DIR}" ]; then echo "Using corpora for ${TARGET}" cd bitcoinfuzz @@ -14,7 +18,7 @@ if [ -d "./${CORPUS_DIR}" ]; then export CXX="${CXX}" [[ -n "${ASAN_OPTIONS}" ]] && export ASAN_OPTIONS="${ASAN_OPTIONS}" make - FUZZ="${TARGET}" ./bitcoinfuzz -runs=1 "../${CORPUS_DIR}" + FUZZ="${TARGET}" ./bitcoinfuzz -runs=1 "$@" "../${CORPUS_DIR}" else echo "Corpus ./${CORPUS_DIR} does not exist. Skipping." fi diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index 1962a653..08a1125f 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -7,6 +7,8 @@ jobs: runs-on: ubuntu-24.04 strategy: fail-fast: false + outputs: + bitcoinfuzz-sha: ${{ steps.bitcoinfuzz-sha.outputs.sha }} steps: - name: Checkout repo @@ -16,13 +18,16 @@ jobs: uses: actions/checkout@v4 with: repository: bitcoinfuzz/bitcoinfuzz + ref: v2 path: bitcoinfuzz submodules: recursive + - name: Record bitcoinfuzz commit + id: bitcoinfuzz-sha + run: echo "sha=$(git -C bitcoinfuzz rev-parse HEAD)" >> "$GITHUB_OUTPUT" + - name: Install rust-toolchain - uses: actions-rs/toolchain@v1.0.6 - with: - toolchain: nightly + uses: dtolnay/rust-toolchain@nightly - name: Install go uses: actions/setup-go@v5 @@ -38,30 +43,25 @@ jobs: - name: Install .NET SDK uses: actions/setup-dotnet@v4 with: - dotnet-version: "9.0.x" - + dotnet-version: "10.0.x" - name: Install build dependencies run: | sudo apt-get update - sudo apt-get install -y lowdown libsodium-dev libboost-all-dev autoconf automake libtool pkg-config + sudo apt-get install -y lowdown libsodium-dev libboost-all-dev npm nodejs quickjs libquickjs autoconf automake libtool pkg-config # libbitcoin-system requires CMake >= 3.30; Ubuntu 24.04 ships 3.28.3. # pip provides a binary cmake that takes precedence on PATH. python3 -m pip install --upgrade --break-system-packages 'cmake>=3.30' + - name: Install Boost 1.86 (libbitcoin-system requires >= 1.86) + uses: ./.github/actions/setup-boost-186 + - name: Setup common environment run: | - export CC=/usr/bin/clang - export CXX=/usr/bin/clang++ echo "CC=/usr/bin/clang" >> $GITHUB_ENV echo "CXX=/usr/bin/clang++" >> $GITHUB_ENV - - name: Save environment variables - run: | - echo "CC=$CC" >> build.env - echo "CXX=$CXX" >> build.env - - - name: Setup Python and install embit, pybitcoinkernel and pycoin + - name: Setup Python and install embit, pybitcoinkernel, pycoin and electrum uses: actions/setup-python@v5 with: python-version: "3.11" @@ -70,49 +70,67 @@ jobs: pip install -r bitcoinfuzz/modules/embit/requirements.txt pip install -r bitcoinfuzz/modules/pybitcoinkernel/requirements.txt pip install -r bitcoinfuzz/modules/pycoin/requirements.txt + pip install -r bitcoinfuzz/modules/electrum/requirements.txt pip install mako - name: Build Modules timeout-minutes: 40 + working-directory: bitcoinfuzz run: | - cd bitcoinfuzz && CXXFLAGS="-DBITCOIN_CORE \ + CXXFLAGS="-DBITCOIN_CORE \ -DRUST_BITCOIN \ -DRUSTBITCOINKERNEL \ -DRUST_MINISCRIPT \ + -DBITCOINERLAB_MINISCRIPT \ + -DRUST_PSBT \ + -DRUST_MUSIG2 \ + -DRUSTCRYPTO_AES \ + -DBDK_SP \ + -DBLUEWALLET_SP \ + -DSPDK \ -DBTCD \ -DNBITCOIN \ -DEMBIT \ -DPYBITCOINKERNEL \ + -DBITCOINKERNEL \ -DLND \ -DLDK \ -DNLIGHTNING \ -DLIGHTNING_KMP \ -DBITCOINJ \ + -DBITCOINS \ -DCLIGHTNING \ -DECLAIR \ - -DLIBWALLY_CORE \ - -DPYCOIN \ -DDECRED_SECP256K1 \ -DSECP256K1 \ -DNBITCOIN_SECP256K1 \ -DRUST_K256 \ + -DGOCOIN \ + -DLIBWALLY_CORE \ -DCUSTOM_MUTATOR_P2P_MESSAGE \ -DRUSTREEXO \ -DUTREEXO \ - -DBITCOINS \ -DPYCOIN \ - -DLIBWALLY_CORE" \ + -DELECTRUM \ -DLIBBITCOIN_SYSTEM" \ ONLY_MODULES=1 \ - ./auto_build.py + ./scripts/auto_build.py - - name: Cache build artifacts - uses: actions/cache/save@v4 + - name: Upload build artifacts + uses: actions/upload-artifact@v7 with: + name: bitcoinfuzz-build path: | - bitcoinfuzz - build.env - key: bitcoinfuzz-build-${{ matrix.os }}-${{ github.sha }}-${{ github.run_id }} + bitcoinfuzz/modules/ + bitcoinfuzz/custommutator/ + bitcoinfuzz/bitcoinfuzz + bitcoinfuzz/NBitcoin.CppBridge.so + bitcoinfuzz/NBitcoin.CppBridge.dylib + bitcoinfuzz/NBitcoinSecp256k1.CppBridge.so + bitcoinfuzz/external/secp256k1/include/ + if-no-files-found: error + include-hidden-files: true + retention-days: 1 test: needs: build @@ -151,7 +169,7 @@ jobs: asan_options: "detect_leaks=0:detect_container_overflow=0" - corpus: "psbt_parse_clean" target: "psbt_parse" - cxxflags: "-DBITCOIN_CORE -DRUST_BITCOIN -DRUST_PSBT -DBTCD -DNBITCOIN -DLIBWALLY_CORE -DBITCOINS" + cxxflags: "-DBITCOIN_CORE -DRUST_PSBT -DBTCD -DNBITCOIN -DLIBWALLY_CORE -DBITCOINS -DCUSTOM_MUTATOR_PSBT" asan_options: "detect_leaks=0:detect_container_overflow=0" setup_java: "true" - corpus: "kernel_block" @@ -190,13 +208,17 @@ jobs: asan_options: "detect_leaks=0:detect_container_overflow=0" - corpus: "bip32_master_keygen" target: "bip32_master_keygen" - cxxflags: "-DNBITCOIN -DRUST_BITCOIN -DBITCOIN_CORE -DBTCD -DBITCOINJ -DLIBWALLY_CORE -DEMBIT -DPYCOIN -DBITCOINS -DLIBBITCOIN_SYSTEM" + cxxflags: "-DNBITCOIN -DRUST_BITCOIN -DBITCOIN_CORE -DBTCD -DBITCOINJ -DLIBWALLY_CORE -DELECTRUM -DEMBIT -DPYCOIN -DBITCOINS -DLIBBITCOIN_SYSTEM" asan_options: "detect_leaks=0:detect_container_overflow=0" setup_java: "true" - corpus: "private_to_public_key" target: "private_to_public_key" cxxflags: "-DDECRED_SECP256K1 -DSECP256K1 -DNBITCOIN_SECP256K1 -DRUST_K256" asan_options: "detect_leaks=0:detect_container_overflow=0" + - corpus: "pubkey_parse" + target: "pubkey_parse" + cxxflags: "-DSECP256K1 -DRUST_K256 -DDECRED_SECP256K1" + asan_options: "detect_leaks=0:detect_container_overflow=0" - corpus: "sign_compact" target: "sign_compact" cxxflags: "-DDECRED_SECP256K1 -DSECP256K1 -DNBITCOIN_SECP256K1 -DRUST_K256" @@ -205,6 +227,10 @@ jobs: target: "sign_der" cxxflags: "-DDECRED_SECP256K1 -DSECP256K1 -DNBITCOIN_SECP256K1 -DRUST_K256" asan_options: "detect_leaks=0:detect_container_overflow=0" + - corpus: "merkle_root_compute" + target: "merkle_root_compute" + cxxflags: "-DBITCOIN_CORE -DRUST_BITCOIN -DBTCD -DGOCOIN" + asan_options: "detect_leaks=0:detect_container_overflow=0" - corpus: "sign_verify" target: "sign_verify" cxxflags: "-DDECRED_SECP256K1 -DSECP256K1 -DNBITCOIN_SECP256K1 -DRUST_K256 -DCUSTOM_MUTATOR_SECP256K1_SIGNATURE" @@ -219,8 +245,7 @@ jobs: asan_options: "detect_leaks=0:detect_container_overflow=0" - corpus: "bip32_deserialize_extended_key_clean" target: "bip32_deserialize_extended_key" - cxxflags: "-DNBITCOIN -DRUST_BITCOIN -DBITCOIN_CORE -DBITCOINJ -DLIBWALLY_CORE -DEMBIT -DPYCOIN -DBITCOINS -DBTCD -DCUSTOM_MUTATOR_EXTENDED_KEY" - cxxflags: "-DNBITCOIN -DRUST_BITCOIN -DBITCOIN_CORE -DBITCOINJ -DLIBWALLY_CORE -DEMBIT -DPYCOIN -DBITCOINS -DCUSTOM_MUTATOR_EXTENDED_KEY -DLIBBITCOIN_SYSTEM" + cxxflags: "-DNBITCOIN -DRUST_BITCOIN -DBITCOIN_CORE -DBITCOINJ -DLIBWALLY_CORE -DELECTRUM -DEMBIT -DPYCOIN -DBITCOINS -DBTCD -DCUSTOM_MUTATOR_EXTENDED_KEY -DLIBBITCOIN_SYSTEM" asan_options: "detect_leaks=0:detect_container_overflow=0" setup_java: "true" - corpus: "transaction_eval" @@ -229,10 +254,10 @@ jobs: asan_options: "detect_leaks=0:detect_container_overflow=0" - corpus: "decode_ellswift" target: "decode_ellswift" - cxxflags: "-DBTCD -DSECP256K1" + cxxflags: "-DBTCD -DSECP256K1 -DRUST_BITCOIN" - corpus: "roundtrip_ellswift" target: "roundtrip_ellswift" - cxxflags: "-DBTCD -DSECP256K1" + cxxflags: "-DBTCD -DSECP256K1 -DRUST_BITCOIN" asan_options: "detect_leaks=0:detect_container_overflow=0" - corpus: "schnorr_verify" target: "schnorr_verify" @@ -253,22 +278,26 @@ jobs: - corpus: "musig2_key_agg" target: "musig2_key_agg" cxxflags: "-DRUST_MUSIG2 -DSECP256K1" - asan_options: "detect_leaks=0:detect_container_overflow=0" + asan_options: "detect_leaks=0:detect_container_overflow=0" + - corpus: "aes256_cbc" + target: "aes256_cbc" + cxxflags: "-DBITCOIN_CORE -DRUSTCRYPTO_AES" + - corpus: "musig2_sign_session" + target: "musig2_sign_session" + cxxflags: "-DRUST_MUSIG2 -DSECP256K1" + asan_options: "detect_leaks=0:detect_container_overflow=0" + - corpus: "silentpayments_create_outputs" + target: "silentpayments_create_outputs" + cxxflags: "-DBDK_SP -DBLUEWALLET_SP -DSPDK -DSECP256K1" + asan_options: "detect_leaks=0:detect_container_overflow=0" + steps: - name: Checkout repo uses: actions/checkout@v4 - uses: ./.github/actions/setup-test-env with: - os: ${{ matrix.os }} - github-sha: ${{ github.sha }} - github-run-id: ${{ github.run_id }} - - - name: Load environment variables - run: | - if [ -f build.env ]; then - cat build.env >> $GITHUB_ENV - fi + bitcoinfuzz-ref: ${{ needs.build.outputs.bitcoinfuzz-sha }} - name: Pre-build step if: matrix.test.pre_build != '' @@ -285,4 +314,12 @@ jobs: - name: Test - ${{ matrix.test.target }} timeout-minutes: 5 run: | - FUZZ=${{ matrix.test.target }} ./.github/scripts/run-fuzz.sh "${{ matrix.test.corpus }}" "${{ matrix.test.target }}" "${{ matrix.test.cxxflags }}" "${{ matrix.test.asan_options }}" "$CXX" + export CC=/usr/bin/clang + export CXX=/usr/bin/clang++ + FUZZ=${{ matrix.test.target }} ./.github/scripts/run-fuzz.sh \ + "${{ matrix.test.corpus }}" \ + "${{ matrix.test.target }}" \ + "${{ matrix.test.cxxflags }}" \ + "${{ matrix.test.asan_options }}" \ + "$CXX" \ + "-max_len=32768"