coverage.yml

name: Code coverage

on:
  workflow_call:
    inputs:
      sha:
        required: true
        type: string

permissions:
  contents: read

env:
  AWS_REGION: us-east-1
  AWS_ACCESS_KEY_ID: "placeholder"
  AWS_SECRET_ACCESS_KEY: "placeholder"
  CARGO_INCREMENTAL: 0
  PUBSUB_EMULATOR_HOST: "localhost:8681"
  QW_S3_ENDPOINT: "http://localhost:4566" # Services are exposed as localhost because we are not running coverage in a container.
  QW_S3_FORCE_PATH_STYLE_ACCESS: 1
  QW_TEST_DATABASE_URL: postgres://quickwit-dev:quickwit-dev@localhost:5432/quickwit-metastore-dev
  RUSTFLAGS: -Dwarnings --cfg tokio_unstable

jobs:
  test:
    name: Coverage
    runs-on: gh-ubuntu-arm64
    timeout-minutes: 40
    permissions:
      contents: read
    # Setting a containing will require to fix the QW_S3_ENDPOINT to http://localstack:4566
    services:
      localstack:
        image: localstack/localstack:4.14.0
        ports:
          - "4566:4566"
          - "4571:4571"
          - "8080:8080"
        env:
          SERVICES: kinesis,s3,sqs
        options: >-
          --health-cmd "curl -k https://localhost:4566"
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5

      postgres:
        image: postgres:latest
        ports:
          - "5432:5432"
        env:
          POSTGRES_USER: quickwit-dev
          POSTGRES_PASSWORD: quickwit-dev
          POSTGRES_DB: quickwit-metastore-dev
        options: >-
          --health-cmd pg_isready
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5

      kafka-broker:
        image: confluentinc/confluent-local:7.4.11
        ports:
          - "9092:9092"
          - "9101:9101"
        env:
          # Mode KRaft (Single Node)
          KAFKA_NODE_ID: 1
          KAFKA_PROCESS_ROLES: 'broker,controller'
          KAFKA_CONTROLLER_QUORUM_VOTERS: '1@localhost:9093'
          KAFKA_LOG4J_LOGGERS: "org.apache.kafka.image.loader.MetadataLoader=WARN"

          # Listeners
          KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT'
          KAFKA_LISTENERS: 'EXTERNAL://0.0.0.0:9092,CONTROLLER://0.0.0.0:9093'
          KAFKA_ADVERTISED_LISTENERS: 'EXTERNAL://localhost:9092'
          KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER'
          KAFKA_INTER_BROKER_LISTENER_NAME: 'EXTERNAL'

          # Configuration simplifiée
          KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
          KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
          KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
          KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1

          # ID du Cluster (Nécessaire pour KRaft)
          CLUSTER_ID: 'MkU3OEVBNTcwNTJENDM2Qk'

          KAFKA_HEAP_OPTS: -Xms256M -Xmx256M

        options: >-
          --health-cmd "ub kafka-ready -b localhost:9092 1 5"
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5

      gcp-pubsub-emulator:
        image: thekevjames/gcloud-pubsub-emulator:550.0.0
        ports:
          - "8681:8681"
        env:
          PUBSUB_PROJECT1: "quickwit-emulator,emulator_topic:emulator_subscription"

    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          ref: ${{ inputs.sha }}
          persist-credentials: false

      - name: Install lib libsasl2
        run: |
          sudo apt update
          sudo apt install libsasl2-dev
          sudo apt install libsasl2-2
          sudo apt install libcurl4-openssl-dev

      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v.7.0.0
        with:
          python-version: '3.11'

      - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
        with:
          path: |
            ~/.cargo/git
            ~/.cargo/registry
          key: ${{ runner.os }}-cargo-test-${{ hashFiles('quickwit/Cargo.lock') }}
          restore-keys: |
            ${{ runner.os }}-cargo-test-${{ hashFiles('quickwit/Cargo.lock') }}
            ${{ runner.os }}-cargo-test

      - name: Install python packages
        run: |
          pip install --user --require-hashes -r ${{ github.workspace }}/.github/workflows/requirements.txt
          pipenv install --deploy --ignore-pipfile
        working-directory: ./quickwit/quickwit-cli/tests

      - name: Prepare LocalStack S3
        run: pipenv run ./prepare_tests.sh
        working-directory: ./quickwit/quickwit-cli/tests

      # GitHub Actions does not allow services to be started with a custom command,
      # so we are running Azurite as a container manually.
      - name: Run Azurite service
        run: DOCKER_SERVICES=azurite make docker-compose-up

      # GitHub Actions does not allow services to be started with a custom command,
      # so we are running fake gcs server as a container manually.
      - name: Run Fake GCS Server service
        run: DOCKER_SERVICES=fake-gcs-server make docker-compose-up

      - name: Run Pulsar service
        run: DOCKER_SERVICES=pulsar make docker-compose-up

      - name: Install Rust
        run: rustup toolchain install
        working-directory: ./quickwit

      - name: Install cargo-llvm-cov, cargo-nextest, and protoc
        uses: taiki-e/install-action@83ac0ad63c0167e6f06796fab0fce28db1bf3db0 # v2.87.22
        with:
          tool: cargo-llvm-cov,nextest,protoc

      - name: Ensure coverage Rust toolchain is complete
        run: |
          needs_reinstall=false
          if ! rustc --version >/dev/null 2>&1 || ! cargo --version >/dev/null 2>&1; then
            needs_reinstall=true
          else
            if ! target_libdir=$(rustc --print target-libdir 2>/dev/null) ||
               ! compgen -G "$target_libdir/libstd-*.rlib" >/dev/null; then
              needs_reinstall=true
            fi
          fi
          if [ "$needs_reinstall" = true ]; then
            # A runner can have a partial installation that rustup considers installed.
            active_toolchain=$(rustup show active-toolchain | cut -d ' ' -f 1)
            rustup toolchain uninstall "$active_toolchain"
            rustup toolchain install
          fi
          rustup component add rustc cargo llvm-tools-preview
          rustup show active-toolchain
          rustup component list --installed
          rustc --version
          cargo --version
          target_libdir=$(rustc --print target-libdir)
          compgen -G "$target_libdir/libstd-*.rlib" >/dev/null
        working-directory: ./quickwit

      # We limit the number of jobs to 4 to avoid OOM errors when linking the binary.
      - name: Generate code coverage
        run: |
          cargo llvm-cov clean --workspace
          cargo llvm-cov nextest --no-report --test failpoints --features fail/failpoints --retries 4
          # increase stack size for test_all_with_s3_localstack_cli, see quickwit#4963
          RUST_MIN_STACK=67108864 CARGO_BUILD_JOBS=4 cargo llvm-cov nextest --no-report --all-features --retries 4
          cargo llvm-cov report --lcov --output-path lcov.info
        working-directory: ./quickwit

      - name: Upload coverage report artifact
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: coverage-report-${{ inputs.sha }}
          path: quickwit/lcov.info
          if-no-files-found: error
          overwrite: true
          retention-days: 7