Fix search-replica routing fairness and add VSCO 3.8.0 image publish workflow

.github/workflows/vsco-publish-opensearch-image.yaml

@@ -0,0 +1,246 @@
+# =============================================================================
+# VSCO FORK ONLY — REMOVE BEFORE OPENING A PR TO opensearch-project/OpenSearch
+#
+# Publishes a patched OpenSearch image to:
+#   ghcr.io/vsco/opensearch:<tag>          e.g. 3.8.0-vsco1
+# (same tag pattern as ome: 0.1.5-vsco7)
+#
+# Approach: derive from the official opensearchproject/opensearch:<version>
+# image and replace only the core jar (lib/opensearch-<version>.jar) built from
+# this fork.
+#
+# Why not :distribution:docker:buildDockerImage — that Gradle task produces the
+# *min* distribution: /usr/share/opensearch/plugins is empty, so there is no
+# opensearch-security (the operator's HTTPS + basic-auth probes never pass) and
+# no opensearch-knn. The Docker Hub image is assembled separately by
+# opensearch-build and bundles ~26 plugins. It is also single-arch, built for
+# whatever the runner happens to be.
+#
+# distribution/build.gradle puts `libs project(':server')` into lib/, so
+# swapping opensearch-<version>.jar is equivalent to rebuilding the
+# distribution with this fork's patch — provided this fork stays at the
+# <version> tag plus core-only changes. The jar is pure bytecode, so one build
+# serves every platform and buildx emits a real multi-arch manifest list off
+# the official multi-arch base (no QEMU needed: the Dockerfile only COPYs).
+#
+# Usage:
+#   Actions → "VSCO (fork) publish OpenSearch image to ghcr.io/vsco" → Run workflow
+#
+# Auth: GITHUB_TOKEN usually works for packages:write in the vsco org.
+#       If pushes fail with 403, add repo secret VSCO_GHCR_TOKEN (PAT with
+#       write:packages).
+# =============================================================================
+
+name: VSCO (fork) publish OpenSearch image to ghcr.io/vsco
+
+on:
+  workflow_dispatch:
+    inputs:
+      image_tag:
+        description: 'Tag for ghcr.io/vsco/opensearch (e.g. 3.8.0-vsco3)'
+        required: true
+        default: '3.8.0-vsco3'
+      java_version:
+        description: 'JDK major version for the Gradle build'
+        required: true
+        default: '21'
+
+permissions:
+  contents: read
+  packages: write
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_ORG: vsco
+  IMAGE_NAME: opensearch
+  BASE_REPO: opensearchproject/opensearch
+  UPSTREAM_REPO: opensearch-project/OpenSearch
+  PLATFORMS: linux/amd64,linux/arm64
+
+jobs:
+  build-push-opensearch:
+    runs-on: self-hosted
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v6
+
+      - name: Install protoc
+        run: |
+          set -euo pipefail
+          if [ "$(uname -m)" = "x86_64" ]; then
+            curl -fsSL -X GET "https://github.com/protocolbuffers/protobuf/releases/download/v33.0/protoc-33.0-linux-x86_64.zip" -o protoc.zip
+          else
+            curl -fsSL -X GET "https://github.com/protocolbuffers/protobuf/releases/download/v33.0/protoc-33.0-linux-aarch_64.zip" -o protoc.zip
+          fi
+          sudo unzip -o protoc.zip -d /usr/local && rm -f protoc.zip
+          protoc --version
+
+      - name: Set up JDK
+        uses: actions/setup-java@v5
+        with:
+          java-version: ${{ inputs.java_version }}
+          distribution: temurin
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to ghcr.io (org vsco packages)
+        uses: docker/login-action@v4
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.VSCO_GHCR_TOKEN || secrets.GITHUB_TOKEN }}
+
+      - name: Resolve upstream OpenSearch version
+        id: version
+        run: |
+          set -euo pipefail
+          # buildSrc/version.properties: opensearch = 3.8.0
+          OS_VERSION="$(awk -F= '/^opensearch[[:space:]]*=/ { gsub(/[[:space:]]/, "", $2); print $2; exit }' buildSrc/version.properties)"
+          if [[ -z "${OS_VERSION}" ]]; then
+            echo "Could not read opensearch version from buildSrc/version.properties" >&2
+            exit 1
+          fi
+          echo "opensearch_version=${OS_VERSION}" >> "$GITHUB_OUTPUT"
+          echo "Building OpenSearch ${OS_VERSION}; publishing as ${{ inputs.image_tag }}"
+
+      - name: Validate image tag matches base release
+        run: |
+          set -euo pipefail
+          TAG="${{ inputs.image_tag }}"
+          BASE="${{ steps.version.outputs.opensearch_version }}"
+          case "${TAG}" in
+            "${BASE}-vsco"*) ;;
+            *)
+              echo "image_tag '${TAG}' must start with '${BASE}-vsco' (e.g. ${BASE}-vsco1)" >&2
+              exit 1
+              ;;
+          esac
+
+      - name: Verify fork is <version> tag plus core-only changes
+        run: |
+          set -euo pipefail
+          BASE="${{ steps.version.outputs.opensearch_version }}"
+          # Anchor on upstream's tag, not the fork's: the base image was built from
+          # upstream ${BASE}, and a fork made with "copy the default branch only"
+          # carries no release tags (3.8.0 lives on the 3.8 branch, not main).
+          git fetch --no-tags --depth=1 "https://github.com/${{ env.UPSTREAM_REPO }}.git" \
+            "refs/tags/${BASE}:refs/tags/upstream-${BASE}"
+          # Swapping only lib/opensearch-<version>.jar is safe only while every
+          # change since the tag lands in :server (or is fork-only tooling).
+          CHANGED="$(git diff --name-only "refs/tags/upstream-${BASE}..HEAD" \
+            | grep -v -E '^(server/|\.github/workflows/vsco-)' || true)"
+          if [[ -n "${CHANGED}" ]]; then
+            echo "Changes outside server/ since upstream tag ${BASE}; a core-jar swap would silently drop them:" >&2
+            echo "${CHANGED}" >&2
+            exit 1
+          fi
+          git diff --stat "refs/tags/upstream-${BASE}..HEAD"
+
+      - name: Build patched core jar (Gradle)
+        run: |
+          set -euo pipefail
+          # -Dbuild.snapshot=false is how opensearch-build produces a release: without
+          # it the artifact is opensearch-<version>-SNAPSHOT.jar and its manifest
+          # advertises a snapshot, which would not match the release plugins bundled
+          # in the base image.
+          ./gradlew :server:jar --parallel -Dbuild.snapshot=false
+
+      - name: Stage build context
+        id: stage
+        run: |
+          set -euo pipefail
+          OS_VERSION="${{ steps.version.outputs.opensearch_version }}"
+
+          # Locate the jar rather than assume server/build/libs: Gradle's layout has
+          # moved before, and a silently-wrong path here is what a bad image looks like.
+          JARS=()
+          mapfile -t JARS < <(find . -type f -name "opensearch-${OS_VERSION}.jar" \
+            -not -path './.git/*' | sort)
+          if [[ ${#JARS[@]} -ne 1 ]]; then
+            echo "Expected exactly one opensearch-${OS_VERSION}.jar, found ${#JARS[@]}." >&2
+            echo "--- every opensearch-*.jar in the workspace ---" >&2
+            find . -type f -name 'opensearch-*.jar' -not -path './.git/*' >&2 || true
+            echo "--- directories under server/ (depth 3) ---" >&2
+            find server -maxdepth 3 -type d >&2 || true
+            exit 1
+          fi
+          JAR="${JARS[0]}"
+          echo "Core jar: ${JAR}"
+          unzip -p "${JAR}" META-INF/MANIFEST.MF | grep -i 'version' || true
+
+          # Guard against grabbing some other artifact that happens to match the name.
+          if ! unzip -l "${JAR}" | grep -q 'org/opensearch/cluster/routing/IndexShardRoutingTable.class'; then
+            echo "${JAR} does not contain the patched class; wrong artifact." >&2
+            exit 1
+          fi
+
+          rm -rf vsco-image && mkdir -p vsco-image
+          cp "${JAR}" vsco-image/opensearch-core.jar
+
+          cat > vsco-image/Dockerfile <<'DOCKERFILE'
+          ARG BASE_IMAGE
+          FROM ${BASE_IMAGE}
+          ARG OS_VERSION
+          # Official image owns /usr/share/opensearch as opensearch(1000):root(0).
+          COPY --chown=1000:0 opensearch-core.jar /usr/share/opensearch/lib/opensearch-${OS_VERSION}.jar
+          DOCKERFILE
+
+          ls -la vsco-image/
+
+      - name: Verify base image layout
+        run: |
+          set -euo pipefail
+          OS_VERSION="${{ steps.version.outputs.opensearch_version }}"
+          BASE_IMAGE="${{ env.BASE_REPO }}:${OS_VERSION}"
+          docker pull "${BASE_IMAGE}"
+          # Confirm the jar we are about to overwrite actually exists, and that
+          # the plugins we depend on are bundled.
+          docker run --rm -e "OS_VERSION=${OS_VERSION}" --entrypoint /bin/bash "${BASE_IMAGE}" -c '
+            set -euo pipefail
+            test -f "/usr/share/opensearch/lib/opensearch-${OS_VERSION}.jar"
+            ls -d /usr/share/opensearch/plugins/opensearch-security
+            ls -d /usr/share/opensearch/plugins/opensearch-knn
+            echo "base image layout OK"
+          '
+
+      - name: Build and push multi-arch image
+        run: |
+          set -euo pipefail
+          OS_VERSION="${{ steps.version.outputs.opensearch_version }}"
+          DST="${{ env.REGISTRY }}/${{ env.IMAGE_ORG }}/${{ env.IMAGE_NAME }}:${{ inputs.image_tag }}"
+          docker buildx build \
+            --platform "${{ env.PLATFORMS }}" \
+            --build-arg "BASE_IMAGE=${{ env.BASE_REPO }}:${OS_VERSION}" \
+            --build-arg "OS_VERSION=${OS_VERSION}" \
+            --provenance=false \
+            --tag "${DST}" \
+            --push \
+            vsco-image
+          echo "Pushed ${DST}"
+
+      - name: Verify pushed manifest covers every platform
+        run: |
+          set -euo pipefail
+          DST="${{ env.REGISTRY }}/${{ env.IMAGE_ORG }}/${{ env.IMAGE_NAME }}:${{ inputs.image_tag }}"
+          docker buildx imagetools inspect "${DST}"
+          for platform in $(echo "${{ env.PLATFORMS }}" | tr ',' ' '); do
+            if ! docker buildx imagetools inspect "${DST}" | grep -q "${platform}"; then
+              echo "Pushed manifest is missing ${platform}" >&2
+              exit 1
+            fi
+          done
+
+      - name: Summary
+        run: |
+          {
+            echo "## VSCO OpenSearch image"
+            echo ""
+            echo "Base: \`${{ env.BASE_REPO }}:${{ steps.version.outputs.opensearch_version }}\`"
+            echo "Patched: \`lib/opensearch-${{ steps.version.outputs.opensearch_version }}.jar\` (built from this fork)"
+            echo "Platforms: \`${{ env.PLATFORMS }}\`"
+            echo "Pushed: \`${{ env.REGISTRY }}/${{ env.IMAGE_ORG }}/${{ env.IMAGE_NAME }}:${{ inputs.image_tag }}\`"
+            echo ""
+            echo "Point the OpenSearch Helm/operator image to that tag."
+            echo "Remove \`.github/workflows/vsco-publish-opensearch-image.yaml\` before submitting a PR upstream."
+          } >> "$GITHUB_STEP_SUMMARY"

buildSrc/build.gradle

@@ -100,6 +100,12 @@ repositories {
       excludeGroupByRegex "adoptium.*|adoptopenjdk.*|openjdk.*"
     }
   }
+  maven {
+    url = uri("https://ci.opensearch.org/m2/")
+    content {
+      excludeGroupByRegex "adoptium.*|adoptopenjdk.*|openjdk.*"
+    }
+  }
   mavenCentral {
     content {
       excludeGroupByRegex "adoptium.*|adoptopenjdk.*|openjdk.*"

buildSrc/src/main/java/org/opensearch/gradle/RepositoriesSetupPlugin.java

@@ -87,6 +87,11 @@ public static void configureRepositories(Project project) {
             repo.setUrl("https://ci.opensearch.org/maven2/");
             repo.content(descriptor -> descriptor.excludeGroupByRegex("adoptium.*|adoptopenjdk.*|openjdk.*"));
         });
+        repos.maven(repo -> {
+            repo.setName("Plugin Mirror");
+            repo.setUrl("https://ci.opensearch.org/m2/");
+            repo.content(descriptor -> descriptor.excludeGroupByRegex("adoptium.*|adoptopenjdk.*|openjdk.*"));
+        });
         repos.mavenCentral(repo -> { repo.content(descriptor -> descriptor.excludeGroupByRegex("adoptium.*|adoptopenjdk.*|openjdk.*")); });
 
         String luceneVersion = VersionProperties.getLucene();

buildSrc/src/main/java/org/opensearch/gradle/test/DistroTestPlugin.java

@@ -77,9 +77,9 @@
 import java.util.stream.Stream;
 
 public class DistroTestPlugin implements Plugin<Project> {
-    private static final String SYSTEM_JDK_VERSION = "25.0.3+9";
+    private static final String SYSTEM_JDK_VERSION = "25.0.4+7";
     private static final String SYSTEM_JDK_VENDOR = "adoptium";
-    private static final String GRADLE_JDK_VERSION = "25.0.3+9";
+    private static final String GRADLE_JDK_VERSION = "25.0.4+7";
     private static final String GRADLE_JDK_VENDOR = "adoptium";
 
     // all distributions used by distro tests. this is temporary until tests are per distribution

gradle/code-coverage.gradle

@@ -15,6 +15,12 @@ repositories {
       excludeGroupByRegex "adoptium.*|adoptopenjdk.*|openjdk.*"
     }
   }
+  maven {
+    url = uri("https://ci.opensearch.org/m2/")
+    content {
+      excludeGroupByRegex "adoptium.*|adoptopenjdk.*|openjdk.*"
+    }
+  }
   mavenCentral {
     content {
       excludeGroupByRegex "adoptium.*|adoptopenjdk.*|openjdk.*"

gradle/ide.gradle

@@ -20,6 +20,9 @@ buildscript {
     maven {
       url = "https://ci.opensearch.org/maven2/"
     }
+    maven {
+      url = "https://ci.opensearch.org/m2/"
+    }
     maven {
       url = "https://plugins.gradle.org/m2/"
     }

gradle/libs.versions.toml

@@ -3,7 +3,7 @@ opensearch        = "3.8.0"
 lucene            = "10.5.0"
 
 bundled_jdk_vendor = "adoptium"
-bundled_jdk = "25.0.3+9"
+bundled_jdk = "25.0.4+7"
 
 # optional dependencies
 spatial4j               = "0.7"
@@ -97,8 +97,8 @@ jzlib             = "1.1.3"
 resteasy          = "6.2.4.Final"
 
 # opentelemetry dependencies
-opentelemetry         = "1.63.0"
-opentelemetrysemconv  = "1.41.0"
+opentelemetry         = "1.64.0"
+opentelemetrysemconv  = "1.43.0"
 
 # arrow dependencies
 arrow                 = "18.1.0"

plugins/repository-gcs/build.gradle

@@ -90,8 +90,9 @@ dependencies {
 
   implementation "org.checkerframework:checker-qual:3.52.1"
 
-  runtimeOnly "io.opentelemetry:opentelemetry-api:1.47.0"
-  runtimeOnly "io.opentelemetry:opentelemetry-context:1.47.0"
+  runtimeOnly libs.opentelemetry.api
+  runtimeOnly libs.opentelemetry.common
+  runtimeOnly libs.opentelemetry.context
   runtimeOnly "com.google.api.grpc:proto-google-cloud-storage-v2:2.60.0"
   runtimeOnly "io.grpc:grpc-api:1.71.0"
 
@@ -137,6 +138,7 @@ tasks.named("dependencyLicenses").configure {
   mapping from: /google-auth-.*/, to: 'google-auth'
   mapping from: /google-http-.*/, to: 'google-http'
   mapping from: /opencensus.*/, to: 'opencensus'
+  mapping from: /opentelemetry-common.*/, to: 'opentelemetry-api'
   mapping from: /protobuf.*/, to: 'protobuf'
   mapping from: /proto-google.*/, to: 'proto-google'
 }

plugins/repository-gcs/licenses/opentelemetry-api-1.64.0.jar.sha1

@@ -0,0 +1 @@
+35d317f2526758575613f4b148b8c70241f7f175

plugins/repository-gcs/licenses/opentelemetry-common-1.64.0.jar.sha1

@@ -0,0 +1 @@
+baae45914b5c233d8da972fc9d4a3ad811322ed8

plugins/repository-gcs/licenses/opentelemetry-context-1.64.0.jar.sha1

@@ -0,0 +1 @@
+04f60a721d458642983d4df7bd7d179287919dd0

plugins/telemetry-otel/licenses/opentelemetry-api-1.64.0.jar.sha1

@@ -0,0 +1 @@
+35d317f2526758575613f4b148b8c70241f7f175

plugins/telemetry-otel/licenses/opentelemetry-api-incubator-1.64.0-alpha.jar.sha1

@@ -0,0 +1 @@
+79cbdcb115c92d8c29c757659abf680419350590

plugins/telemetry-otel/licenses/opentelemetry-common-1.64.0.jar.sha1

@@ -0,0 +1 @@
+baae45914b5c233d8da972fc9d4a3ad811322ed8

plugins/telemetry-otel/licenses/opentelemetry-context-1.64.0.jar.sha1

@@ -0,0 +1 @@
+04f60a721d458642983d4df7bd7d179287919dd0

plugins/telemetry-otel/licenses/opentelemetry-exporter-common-1.64.0.jar.sha1

@@ -0,0 +1 @@
+ee714eb3ac9ed82d3370bc8fd2f44216f19268dd

plugins/telemetry-otel/licenses/opentelemetry-exporter-logging-1.64.0.jar.sha1

@@ -0,0 +1 @@
+596108744459e001f71346e948f8c97dfec38f30