From ff01cea88439343c8e72289ef0dd2b8600b4e60e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Wed, 6 May 2026 10:18:33 +0200 Subject: [PATCH 1/9] ci: add setup-key auth instrumentation test Adds SetupKeyAuthTest, a UiAutomator-driven instrumentation test that drives the in-app "Change server" screen end-to-end with a setup key and waits for the success dialog. The test taps "Use NetBird", so the management URL is the one hard-coded in the app (Preferences.defaultServer()). The setup key comes from an instrumentation runner argument so CI can inject it as a secret without baking it into the APK. Wires it into the existing build-debug workflow as a workflow_dispatch-only job that reuses the netbird-aar artifact, so PR builds are unaffected and the AAR is built only once per run. Required repo config: - Secret: INSTRUMENTATION_NB_SETUP_KEY (UUID, ideally reusable + ephemeral) --- .github/workflows/build-debug.yml | 17 +- .../io/netbird/client/SetupKeyAuthTest.java | 178 ++++++++++++++++++ 2 files changed, 194 insertions(+), 1 deletion(-) create mode 100644 app/src/androidTest/java/io/netbird/client/SetupKeyAuthTest.java diff --git a/.github/workflows/build-debug.yml b/.github/workflows/build-debug.yml index 936762e4..321ce95b 100644 --- a/.github/workflows/build-debug.yml +++ b/.github/workflows/build-debug.yml @@ -5,6 +5,7 @@ on: push: branches: - main + workflow_dispatch: permissions: contents: read @@ -87,7 +88,10 @@ jobs: instrumented-tests: needs: build-debug + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository runs-on: ubuntu-latest + timeout-minutes: 30 + environment: instrumentation-test-secrets steps: - name: Checkout repository uses: actions/checkout@v4 @@ -107,6 +111,15 @@ jobs: name: netbird-aar path: gomobile + - name: Verify required secrets + env: + INSTRUMENTATION_NB_SETUP_KEY: ${{ secrets.INSTRUMENTATION_NB_SETUP_KEY }} + run: | + if [ -z "$INSTRUMENTATION_NB_SETUP_KEY" ]; then + echo "::error::INSTRUMENTATION_NB_SETUP_KEY repository secret is not configured" + exit 1 + fi + - name: Enable KVM group perms run: | echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules @@ -115,6 +128,8 @@ jobs: - name: Run instrumented tests uses: reactivecircus/android-emulator-runner@v2 + env: + INSTRUMENTATION_NB_SETUP_KEY: ${{ secrets.INSTRUMENTATION_NB_SETUP_KEY }} with: api-level: 30 target: google_apis @@ -123,7 +138,7 @@ jobs: disk-size: 4096M heap-size: 512M disable-animations: true - script: ./gradlew connectedDebugAndroidTest --no-daemon -Pandroid.testInstrumentationRunnerArguments.notClass=io.netbird.client.NetworkConnectivityStressTest + script: ./gradlew --no-daemon connectedDebugAndroidTest -Pandroid.testInstrumentationRunnerArguments.notClass=io.netbird.client.NetworkConnectivityStressTest -Pandroid.testInstrumentationRunnerArguments.setupKey="$INSTRUMENTATION_NB_SETUP_KEY" - name: Upload test results if: always() diff --git a/app/src/androidTest/java/io/netbird/client/SetupKeyAuthTest.java b/app/src/androidTest/java/io/netbird/client/SetupKeyAuthTest.java new file mode 100644 index 00000000..bb973969 --- /dev/null +++ b/app/src/androidTest/java/io/netbird/client/SetupKeyAuthTest.java @@ -0,0 +1,178 @@ +package io.netbird.client; + +import android.os.Bundle; +import android.util.Log; +import android.view.View; + +import java.io.File; + +import androidx.navigation.NavController; +import androidx.navigation.NavOptions; +import androidx.navigation.Navigation; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import androidx.test.platform.app.InstrumentationRegistry; +import androidx.test.rule.ActivityTestRule; +import androidx.test.uiautomator.By; +import androidx.test.uiautomator.UiDevice; +import androidx.test.uiautomator.UiObject2; +import androidx.test.uiautomator.Until; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; + +import io.netbird.client.ui.server.ChangeServerFragment; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * Drives the "Change server" UI to authenticate against the default NetBird + * management server with a setup key — exactly the flow a user would use, but + * automated. + * + *

The setup key is read from an instrumentation runner argument so CI can + * inject it as a secret without baking it into the APK: + * 

+ *   ./gradlew connectedDebugAndroidTest \
+ *     -Pandroid.testInstrumentationRunnerArguments.setupKey=<UUID>
+ *
+ * + *

The test navigates straight to {@code nav_change_server} (skipping the + * first-install teaser screen), fills the setup key and taps the + * "Use NetBird" button, which uses the management URL hard-coded in the app + * ({@code Preferences.defaultServer()}). Then it waits for the success dialog. + */ +@RunWith(AndroidJUnit4.class) +public class SetupKeyAuthTest { + + private static final String TAG = "NBSetupKeyAuthTest"; + private static final String PACKAGE = "io.netbird.client"; + private static final long UI_TIMEOUT_MS = 5_000; + private static final long LOGIN_TIMEOUT_MS = 15_000; + + @SuppressWarnings("deprecation") + @Rule + public ActivityTestRule activityRule = + new ActivityTestRule<>(MainActivity.class, true, true); + + @Test + public void loginWithSetupKeyViaUi() throws Exception { + Bundle args = InstrumentationRegistry.getArguments(); + String setupKey = args.getString("setupKey"); + + assertNotNull("setupKey instrumentation argument is required", setupKey); + assertTrue("setupKey must not be blank", !setupKey.trim().isEmpty()); + + UiDevice device = UiDevice.getInstance(InstrumentationRegistry.getInstrumentation()); + device.waitForIdle(); + + // Try the navigation up to 2 times — on first launch the MainActivity + // pushes firstInstallFragment after onCreate, which can race with our + // navigate() call from the test thread. + UiObject2 setupKeyLabel = null; + for (int attempt = 1; attempt <= 2 && setupKeyLabel == null; attempt++) { + Log.i(TAG, "navigateToChangeServer attempt " + attempt); + navigateToChangeServer(); + dismissConfirmChangeServerDialog(device); + setupKeyLabel = device.wait( + Until.findObject(By.res(PACKAGE, "text_setup_key_label")), UI_TIMEOUT_MS); + } + if (setupKeyLabel == null) { + dumpScreenshot(device, "navigation-failed"); + fail("text_setup_key_label not found after 2 navigation attempts"); + } + setupKeyLabel.click(); + + UiObject2 setupKeyField = device.wait( + Until.findObject(By.res(PACKAGE, "edit_text_setup_key")), UI_TIMEOUT_MS); + assertNotNull("edit_text_setup_key must be present", setupKeyField); + setupKeyField.setText(setupKey.trim()); + + // "Use NetBird" submits with the app's default management URL. + UiObject2 submit = device.wait( + Until.findObject(By.res(PACKAGE, "btn_use_netbird")), UI_TIMEOUT_MS); + assertNotNull("btn_use_netbird must be present", submit); + submit.click(); + + // Either the success dialog ("btn_close") shows up, or the form re-enables + // itself with an error. + long deadline = System.currentTimeMillis() + LOGIN_TIMEOUT_MS; + while (System.currentTimeMillis() < deadline) { + UiObject2 closeBtn = device.findObject(By.res(PACKAGE, "btn_close")); + if (closeBtn != null) { + Log.i(TAG, "Setup-key login succeeded"); + closeBtn.click(); + return; + } + // If the "Change server" button is enabled again, the request came + // back with an error. + UiObject2 submitAgain = device.findObject(By.res(PACKAGE, "btn_change_server")); + if (submitAgain != null && submitAgain.isEnabled()) { + fail("Login failed: submit button re-enabled without success dialog"); + } + Thread.sleep(200); + } + dumpScreenshot(device, "login-timeout"); + fail("Login did not complete within " + (LOGIN_TIMEOUT_MS / 1000) + "s"); + } + + /** + * Skip the first-install teaser and jump straight to the "Change server" screen. + * {@code hideAlert=true} suppresses the "are you sure?" warning dialog so this + * is non-interactive. + */ + private void navigateToChangeServer() throws InterruptedException { + MainActivity activity = activityRule.getActivity(); + assertNotNull("MainActivity must be available", activity); + + activity.runOnUiThread(() -> { + View host = activity.findViewById(R.id.nav_host_fragment_content_main); + NavController nav = Navigation.findNavController(host); + Bundle bundle = new Bundle(); + bundle.putBoolean(ChangeServerFragment.HideAlertBundleArg, true); + // Same nav options the FirstInstallFragment uses when the user taps + // its "change_server" link, so we land in the same place. + NavOptions opts = new NavOptions.Builder() + .setPopUpTo(R.id.firstInstallFragment, true) + .build(); + nav.navigate(R.id.nav_change_server, bundle, opts); + }); + // Let the fragment transaction commit before UiAutomator looks for views. + Thread.sleep(1500); + } + + /** + * The Change Server fragment shows a "this will erase the local config" + * confirmation dialog whenever it opens — even when the caller passed + * {@code hideAlert=true}, because that arg is not currently honoured by + * the fragment. Tap Yes to dismiss so we can interact with the form. + */ + private static void dismissConfirmChangeServerDialog(UiDevice device) { + UiObject2 yes = device.wait( + Until.findObject(By.res(PACKAGE, "btn_yes")), UI_TIMEOUT_MS); + if (yes != null) { + Log.i(TAG, "Dismissing change-server confirmation dialog"); + yes.click(); + device.waitForIdle(); + } + } + + /** + * Take a screenshot via UiAutomator and write it into the test runner's + * working dir (cwd is /data/local/tmp/io.netbird.client.test on most + * devices, which `adb pull` can read). + */ + private static void dumpScreenshot(UiDevice device, String name) { + try { + File png = new File("/sdcard/Pictures/" + name + ".png"); + //noinspection ResultOfMethodCallIgnored + png.getParentFile().mkdirs(); + boolean ok = device.takeScreenshot(png); + Log.i(TAG, "Screenshot " + (ok ? "saved to " : "FAILED for ") + png); + } catch (Throwable t) { + Log.w(TAG, "Failed to dump screenshot: " + t.getMessage()); + } + } +} From 6c44c95df400c35fd532146953b5c3a3a950b735 Mon Sep 17 00:00:00 2001 From: Zoltan Papp Date: Sun, 10 May 2026 21:45:58 +0200 Subject: [PATCH 2/9] ci: record emulator screen during instrumented tests Run adb screenrecord in a background loop (180s segments) for the full duration of connectedDebugAndroidTest, then upload the segments as an artifact. Helps diagnose UiAutomator failures that only repro in CI. --- .github/workflows/build-debug.yml | 39 ++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-debug.yml b/.github/workflows/build-debug.yml index 321ce95b..a05aaa0f 100644 --- a/.github/workflows/build-debug.yml +++ b/.github/workflows/build-debug.yml @@ -138,7 +138,35 @@ jobs: disk-size: 4096M heap-size: 512M disable-animations: true - script: ./gradlew --no-daemon connectedDebugAndroidTest -Pandroid.testInstrumentationRunnerArguments.notClass=io.netbird.client.NetworkConnectivityStressTest -Pandroid.testInstrumentationRunnerArguments.setupKey="$INSTRUMENTATION_NB_SETUP_KEY" + script: | + set +e + mkdir -p screen-recordings + adb shell mkdir -p /sdcard/recordings + + # screenrecord caps each segment at 180s, so loop until we stop it. + ( + i=0 + while [ -f /tmp/record_active ]; do + seg=$(printf "seg_%03d.mp4" "$i") + adb shell screenrecord --time-limit 180 --bit-rate 4000000 /sdcard/recordings/$seg + i=$((i+1)) + done + ) & + REC_LOOP_PID=$! + touch /tmp/record_active + + ./gradlew --no-daemon connectedDebugAndroidTest \ + -Pandroid.testInstrumentationRunnerArguments.notClass=io.netbird.client.NetworkConnectivityStressTest \ + -Pandroid.testInstrumentationRunnerArguments.setupKey="$INSTRUMENTATION_NB_SETUP_KEY" + TEST_EXIT=$? + + rm -f /tmp/record_active + adb shell pkill -SIGINT screenrecord 2>/dev/null || true + wait $REC_LOOP_PID 2>/dev/null || true + sleep 3 + adb pull /sdcard/recordings ./screen-recordings/ || true + + exit $TEST_EXIT - name: Upload test results if: always() @@ -149,3 +177,12 @@ jobs: app/build/reports/androidTests/ tool/build/reports/androidTests/ retention-days: 3 + + - name: Upload screen recordings + if: always() + uses: actions/upload-artifact@v4 + with: + name: instrumented-test-screen-recordings + path: screen-recordings/ + if-no-files-found: warn + retention-days: 3 From 3692291d069c52f8e4bfb8a109cfd0f599cc80cb Mon Sep 17 00:00:00 2001 From: Zoltan Papp Date: Sun, 10 May 2026 21:57:46 +0200 Subject: [PATCH 3/9] ci: cache NDK/gomobile/AVD and move test runner to a script - setup-go: point cache-dependency-path to netbird/go.sum so the Go module cache actually restores (build-android-lib.sh shaves ~1-2 min). - Cache the NDK install dir and the gomobile binary by pinned version, skipping the corresponding install step on cache hit. - Cache the AVD plus the API 30 google_apis x86_64 system image, add a warm-up step that boots once to capture a snapshot, and run tests with force-avd-creation=false plus -no-snapshot-save so subsequent jobs load the cached snapshot without overwriting it. - Move the screenrecord/gradle/pull pipeline to .github/scripts/run-instrumented-tests.sh because the emulator-runner action executes each line of the inline script as a separate sh -c, which broke the multi-line subshell loop. --- .github/actions/build-android/action.yml | 17 +++++++ .github/scripts/run-instrumented-tests.sh | 35 ++++++++++++++ .github/workflows/build-debug.yml | 57 +++++++++++------------ 3 files changed, 80 insertions(+), 29 deletions(-) create mode 100755 .github/scripts/run-instrumented-tests.sh diff --git a/.github/actions/build-android/action.yml b/.github/actions/build-android/action.yml index a92c7efb..cdd59187 100644 --- a/.github/actions/build-android/action.yml +++ b/.github/actions/build-android/action.yml @@ -49,8 +49,17 @@ runs: uses: actions/setup-go@v5 with: go-version-file: "netbird/go.mod" + cache-dependency-path: "netbird/go.sum" + + - name: Cache Android NDK + id: ndk-cache + uses: actions/cache@v4 + with: + path: ${{ env.ANDROID_HOME }}/ndk/23.1.7779620 + key: ndk-23.1.7779620 - name: Setup NDK + if: steps.ndk-cache.outputs.cache-hit != 'true' shell: bash run: ${ANDROID_HOME}/cmdline-tools/latest/bin/sdkmanager --install "ndk;23.1.7779620" @@ -58,7 +67,15 @@ runs: shell: bash run: echo "ANDROID_NDK_HOME=${ANDROID_HOME}/ndk/23.1.7779620" >> $GITHUB_ENV + - name: Cache gomobile binary + id: gomobile-cache + uses: actions/cache@v4 + with: + path: ~/go/bin/gomobile + key: gomobile-v0.0.0-20251113184115-a159579294ab + - name: Install gomobile + if: steps.gomobile-cache.outputs.cache-hit != 'true' shell: bash run: go install golang.org/x/mobile/cmd/gomobile@v0.0.0-20251113184115-a159579294ab diff --git a/.github/scripts/run-instrumented-tests.sh b/.github/scripts/run-instrumented-tests.sh new file mode 100755 index 00000000..1d4548f1 --- /dev/null +++ b/.github/scripts/run-instrumented-tests.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# Runs connectedDebugAndroidTest while recording the emulator screen in the +# background. screenrecord caps each clip at 180s, so we loop until the test +# finishes, then upload the segments as an artifact from the workflow. +# +# Expects $INSTRUMENTATION_NB_SETUP_KEY in the environment. + +set +e + +mkdir -p screen-recordings +adb shell mkdir -p /sdcard/recordings + +( + i=0 + while [ -f /tmp/record_active ]; do + seg=$(printf "seg_%03d.mp4" "$i") + adb shell screenrecord --time-limit 180 --bit-rate 4000000 "/sdcard/recordings/$seg" + i=$((i + 1)) + done +) & +REC_LOOP_PID=$! +touch /tmp/record_active + +./gradlew --no-daemon connectedDebugAndroidTest \ + -Pandroid.testInstrumentationRunnerArguments.notClass=io.netbird.client.NetworkConnectivityStressTest \ + -Pandroid.testInstrumentationRunnerArguments.setupKey="$INSTRUMENTATION_NB_SETUP_KEY" +TEST_EXIT=$? + +rm -f /tmp/record_active +adb shell pkill -SIGINT screenrecord 2>/dev/null || true +wait "$REC_LOOP_PID" 2>/dev/null || true +sleep 3 +adb pull /sdcard/recordings ./screen-recordings/ || true + +exit $TEST_EXIT diff --git a/.github/workflows/build-debug.yml b/.github/workflows/build-debug.yml index a05aaa0f..6e1a4d6c 100644 --- a/.github/workflows/build-debug.yml +++ b/.github/workflows/build-debug.yml @@ -126,6 +126,31 @@ jobs: sudo udevadm control --reload-rules sudo udevadm trigger --name-match=kvm + - name: AVD cache + id: avd-cache + uses: actions/cache@v4 + with: + path: | + ~/.android/avd/* + ~/.android/adb* + ${{ env.ANDROID_HOME }}/system-images/android-30/google_apis/x86_64 + key: avd-api30-google_apis-x86_64-pixel_3a-v1 + + - name: Create AVD snapshot + if: steps.avd-cache.outputs.cache-hit != 'true' + uses: reactivecircus/android-emulator-runner@v2 + with: + api-level: 30 + target: google_apis + arch: x86_64 + profile: pixel_3a + disk-size: 4096M + heap-size: 512M + force-avd-creation: false + emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim + disable-animations: true + script: echo "Generated AVD snapshot for caching." + - name: Run instrumented tests uses: reactivecircus/android-emulator-runner@v2 env: @@ -137,36 +162,10 @@ jobs: profile: pixel_3a disk-size: 4096M heap-size: 512M + force-avd-creation: false + emulator-options: -no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim disable-animations: true - script: | - set +e - mkdir -p screen-recordings - adb shell mkdir -p /sdcard/recordings - - # screenrecord caps each segment at 180s, so loop until we stop it. - ( - i=0 - while [ -f /tmp/record_active ]; do - seg=$(printf "seg_%03d.mp4" "$i") - adb shell screenrecord --time-limit 180 --bit-rate 4000000 /sdcard/recordings/$seg - i=$((i+1)) - done - ) & - REC_LOOP_PID=$! - touch /tmp/record_active - - ./gradlew --no-daemon connectedDebugAndroidTest \ - -Pandroid.testInstrumentationRunnerArguments.notClass=io.netbird.client.NetworkConnectivityStressTest \ - -Pandroid.testInstrumentationRunnerArguments.setupKey="$INSTRUMENTATION_NB_SETUP_KEY" - TEST_EXIT=$? - - rm -f /tmp/record_active - adb shell pkill -SIGINT screenrecord 2>/dev/null || true - wait $REC_LOOP_PID 2>/dev/null || true - sleep 3 - adb pull /sdcard/recordings ./screen-recordings/ || true - - exit $TEST_EXIT + script: bash .github/scripts/run-instrumented-tests.sh - name: Upload test results if: always() From 173fd62e580c93d6d722fd44fda394df667ace54 Mon Sep 17 00:00:00 2001 From: Zoltan Papp Date: Sun, 10 May 2026 22:12:10 +0200 Subject: [PATCH 4/9] ci: fix cache path resolution and screenrecord loop race The ${{ env.ANDROID_HOME }} expression resolves against workflow-level env, not the runner's process environment, so the NDK and AVD system- image cache paths came out empty-prefixed and the saves were dropped with "Path Validation Error". Hardcode the ubuntu-latest sdk path (/usr/local/lib/android/sdk) for both. Also fix the screen recording: touch the sentinel file BEFORE launching the background loop, otherwise the loop sees no file on its first iteration and exits immediately, producing zero recordings. --- .github/actions/build-android/action.yml | 4 +++- .github/scripts/run-instrumented-tests.sh | 7 ++++++- .github/workflows/build-debug.yml | 5 ++++- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/.github/actions/build-android/action.yml b/.github/actions/build-android/action.yml index cdd59187..744839f3 100644 --- a/.github/actions/build-android/action.yml +++ b/.github/actions/build-android/action.yml @@ -55,7 +55,9 @@ runs: id: ndk-cache uses: actions/cache@v4 with: - path: ${{ env.ANDROID_HOME }}/ndk/23.1.7779620 + # ANDROID_HOME is set by the runner image but not visible to ${{ env.X }} + # in composite actions; the ubuntu-latest image pins it to this path. + path: /usr/local/lib/android/sdk/ndk/23.1.7779620 key: ndk-23.1.7779620 - name: Setup NDK diff --git a/.github/scripts/run-instrumented-tests.sh b/.github/scripts/run-instrumented-tests.sh index 1d4548f1..2c3009d6 100755 --- a/.github/scripts/run-instrumented-tests.sh +++ b/.github/scripts/run-instrumented-tests.sh @@ -10,16 +10,21 @@ set +e mkdir -p screen-recordings adb shell mkdir -p /sdcard/recordings +# Sentinel must exist before the background loop starts; otherwise the first +# iteration sees no file and exits immediately. +touch /tmp/record_active ( i=0 while [ -f /tmp/record_active ]; do seg=$(printf "seg_%03d.mp4" "$i") + echo "[record] starting $seg" adb shell screenrecord --time-limit 180 --bit-rate 4000000 "/sdcard/recordings/$seg" + echo "[record] $seg exited" i=$((i + 1)) done + echo "[record] loop ended" ) & REC_LOOP_PID=$! -touch /tmp/record_active ./gradlew --no-daemon connectedDebugAndroidTest \ -Pandroid.testInstrumentationRunnerArguments.notClass=io.netbird.client.NetworkConnectivityStressTest \ diff --git a/.github/workflows/build-debug.yml b/.github/workflows/build-debug.yml index 6e1a4d6c..359d6cf2 100644 --- a/.github/workflows/build-debug.yml +++ b/.github/workflows/build-debug.yml @@ -130,10 +130,13 @@ jobs: id: avd-cache uses: actions/cache@v4 with: + # ANDROID_HOME is set by the runner image but not visible to + # ${{ env.X }} at expression-eval time; hardcoded to the path on + # ubuntu-latest. path: | ~/.android/avd/* ~/.android/adb* - ${{ env.ANDROID_HOME }}/system-images/android-30/google_apis/x86_64 + /usr/local/lib/android/sdk/system-images/android-30/google_apis/x86_64 key: avd-api30-google_apis-x86_64-pixel_3a-v1 - name: Create AVD snapshot From c85852eabf8ad35f7411c38cbb6075a95754e80e Mon Sep 17 00:00:00 2001 From: Zoltan Papp Date: Sun, 10 May 2026 22:23:35 +0200 Subject: [PATCH 5/9] ci: stream adb logcat alongside the screen recording Save threadtime-format logcat to screen-recordings/logcat.log in the background, in parallel with the screenrecord loop, and stop it after the test exits. The existing screen-recordings/ artifact upload picks it up automatically. --- .github/scripts/run-instrumented-tests.sh | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/.github/scripts/run-instrumented-tests.sh b/.github/scripts/run-instrumented-tests.sh index 2c3009d6..21411b7b 100755 --- a/.github/scripts/run-instrumented-tests.sh +++ b/.github/scripts/run-instrumented-tests.sh @@ -1,7 +1,8 @@ #!/usr/bin/env bash -# Runs connectedDebugAndroidTest while recording the emulator screen in the -# background. screenrecord caps each clip at 180s, so we loop until the test -# finishes, then upload the segments as an artifact from the workflow. +# Runs connectedDebugAndroidTest while capturing the emulator screen and +# logcat in the background. screenrecord caps each clip at 180s, so we loop +# until the test finishes; logcat streams continuously to a file. Both end +# up in screen-recordings/ and are uploaded as an artifact from the workflow. # # Expects $INSTRUMENTATION_NB_SETUP_KEY in the environment. @@ -10,6 +11,10 @@ set +e mkdir -p screen-recordings adb shell mkdir -p /sdcard/recordings +adb logcat -c +adb logcat -v threadtime > screen-recordings/logcat.log 2>&1 & +LOGCAT_PID=$! + # Sentinel must exist before the background loop starts; otherwise the first # iteration sees no file and exits immediately. touch /tmp/record_active @@ -37,4 +42,7 @@ wait "$REC_LOOP_PID" 2>/dev/null || true sleep 3 adb pull /sdcard/recordings ./screen-recordings/ || true +kill "$LOGCAT_PID" 2>/dev/null || true +wait "$LOGCAT_PID" 2>/dev/null || true + exit $TEST_EXIT From 48a13762e8def5d5e0397a060262cd2345350ccb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Mon, 15 Jun 2026 23:55:18 +0200 Subject: [PATCH 6/9] test: port client-tests.robot e2e suite to Android instrumentation Add on-device instrumented tests mirroring the netbird-cloud client-tests.robot cases, driven through the real UI on a device/emulator: peer connectivity (with/without relay), port ACL enforcement, DNS resolution, and exit-node egress. - LoginFlow: shared change-server login + profile create/switch/remove UI flow, one-time force-relay disable, scroll/dialog helpers - VpnTestHarness: switchConnection+StateListener connect, ping/TCP/nslookup/HTTPS probes through the tunnel - E2eSuite/E2eAppRule: run the cases reusing a single live MainActivity - FailFast: abort the run after the first failure (Gradle UTP ignores failFast) - build.gradle.kts: read setup keys from env; run-instrumented-tests.sh wires them in and runs the suite with screen recording + logcat --- .github/scripts/run-instrumented-tests.sh | 14 +- .github/workflows/build-debug.yml | 1 + app/build.gradle.kts | 13 +- .../io/netbird/client/DnsResolutionTest.java | 127 ++++++ .../java/io/netbird/client/E2eAppRule.java | 73 ++++ .../java/io/netbird/client/E2eSuite.java | 26 ++ .../io/netbird/client/ExitNodeRouteTest.java | 115 ++++++ .../java/io/netbird/client/FailFast.java | 32 ++ .../java/io/netbird/client/LoginFlow.java | 365 ++++++++++++++++++ .../netbird/client/PeerConnectivityTest.java | 140 +++++++ .../java/io/netbird/client/PortAclTest.java | 164 ++++++++ .../io/netbird/client/SetupKeyAuthTest.java | 155 +------- .../io/netbird/client/VpnTestHarness.java | 276 +++++++++++++ 13 files changed, 1357 insertions(+), 144 deletions(-) create mode 100644 app/src/androidTest/java/io/netbird/client/DnsResolutionTest.java create mode 100644 app/src/androidTest/java/io/netbird/client/E2eAppRule.java create mode 100644 app/src/androidTest/java/io/netbird/client/E2eSuite.java create mode 100644 app/src/androidTest/java/io/netbird/client/ExitNodeRouteTest.java create mode 100644 app/src/androidTest/java/io/netbird/client/FailFast.java create mode 100644 app/src/androidTest/java/io/netbird/client/LoginFlow.java create mode 100644 app/src/androidTest/java/io/netbird/client/PeerConnectivityTest.java create mode 100644 app/src/androidTest/java/io/netbird/client/PortAclTest.java create mode 100644 app/src/androidTest/java/io/netbird/client/VpnTestHarness.java diff --git a/.github/scripts/run-instrumented-tests.sh b/.github/scripts/run-instrumented-tests.sh index 21411b7b..28c27258 100755 --- a/.github/scripts/run-instrumented-tests.sh +++ b/.github/scripts/run-instrumented-tests.sh @@ -9,6 +9,7 @@ set +e mkdir -p screen-recordings +adb shell rm -rf /sdcard/recordings adb shell mkdir -p /sdcard/recordings adb logcat -c @@ -31,9 +32,18 @@ touch /tmp/record_active ) & REC_LOOP_PID=$! -./gradlew --no-daemon connectedDebugAndroidTest \ - -Pandroid.testInstrumentationRunnerArguments.notClass=io.netbird.client.NetworkConnectivityStressTest \ +GRADLE_ARGS=( + --no-daemon :app:connectedDebugAndroidTest + -Pandroid.testInstrumentationRunnerArguments.class=io.netbird.client.E2eSuite + -Pandroid.testInstrumentationRunnerArguments.listener=io.netbird.client.FailFast -Pandroid.testInstrumentationRunnerArguments.setupKey="$INSTRUMENTATION_NB_SETUP_KEY" +) + +if [ -n "$INSTRUMENTATION_EXIT_NODE_SETUP_KEY" ]; then + GRADLE_ARGS+=(-Pandroid.testInstrumentationRunnerArguments.exitNodeSetupKey="$INSTRUMENTATION_EXIT_NODE_SETUP_KEY") +fi + +./gradlew "${GRADLE_ARGS[@]}" TEST_EXIT=$? rm -f /tmp/record_active diff --git a/.github/workflows/build-debug.yml b/.github/workflows/build-debug.yml index 359d6cf2..1c767776 100644 --- a/.github/workflows/build-debug.yml +++ b/.github/workflows/build-debug.yml @@ -158,6 +158,7 @@ jobs: uses: reactivecircus/android-emulator-runner@v2 env: INSTRUMENTATION_NB_SETUP_KEY: ${{ secrets.INSTRUMENTATION_NB_SETUP_KEY }} + INSTRUMENTATION_EXIT_NODE_SETUP_KEY: ${{ secrets.INSTRUMENTATION_EXIT_NODE_SETUP_KEY }} with: api-level: 30 target: google_apis diff --git a/app/build.gradle.kts b/app/build.gradle.kts index febc2632..75fa55da 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -45,7 +45,18 @@ android { versionName = rootProject.extra["appVersionName"] as String testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner" - testInstrumentationRunnerArguments["timeout_msec"] = "3600000" + testInstrumentationRunnerArguments["timeout_msec"] = "300000" + + // Pass setup keys from the environment to the e2e tests (a -P arg still overrides). + mapOf( + "setupKey" to "INSTRUMENTATION_NB_SETUP_KEY", + "exitNodeSetupKey" to "INSTRUMENTATION_EXIT_NODE_SETUP_KEY" + ).forEach { (arg, envVar) -> + val value = System.getenv(envVar) + if (!value.isNullOrBlank()) { + testInstrumentationRunnerArguments[arg] = value + } + } } buildTypes { diff --git a/app/src/androidTest/java/io/netbird/client/DnsResolutionTest.java b/app/src/androidTest/java/io/netbird/client/DnsResolutionTest.java new file mode 100644 index 00000000..41edfeff --- /dev/null +++ b/app/src/androidTest/java/io/netbird/client/DnsResolutionTest.java @@ -0,0 +1,127 @@ +package io.netbird.client; + +import android.os.Bundle; +import android.util.Log; + +import androidx.test.ext.junit.runners.AndroidJUnit4; +import androidx.test.platform.app.InstrumentationRegistry; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +/** + * DNS resolution test — the Android port of the Robot + * {@code client-tests.robot} case "Should resolve the domain and hostname". + * + *

With the tunnel up, the {@code dnstest} peer's internal name must resolve + * to its private address {@code 172.20.3.158} through the NetBird DNS. The + * original runs {@code dig }; here we run a real {@code nslookup} on the + * device (via the shell, like the ping tests), so it exercises the device + * resolver / VpnService DNS exactly as a user's traffic would. + * + *

Mirrors the two original assertions: + *

+ * + *

The original additionally checks the main system resolver is a NetBird + * {@code 100.x} address via {@code /etc/resolv.conf} / {@code resolvectl}. That + * is Linux-specific and not portable to Android (no {@code resolv.conf} in the + * usual sense; the VpnService owns DNS), so it is omitted — the resolution + * results above already prove tunnel DNS is in effect. + * + *

Only the setup key is injected: + * 

+ *   ./gradlew connectedDebugAndroidTest \
+ *     -Pandroid.testInstrumentationRunnerArguments.setupKey=<UUID>
+ *
+ */ +@RunWith(AndroidJUnit4.class) +public class DnsResolutionTest { + + private static final String TAG = "NBDnsTest"; + + /** dnstest peer's internal hostname and its expected private address. */ + private static final String PEER_FQDN = "ip-172-20-3-158.eu-central-1.compute.internal"; + private static final String PEER_UNQUALIFIED = "ip-172-20-3-158"; + private static final String EXPECTED_IP = "172.20.3.158"; + + /** Matches the Robot suite's peer-connected window (3 min). */ + private static final long CONNECT_TIMEOUT_SEC = 20; + /** Time budget for DNS to start resolving once the engine is connected. */ + private static final long RESOLVE_TIMEOUT_SEC = 20; + private VpnTestHarness harness; + private String profileName; + + @Before + + public void skipIfPreviousFailed() { + + FailFast.skipIfAborted(); + + } + + + @After + public void tearDown() throws Exception { + if (profileName != null && harness != null) { + harness.disableTouchVisualization(); + LoginFlow.removeProfile(E2eAppRule.activity(), harness.device(), profileName); + } + } + + @Test + public void resolvesPeerNameThroughTunnel() throws Exception { + Bundle args = InstrumentationRegistry.getArguments(); + String setupKey = args.getString("setupKey"); + + assertNotNull("setupKey instrumentation argument is required", setupKey); + assertTrue("setupKey must not be blank", !setupKey.trim().isEmpty()); + + MainActivity activity = E2eAppRule.activity(); + assertNotNull("MainActivity must be available", activity); + harness = new VpnTestHarness(activity); + harness.enableTouchVisualization(); + + harness.grantVpnConsent(); + LoginFlow.ensureForceRelayDisabled(activity, harness.device()); + + profileName = LoginFlow.createAndSwitchToFreshProfile(activity, harness.device(), "dns"); + LoginFlow.loginWithSetupKey(activity, harness.device(), setupKey); + + boolean connected = harness.connectAndAwait(CONNECT_TIMEOUT_SEC); + if (!connected) { + LoginFlow.dumpScreenshot(harness.device(), "vpn-connect-timeout"); + } + assertTrue("VPN did not reach connected state within " + CONNECT_TIMEOUT_SEC + "s", + connected); + + // FQDN resolves to the peer's private address through the tunnel DNS. + boolean fqdnResolved = harness.waitForResolve(PEER_FQDN, EXPECTED_IP, RESOLVE_TIMEOUT_SEC); + if (!fqdnResolved) { + LoginFlow.dumpScreenshot(harness.device(), "dns-fqdn-unresolved"); + } + assertTrue(PEER_FQDN + " did not resolve to " + EXPECTED_IP + " within " + + RESOLVE_TIMEOUT_SEC + "s", fqdnResolved); + + // Search-domain: the unqualified name resolves to the same address. + boolean searchResolved = + harness.waitForResolve(PEER_UNQUALIFIED, EXPECTED_IP, RESOLVE_TIMEOUT_SEC); + if (!searchResolved) { + LoginFlow.dumpScreenshot(harness.device(), "dns-search-unresolved"); + } + assertTrue(PEER_UNQUALIFIED + " (search domain) did not resolve to " + EXPECTED_IP + + " within " + RESOLVE_TIMEOUT_SEC + "s", searchResolved); + + Log.i(TAG, "DNS resolution verified: " + PEER_FQDN + " and " + PEER_UNQUALIFIED + + " -> " + EXPECTED_IP); + } +} diff --git a/app/src/androidTest/java/io/netbird/client/E2eAppRule.java b/app/src/androidTest/java/io/netbird/client/E2eAppRule.java new file mode 100644 index 00000000..79826dbc --- /dev/null +++ b/app/src/androidTest/java/io/netbird/client/E2eAppRule.java @@ -0,0 +1,73 @@ +package io.netbird.client; + +import android.app.Activity; +import android.app.Instrumentation; +import android.content.Intent; + +import androidx.test.platform.app.InstrumentationRegistry; +import androidx.test.runner.lifecycle.ActivityLifecycleMonitorRegistry; +import androidx.test.runner.lifecycle.Stage; + +import java.util.Collection; + +/** + * Provides the single shared {@link MainActivity} for the e2e suite WITHOUT a + * JUnit rule: {@link #activity()} returns the currently-resumed MainActivity, + * launching one if none is up. This survives the test framework finishing + * activities between test classes (which made a suite-level ActivityTestRule / + * ActivityScenario hand back a destroyed activity). Because the launch reuses + * the existing task, the app is not torn down and recreated between cases. + */ +final class E2eAppRule { + + private static final long LAUNCH_TIMEOUT_MS = 10_000; + + private E2eAppRule() { + } + + /** The running MainActivity, launched on demand if not already resumed. */ + static MainActivity activity() { + MainActivity existing = resumedMainActivity(); + if (existing != null) { + return existing; + } + + Instrumentation instrumentation = InstrumentationRegistry.getInstrumentation(); + Intent intent = new Intent(instrumentation.getTargetContext(), MainActivity.class) + .addFlags(Intent.FLAG_ACTIVITY_NEW_TASK); + instrumentation.startActivitySync(intent); + instrumentation.waitForIdleSync(); + + long deadline = System.currentTimeMillis() + LAUNCH_TIMEOUT_MS; + MainActivity activity = resumedMainActivity(); + while (activity == null && System.currentTimeMillis() < deadline) { + try { + Thread.sleep(100); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + break; + } + activity = resumedMainActivity(); + } + if (activity == null) { + throw new IllegalStateException("MainActivity did not reach RESUMED within " + + (LAUNCH_TIMEOUT_MS / 1000) + "s"); + } + return activity; + } + + private static MainActivity resumedMainActivity() { + MainActivity[] found = new MainActivity[1]; + InstrumentationRegistry.getInstrumentation().runOnMainSync(() -> { + Collection resumed = ActivityLifecycleMonitorRegistry.getInstance() + .getActivitiesInStage(Stage.RESUMED); + for (Activity a : resumed) { + if (a instanceof MainActivity) { + found[0] = (MainActivity) a; + break; + } + } + }); + return found[0]; + } +} diff --git a/app/src/androidTest/java/io/netbird/client/E2eSuite.java b/app/src/androidTest/java/io/netbird/client/E2eSuite.java new file mode 100644 index 00000000..279fab4b --- /dev/null +++ b/app/src/androidTest/java/io/netbird/client/E2eSuite.java @@ -0,0 +1,26 @@ +package io.netbird.client; + +import org.junit.runner.RunWith; +import org.junit.runners.Suite; + +/** + * Runs the on-device client e2e tests (the Android port of the Robot + * {@code client-tests.robot} suite). Tests obtain the shared {@link MainActivity} + * via {@link E2eAppRule#activity()}, which reuses the running activity (launching + * one only if none is up), so the app is not restarted between cases. + * + *

Run with: + * 

+ *   -Pandroid.testInstrumentationRunnerArguments.class=io.netbird.client.E2eSuite
+ *
+ */ +@RunWith(Suite.class) +@Suite.SuiteClasses({ + SetupKeyAuthTest.class, + PeerConnectivityTest.class, + PortAclTest.class, + DnsResolutionTest.class, + ExitNodeRouteTest.class, +}) +public class E2eSuite { +} diff --git a/app/src/androidTest/java/io/netbird/client/ExitNodeRouteTest.java b/app/src/androidTest/java/io/netbird/client/ExitNodeRouteTest.java new file mode 100644 index 00000000..5e08b5c7 --- /dev/null +++ b/app/src/androidTest/java/io/netbird/client/ExitNodeRouteTest.java @@ -0,0 +1,115 @@ +package io.netbird.client; + +import android.os.Bundle; +import android.util.Log; + +import androidx.test.ext.junit.runners.AndroidJUnit4; +import androidx.test.platform.app.InstrumentationRegistry; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +/** + * Exit-node route test — the Android port of the Robot + * {@code client-tests.robot} case "Should use exit node route". + * + *

The client logs in with a setup key whose profile routes all traffic + * through an exit node (the Robot suite's {@code EXIT_NODE_TEST_SETUP_KEY}). + * With the tunnel up, a request to {@code https://api.ipify.org} must report + * the exit node's public IP ({@code 3.121.38.77}) rather than the device's own + * — proving egress goes through the exit node. This is a real HTTPS request + * over the tunnel, the equivalent of the Robot {@code GET https://api.ipify.org}. + * + *

Unlike the other cases this needs a separate setup key (the + * exit-node profile), injected as its own argument — mirroring the original's + * distinct {@code EXIT_NODE_TEST_SETUP_KEY}: + * 

+ *   ./gradlew connectedDebugAndroidTest \
+ *     -Pandroid.testInstrumentationRunnerArguments.exitNodeSetupKey=<UUID>
+ *
+ * + *

If {@code exitNodeSetupKey} is not provided the test fails fast on its own + * assertion, so the rest of the suite is unaffected. + */ +@RunWith(AndroidJUnit4.class) +public class ExitNodeRouteTest { + + private static final String TAG = "NBExitNodeTest"; + + /** Where to ask for our public egress IP. */ + private static final String EGRESS_CHECK_URL = "https://api.ipify.org"; + /** The exit node's public IP — egress must appear to come from here. */ + private static final String EXIT_NODE_PUBLIC_IP = "3.121.38.77"; + + /** Matches the Robot suite's peer-connected window (3 min). */ + private static final long CONNECT_TIMEOUT_SEC = 20; + /** Time budget for the exit-node route to take effect after connecting. */ + private static final long EGRESS_TIMEOUT_SEC = 20; + private VpnTestHarness harness; + private String profileName; + + @Before + + public void skipIfPreviousFailed() { + + FailFast.skipIfAborted(); + + } + + + @After + public void tearDown() throws Exception { + if (profileName != null && harness != null) { + harness.disableTouchVisualization(); + LoginFlow.removeProfile(E2eAppRule.activity(), harness.device(), profileName); + } + } + + @Test + public void egressGoesThroughExitNode() throws Exception { + Bundle args = InstrumentationRegistry.getArguments(); + // Separate key from the basic tests, like the Robot EXIT_NODE_TEST_SETUP_KEY. + String setupKey = args.getString("exitNodeSetupKey"); + + assertNotNull("exitNodeSetupKey instrumentation argument is required", setupKey); + assertTrue("exitNodeSetupKey must not be blank", !setupKey.trim().isEmpty()); + + MainActivity activity = E2eAppRule.activity(); + assertNotNull("MainActivity must be available", activity); + harness = new VpnTestHarness(activity); + harness.enableTouchVisualization(); + + harness.grantVpnConsent(); + LoginFlow.ensureForceRelayDisabled(activity, harness.device()); + + profileName = LoginFlow.createAndSwitchToFreshProfile(activity, harness.device(), "exit-node"); + LoginFlow.loginWithSetupKey(activity, harness.device(), setupKey); + + boolean connected = harness.connectAndAwait(CONNECT_TIMEOUT_SEC); + if (!connected) { + LoginFlow.dumpScreenshot(harness.device(), "vpn-connect-timeout"); + } + assertTrue("VPN did not reach connected state within " + CONNECT_TIMEOUT_SEC + "s", + connected); + + // Best-effort: log the route table for debugging (the egress IP below + // is the real assertion). Mirrors the Robot route-table dump. + Log.i(TAG, "ip route:\n" + harness.shell("ip route show table all")); + + // The real check: our public egress IP must be the exit node's. + boolean viaExitNode = harness.waitForHttpBodyContains( + EGRESS_CHECK_URL, EXIT_NODE_PUBLIC_IP, EGRESS_TIMEOUT_SEC); + if (!viaExitNode) { + LoginFlow.dumpScreenshot(harness.device(), "exit-node-egress-mismatch"); + } + assertTrue("Egress IP from " + EGRESS_CHECK_URL + " was not the exit node's " + + EXIT_NODE_PUBLIC_IP + " within " + EGRESS_TIMEOUT_SEC + "s", viaExitNode); + + Log.i(TAG, "Egress verified through exit node " + EXIT_NODE_PUBLIC_IP); + } +} diff --git a/app/src/androidTest/java/io/netbird/client/FailFast.java b/app/src/androidTest/java/io/netbird/client/FailFast.java new file mode 100644 index 00000000..d81ba059 --- /dev/null +++ b/app/src/androidTest/java/io/netbird/client/FailFast.java @@ -0,0 +1,32 @@ +package io.netbird.client; + +import androidx.test.internal.runner.listener.InstrumentationRunListener; + +import org.junit.AssumptionViolatedException; +import org.junit.runner.Description; +import org.junit.runner.notification.Failure; + +/** + * Aborts the run after the first failure. The Gradle UTP layer ignores the + * {@code failFast} runner argument, so we do it inside the test process: this + * {@link org.junit.runner.notification.RunListener} (registered via the runner + * {@code listener} argument) flips a flag on the first failure, and + * {@link #skipIfAborted()} — called from each test's {@code @Before} — turns + * every subsequent test into a skipped (assumption-failed) result. + */ +public final class FailFast extends InstrumentationRunListener { + + private static volatile boolean aborted = false; + + @Override + public void testFailure(Failure failure) { + aborted = true; + } + + /** Skip (not fail) the current test if an earlier test already failed. */ + static void skipIfAborted() { + if (aborted) { + throw new AssumptionViolatedException("Skipped: a previous test already failed (fail-fast)"); + } + } +} diff --git a/app/src/androidTest/java/io/netbird/client/LoginFlow.java b/app/src/androidTest/java/io/netbird/client/LoginFlow.java new file mode 100644 index 00000000..5cdcbc38 --- /dev/null +++ b/app/src/androidTest/java/io/netbird/client/LoginFlow.java @@ -0,0 +1,365 @@ +package io.netbird.client; + +import android.os.Bundle; +import android.util.Log; +import android.view.View; + +import java.io.File; +import java.util.List; +import java.util.Random; + +import androidx.navigation.NavController; +import androidx.navigation.NavOptions; +import androidx.navigation.Navigation; +import androidx.test.uiautomator.By; +import androidx.test.uiautomator.BySelector; +import androidx.test.uiautomator.UiDevice; +import androidx.test.uiautomator.UiObject2; +import androidx.test.uiautomator.UiScrollable; +import androidx.test.uiautomator.UiSelector; +import androidx.test.uiautomator.Until; + +import io.netbird.client.ui.server.ChangeServerFragment; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.fail; + +/** + * Shared "log in with a setup key via the UI" flow, extracted so both + * {@link SetupKeyAuthTest} (login smoke test) and {@link PeerConnectivityTest} + * (doc test case 1 — connectivity to a peer) drive the exact same screens. + * + *

This mirrors what a user does by hand: open "Change server", type the + * setup key, submit, and dismiss the success dialog. It targets the management + * URL hard-coded in the app ({@code Preferences.defaultServer()}, the + * production server) via the "Use NetBird" button. + * + *

The setup key comes from the {@code setupKey} instrumentation argument so + * CI can inject it as a secret: + * 

+ *   ./gradlew connectedDebugAndroidTest \
+ *     -Pandroid.testInstrumentationRunnerArguments.setupKey=<UUID>
+ *
+ */ +final class LoginFlow { + + private static final String TAG = "NBLoginFlow"; + static final String PACKAGE = "io.netbird.client"; + private static final String DEFAULT_PROFILE = "default"; + + private static final long UI_TIMEOUT_MS = 5_000; + private static final long LOGIN_TIMEOUT_MS = 15_000; + + private LoginFlow() { + } + + /** + * Drives the "Change server" UI to log in with {@code setupKey} against the + * app's default (production) management server, then dismisses the success + * dialog so the caller lands back on the Home screen. + * + * @param activity the running MainActivity (from the test's ActivityTestRule) + * @param device the shared UiDevice + * @param setupKey the NetBird setup key (already trimmed/validated by caller) + * @throws Exception if any expected view never appears or login does not + * complete within {@link #LOGIN_TIMEOUT_MS} + */ + static void loginWithSetupKey(MainActivity activity, UiDevice device, String setupKey) + throws Exception { + device.waitForIdle(); + + // Try the navigation up to 2 times — on first launch the MainActivity + // pushes firstInstallFragment after onCreate, which can race with our + // navigate() call from the test thread. + UiObject2 setupKeyLabel = null; + for (int attempt = 1; attempt <= 2 && setupKeyLabel == null; attempt++) { + Log.i(TAG, "navigateToChangeServer attempt " + attempt); + navigateToChangeServer(activity); + dismissConfirmChangeServerDialog(device); + setupKeyLabel = device.wait( + Until.findObject(By.res(PACKAGE, "text_setup_key_label")), UI_TIMEOUT_MS); + } + if (setupKeyLabel == null) { + dumpScreenshot(device, "navigation-failed"); + fail("text_setup_key_label not found after 2 navigation attempts"); + } + setupKeyLabel.click(); + + UiObject2 setupKeyField = device.wait( + Until.findObject(By.res(PACKAGE, "edit_text_setup_key")), UI_TIMEOUT_MS); + assertNotNull("edit_text_setup_key must be present", setupKeyField); + setupKeyField.setText(setupKey.trim()); + + // "Use NetBird" submits with the app's default management URL. + UiObject2 submit = device.wait( + Until.findObject(By.res(PACKAGE, "btn_use_netbird")), UI_TIMEOUT_MS); + assertNotNull("btn_use_netbird must be present", submit); + submit.click(); + + // Either the success dialog ("btn_close") shows up, or the form + // re-enables itself with an error. + long deadline = System.currentTimeMillis() + LOGIN_TIMEOUT_MS; + while (System.currentTimeMillis() < deadline) { + UiObject2 closeBtn = device.findObject(By.res(PACKAGE, "btn_close")); + if (closeBtn != null) { + Log.i(TAG, "Setup-key login succeeded"); + closeBtn.click(); + device.waitForIdle(); + return; + } + // If the "Change server" button is enabled again, the request came + // back with an error. + UiObject2 submitAgain = device.findObject(By.res(PACKAGE, "btn_change_server")); + if (submitAgain != null && submitAgain.isEnabled()) { + fail("Login failed: submit button re-enabled without success dialog"); + } + Thread.sleep(200); + } + dumpScreenshot(device, "login-timeout"); + fail("Login did not complete within " + (LOGIN_TIMEOUT_MS / 1000) + "s"); + } + + /** + * Skip the first-install teaser and jump straight to the "Change server" + * screen. {@code hideAlert=true} suppresses the "are you sure?" warning + * dialog so this is non-interactive. + */ + private static void navigateToChangeServer(MainActivity activity) throws InterruptedException { + assertNotNull("MainActivity must be available", activity); + + activity.runOnUiThread(() -> { + NavController nav = Navigation.findNavController(activity, R.id.nav_host_fragment_content_main); + Bundle bundle = new Bundle(); + bundle.putBoolean(ChangeServerFragment.HideAlertBundleArg, true); + // Same nav options the FirstInstallFragment uses when the user taps + // its "change_server" link, so we land in the same place. + NavOptions opts = new NavOptions.Builder() + .setPopUpTo(R.id.firstInstallFragment, true) + .build(); + nav.navigate(R.id.nav_change_server, bundle, opts); + }); + // Let the fragment transaction commit before UiAutomator looks for views. + Thread.sleep(1500); + } + + /** + * The Change Server fragment shows a "this will erase the local config" + * confirmation dialog whenever it opens — even when the caller passed + * {@code hideAlert=true}, because that arg is not currently honoured by the + * fragment. Tap Yes to dismiss so we can interact with the form. + */ + private static void dismissConfirmChangeServerDialog(UiDevice device) { + UiObject2 yes = device.wait( + Until.findObject(By.res(PACKAGE, "btn_yes")), UI_TIMEOUT_MS); + if (yes != null) { + Log.i(TAG, "Dismissing change-server confirmation dialog"); + yes.click(); + device.waitForIdle(); + } + } + + /** + * Create a fresh, isolated profile through the Profiles UI and switch to + * it, mirroring the Robot suite's {@code netbird profile add test-} + * step (which it runs from the CLI). Driving it through the UI keeps the + * test on the same user-facing path the rest of the flow uses. + * + * @param scenario short label for the calling test, woven into the profile + * name so it is identifiable in the UI / on the recording + * @return the generated profile name (pass it to {@link #removeProfile} to + * clean up afterwards) + */ + static String createAndSwitchToFreshProfile(MainActivity activity, UiDevice device, String scenario) + throws Exception { + String profileName = "e2e-" + sanitizeScenario(scenario) + "-" + randomLowercase(4); + + navigateTo(activity, R.id.nav_profiles); + + UiObject2 addBtn = device.wait( + Until.findObject(By.res(PACKAGE, "btn_add_profile")), UI_TIMEOUT_MS); + assertNotNull("btn_add_profile must be present", addBtn); + addBtn.click(); + + UiObject2 nameField = device.wait( + Until.findObject(By.res(PACKAGE, "edit_text_dialog")), UI_TIMEOUT_MS); + assertNotNull("edit_text_dialog must be present", nameField); + nameField.setText(profileName); + confirmDialog(device); + + // The new profile is added but not active yet — switch to it so the + // subsequent login writes into this profile. + UiObject2 switchBtn = rowAction(device, profileName, "btn_switch"); + if (switchBtn == null) { + dumpScreenshot(device, "profile-switch-missing"); + fail("btn_switch not found for profile " + profileName); + } + switchBtn.click(); + confirmDialog(device); + device.waitForIdle(); + + Log.i(TAG, "Created and switched to profile " + profileName); + return profileName; + } + + /** + * Remove a profile created by {@link #createAndSwitchToFreshProfile}. A + * profile can't be removed while active, so we switch to the built-in + * "default" profile first, then remove the test profile. We assert on the + * profile UI: a missing switch/remove button fails the test — it's a real + * UI defect, not something to swallow. + */ + static void removeProfile(MainActivity activity, UiDevice device, String profileName) + throws InterruptedException { + navigateTo(activity, R.id.nav_profiles); + + UiObject2 switchBtn = rowAction(device, DEFAULT_PROFILE, "btn_switch"); + if (switchBtn == null) { + dumpScreenshot(device, "default-switch-missing"); + fail("btn_switch not found for the " + DEFAULT_PROFILE + " profile"); + } + switchBtn.click(); + confirmDialog(device); + device.waitForIdle(); + + // Switching profiles bounces back to Home, so return to the profiles + // screen before looking for the test profile's remove button. + navigateTo(activity, R.id.nav_profiles); + + UiObject2 removeBtn = rowAction(device, profileName, "btn_remove"); + if (removeBtn == null) { + dumpScreenshot(device, "remove-button-missing"); + fail("btn_remove not found for profile " + profileName); + } + removeBtn.click(); + confirmDialog(device); + device.waitForIdle(); + Log.i(TAG, "Removed profile " + profileName); + } + + private static boolean forceRelayDisabled = false; + + /** + * Turn off the "Force relay connection" toggle in the Advanced screen so + * peers can connect directly (P2P). It defaults ON (a global setting), which + * prevents the relay-less peer from connecting. This is a one-time suite + * pre-step: every test's {@code @Before} calls it, but it only acts once. + */ + static void ensureForceRelayDisabled(MainActivity activity, UiDevice device) throws InterruptedException { + if (forceRelayDisabled) { + return; + } + navigateTo(activity, R.id.nav_advanced); + + // The switch is far down the scrollable Advanced screen — scroll to it. + scrollTo(device, "Force relay connection"); + + UiObject2 toggle = rowControl(device, "Force relay connection", "switch_control"); + if (toggle == null) { + dumpScreenshot(device, "force-relay-switch-missing"); + fail("Force relay connection switch not found in Advanced screen"); + } + if (toggle.isChecked()) { + toggle.click(); + // Toggling pops a "reconnection needed" warning; dismiss it. + confirmDialog(device); + Log.i(TAG, "Disabled force relay connection"); + } else { + Log.i(TAG, "Force relay connection already off"); + } + forceRelayDisabled = true; + } + + /** + * Find the action button with {@code resId} inside the profile row whose + * {@code text_profile_name} matches {@code profileName}. Returns null if no + * such row/button is visible. + */ + private static UiObject2 rowAction(UiDevice device, String profileName, String resId) { + return rowControl(device, "text_profile_name", profileName, resId, true); + } + + /** Like {@link #rowAction} but the row is matched by visible text on any label. */ + private static UiObject2 rowControl(UiDevice device, String rowText, String resId) { + return rowControl(device, null, rowText, resId, false); + } + + private static UiObject2 rowControl(UiDevice device, String labelResId, String text, + String resId, boolean labelByRes) { + BySelector labelSel = labelByRes + ? By.res(PACKAGE, labelResId).text(text) + : By.text(text); + UiObject2 label = device.wait(Until.findObject(labelSel), UI_TIMEOUT_MS); + if (label == null) { + return null; + } + // The control lives in the same row; walk up and search within it. + UiObject2 row = label.getParent(); + BySelector sel = By.res(PACKAGE, resId); + for (int i = 0; i < 5 && row != null; i++) { + UiObject2 ctrl = row.findObject(sel); + if (ctrl != null) { + return ctrl; + } + row = row.getParent(); + } + return null; + } + + /** Scroll a scrollable screen until an element with {@code text} is visible. */ + private static void scrollTo(UiDevice device, String text) { + try { + UiScrollable scrollable = new UiScrollable(new UiSelector().scrollable(true)); + scrollable.scrollTextIntoView(text); + } catch (Exception e) { + Log.w(TAG, "scrollTo(" + text + ") failed: " + e.getMessage()); + } + device.waitForIdle(); + } + + private static void confirmDialog(UiDevice device) { + UiObject2 ok = device.wait( + Until.findObject(By.res(PACKAGE, "btn_ok_dialog")), UI_TIMEOUT_MS); + assertNotNull("btn_ok_dialog must be present", ok); + ok.click(); + device.waitForIdle(); + } + + private static void navigateTo(MainActivity activity, int destId) throws InterruptedException { + assertNotNull("MainActivity must be available", activity); + activity.runOnUiThread(() -> { + NavController nav = Navigation.findNavController(activity, R.id.nav_host_fragment_content_main); + nav.navigate(destId); + }); + Thread.sleep(1000); + } + + private static String sanitizeScenario(String scenario) { + String s = scenario.toLowerCase().replaceAll("[^a-z0-9-]+", "-").replaceAll("(^-+|-+$)", ""); + return s.isEmpty() ? "test" : s; + } + + private static String randomLowercase(int len) { + Random random = new Random(); + StringBuilder sb = new StringBuilder(len); + for (int i = 0; i < len; i++) { + sb.append((char) ('a' + random.nextInt(26))); + } + return sb.toString(); + } + + /** + * Take a screenshot via UiAutomator and write it under /sdcard/Pictures so + * the CI workflow can {@code adb pull} it alongside the screen recording. + */ + static void dumpScreenshot(UiDevice device, String name) { + try { + File png = new File("/sdcard/Pictures/" + name + ".png"); + //noinspection ResultOfMethodCallIgnored + png.getParentFile().mkdirs(); + boolean ok = device.takeScreenshot(png); + Log.i(TAG, "Screenshot " + (ok ? "saved to " : "FAILED for ") + png); + } catch (Throwable t) { + Log.w(TAG, "Failed to dump screenshot: " + t.getMessage()); + } + } +} diff --git a/app/src/androidTest/java/io/netbird/client/PeerConnectivityTest.java b/app/src/androidTest/java/io/netbird/client/PeerConnectivityTest.java new file mode 100644 index 00000000..87683067 --- /dev/null +++ b/app/src/androidTest/java/io/netbird/client/PeerConnectivityTest.java @@ -0,0 +1,140 @@ +package io.netbird.client; + +import android.os.Bundle; +import android.util.Log; + +import androidx.test.ext.junit.runners.AndroidJUnit4; +import androidx.test.platform.app.InstrumentationRegistry; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; + +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +/** + * End-to-end connectivity test — the Android port of the Robot + * {@code client-tests.robot} "Should be able to connect to peer" cases. + * + *

Mirrors the Robot {@code Try Peer Connectivity} keyword. It creates a + * fresh, isolated profile (via the Profiles UI — the Android equivalent of + * {@code netbird profile add test-}), logs in to the production + * management server with a setup key (shared UI flow in {@link LoginFlow}), + * brings the VPN up, then verifies the data plane by pinging a remote peer's + * FQDN through the tunnel. The NetBird tunnel DNS resolves the name to + * the peer's overlay IP. The remote peers are live, externally-running NetBird + * containers; this test does not create or tear them down. The profile is + * removed in teardown. + * + *

Two cases, exactly as in the Robot suite (peer FQDNs hard-coded as in the + * original): + *

    + *
  • {@link #connectsWithRelay()} — {@code pingtest.netbird.cloud} (case + * "with relay support")
    • + *
  • {@link #connectsWithoutRelay()} — {@code pingtest-pre-relay.netbird.cloud} + * (case "without relay support")
    • + *
+ * + *

Only the setup key is injected: + *

    + *
  • {@code setupKey} — NetBird setup key for the client under test + * (CI injects the {@code INSTRUMENTATION_NB_SETUP_KEY} secret)
    • + *
+ * + * 
+ *   ./gradlew connectedDebugAndroidTest \
+ *     -Pandroid.testInstrumentationRunnerArguments.setupKey=<UUID>
+ *
+ * + *

Timeouts follow the Robot {@code Wait For Peer Ready} keyword: up to + * ~3 minutes for the peer to come up and the tunnel to carry traffic. + */ +@RunWith(AndroidJUnit4.class) +public class PeerConnectivityTest { + + private static final String TAG = "NBPeerConnTest"; + + /** Peer reachable with relay support (Robot "with relay support" case). */ + private static final String PEER_FQDN_RELAY = "pingtest.netbird.cloud"; + /** Peer reachable without relay (Robot "without relay support" case). */ + private static final String PEER_FQDN_NO_RELAY = "pingtest-pre-relay.netbird.cloud"; + + /** Matches the Robot suite's peer-connected window (3 min). */ + private static final long CONNECT_TIMEOUT_SEC = 20; + /** How long to keep retrying the ping once the engine reports connected. */ + private static final long PING_TIMEOUT_SEC = 20; + + private VpnTestHarness harness; + private String profileName; + + @Before + public void skipIfPreviousFailed() { + FailFast.skipIfAborted(); + } + + @After + public void tearDown() throws Exception { + if (profileName != null && harness != null) { + harness.disableTouchVisualization(); + LoginFlow.removeProfile(E2eAppRule.activity(), harness.device(), profileName); + } + } + + @Test + public void connectsWithRelay() throws Exception { + connectAndPing(PEER_FQDN_RELAY, "relay"); + } + + @Test + public void connectsWithoutRelay() throws Exception { + connectAndPing(PEER_FQDN_NO_RELAY, "no-relay"); + } + + /** + * Shared body: fresh profile → login → connect → ping {@code peerFqdn} + * through the tunnel. This is the Android equivalent of the Robot + * {@code Try Peer Connectivity ${peer_name}} keyword. + */ + private void connectAndPing(String peerFqdn, String scenario) throws Exception { + Bundle args = InstrumentationRegistry.getArguments(); + String setupKey = args.getString("setupKey"); + + assertNotNull("setupKey instrumentation argument is required", setupKey); + assertTrue("setupKey must not be blank", !setupKey.trim().isEmpty()); + + MainActivity activity = E2eAppRule.activity(); + harness = new VpnTestHarness(activity); + harness.enableTouchVisualization(); + + harness.grantVpnConsent(); + // Global pre-step (once for the whole suite): peers can't connect P2P + // with force-relay on, which it is by default. + LoginFlow.ensureForceRelayDisabled(activity, harness.device()); + + // 1. Create a fresh, isolated profile (Android equivalent of the Robot + // suite's `netbird profile add test-`), then log in into it. + profileName = LoginFlow.createAndSwitchToFreshProfile(activity, harness.device(), scenario); + LoginFlow.loginWithSetupKey(activity, harness.device(), setupKey); + + // 2. Bring the VPN up and wait for the engine to report connected. + boolean connected = harness.connectAndAwait(CONNECT_TIMEOUT_SEC); + if (!connected) { + LoginFlow.dumpScreenshot(harness.device(), "vpn-connect-timeout"); + } + assertTrue("VPN did not reach connected state within " + CONNECT_TIMEOUT_SEC + "s", + connected); + + // 3. Verify the data plane: the remote peer must be reachable over the + // tunnel by its FQDN (the tunnel DNS resolves it to the overlay IP). + boolean reachable = harness.waitForPing(peerFqdn, PING_TIMEOUT_SEC); + if (!reachable) { + LoginFlow.dumpScreenshot(harness.device(), "peer-ping-failed"); + } + assertTrue("Peer " + peerFqdn + " was not reachable over the tunnel within " + + PING_TIMEOUT_SEC + "s", reachable); + + Log.i(TAG, "Peer " + peerFqdn + " reachable over the NetBird tunnel"); + } +} diff --git a/app/src/androidTest/java/io/netbird/client/PortAclTest.java b/app/src/androidTest/java/io/netbird/client/PortAclTest.java new file mode 100644 index 00000000..e4da1692 --- /dev/null +++ b/app/src/androidTest/java/io/netbird/client/PortAclTest.java @@ -0,0 +1,164 @@ +package io.netbird.client; + +import android.os.Bundle; +import android.util.Log; + +import androidx.test.ext.junit.runners.AndroidJUnit4; +import androidx.test.platform.app.InstrumentationRegistry; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +/** + * ACL / allowed-ports test — the Android port of the Robot + * {@code client-tests.robot} case "Should connect only to the allowed ports". + * + *

The {@code acltest.netbird.cloud} peer has an ACL that blocks ICMP + * but allows TCP port 80. So, with the tunnel up, the original asserts: + *

    + *
  • ping to the peer FAILS (ICMP dropped) — Robot {@code Should Not Be + * Equal As Numbers ${result.rc} 0} with a {@code 0 received} regex;
    • + *
  • a TCP connection to port 80 SUCCEEDS — Robot {@code Open Connection + * acltest.netbird.cloud port=80}.
    • + *
+ * + *

This is the proof the ACL is actually enforced: the peer is reachable + * (control plane connected, port 80 open) yet ICMP is filtered. + * + *

Reuses the same fresh-profile + login + connect flow as + * {@link PeerConnectivityTest}. Only the setup key is injected: + * 

+ *   ./gradlew connectedDebugAndroidTest \
+ *     -Pandroid.testInstrumentationRunnerArguments.setupKey=<UUID>
+ *
+ */ +@RunWith(AndroidJUnit4.class) +public class PortAclTest { + + private static final String TAG = "NBPortAclTest"; + + /** Peer with an ICMP-blocking, port-80-allowing ACL. */ + private static final String PEER_FQDN = "acltest.netbird.cloud"; + private static final int ALLOWED_PORT = 80; + + /** Matches the Robot suite's peer-connected window (3 min). */ + private static final long CONNECT_TIMEOUT_SEC = 20; + /** + * Time budget for the allowed port to become reachable once the engine is + * connected (peer + ACL need a moment to settle). Loosely mirrors the + * Robot peer-ready/handshake windows. + */ + private static final long PORT_TIMEOUT_SEC = 20; + private static final int TCP_CONNECT_TIMEOUT_MS = 3000; + /** + * How long to keep trying ICMP before concluding it is blocked. The ACL + * drops it, so this should always time out — kept short so a genuinely + * blocked peer does not slow the test much, but long enough that a slow + * first packet is not mistaken for a block (the port check above already + * proved the peer is reachable, so a few seconds is plenty). + */ + private static final long PING_BLOCKED_PROBE_SEC = 10; + private VpnTestHarness harness; + private String profileName; + + @Before + + public void skipIfPreviousFailed() { + + FailFast.skipIfAborted(); + + } + + + @After + public void tearDown() throws Exception { + if (profileName != null && harness != null) { + harness.disableTouchVisualization(); + LoginFlow.removeProfile(E2eAppRule.activity(), harness.device(), profileName); + } + } + + @Test + public void connectsOnlyToAllowedPorts() throws Exception { + Bundle args = InstrumentationRegistry.getArguments(); + String setupKey = args.getString("setupKey"); + + assertNotNull("setupKey instrumentation argument is required", setupKey); + assertTrue("setupKey must not be blank", !setupKey.trim().isEmpty()); + + MainActivity activity = E2eAppRule.activity(); + assertNotNull("MainActivity must be available", activity); + harness = new VpnTestHarness(activity); + harness.enableTouchVisualization(); + + harness.grantVpnConsent(); + LoginFlow.ensureForceRelayDisabled(activity, harness.device()); + + // Fresh profile + login, like the Robot suite's per-test InitNetBird. + profileName = LoginFlow.createAndSwitchToFreshProfile(activity, harness.device(), "port-acl"); + LoginFlow.loginWithSetupKey(activity, harness.device(), setupKey); + + boolean connected = harness.connectAndAwait(CONNECT_TIMEOUT_SEC); + if (!connected) { + LoginFlow.dumpScreenshot(harness.device(), "vpn-connect-timeout"); + } + assertTrue("VPN did not reach connected state within " + CONNECT_TIMEOUT_SEC + "s", + connected); + + // Allowed port: TCP 80 must be reachable. This also serves as the + // "peer is ready" signal (the Robot suite uses ping for that, but here + // ping is blocked, so we rely on the allowed port instead). + boolean portOpen = waitForTcp(PEER_FQDN, ALLOWED_PORT, PORT_TIMEOUT_SEC); + if (!portOpen) { + LoginFlow.dumpScreenshot(harness.device(), "acl-port80-unreachable"); + } + assertTrue("Allowed port " + ALLOWED_PORT + " on " + PEER_FQDN + + " was not reachable within " + PORT_TIMEOUT_SEC + "s", portOpen); + + // Blocked protocol: ICMP must NOT get through (ACL drops it). The peer + // is provably reachable (port 80 just connected), so any ping success + // here would mean the ACL is not being enforced. + boolean pingGotThrough = pingSucceedsWithin(PEER_FQDN, PING_BLOCKED_PROBE_SEC); + if (pingGotThrough) { + LoginFlow.dumpScreenshot(harness.device(), "acl-icmp-leaked"); + } + assertFalse("ICMP to " + PEER_FQDN + " should be blocked by the ACL, but ping succeeded", + pingGotThrough); + + Log.i(TAG, "ACL enforced: port " + ALLOWED_PORT + " open, ICMP blocked on " + PEER_FQDN); + } + + /** Retry a TCP connect until it succeeds or the timeout elapses. */ + private boolean waitForTcp(String host, int port, long timeoutSec) throws InterruptedException { + long deadline = System.currentTimeMillis() + (timeoutSec * 1000L); + while (System.currentTimeMillis() < deadline) { + if (harness.tcpConnects(host, port, TCP_CONNECT_TIMEOUT_MS)) { + return true; + } + Thread.sleep(3000); + } + return false; + } + + /** + * Probe ICMP for up to {@code timeoutSec}; returns true as soon as any ping + * gets through. Used to assert ICMP is blocked, so a true result is a + * failure for the caller. + */ + private boolean pingSucceedsWithin(String host, long timeoutSec) throws InterruptedException { + long deadline = System.currentTimeMillis() + (timeoutSec * 1000L); + while (System.currentTimeMillis() < deadline) { + if (harness.pingOnce(host)) { + return true; + } + Thread.sleep(2000); + } + return false; + } +} diff --git a/app/src/androidTest/java/io/netbird/client/SetupKeyAuthTest.java b/app/src/androidTest/java/io/netbird/client/SetupKeyAuthTest.java index bb973969..f53a204f 100644 --- a/app/src/androidTest/java/io/netbird/client/SetupKeyAuthTest.java +++ b/app/src/androidTest/java/io/netbird/client/SetupKeyAuthTest.java @@ -1,36 +1,27 @@ package io.netbird.client; import android.os.Bundle; -import android.util.Log; -import android.view.View; -import java.io.File; - -import androidx.navigation.NavController; -import androidx.navigation.NavOptions; -import androidx.navigation.Navigation; import androidx.test.ext.junit.runners.AndroidJUnit4; import androidx.test.platform.app.InstrumentationRegistry; -import androidx.test.rule.ActivityTestRule; -import androidx.test.uiautomator.By; import androidx.test.uiautomator.UiDevice; -import androidx.test.uiautomator.UiObject2; -import androidx.test.uiautomator.Until; -import org.junit.Rule; +import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; -import io.netbird.client.ui.server.ChangeServerFragment; - import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; /** - * Drives the "Change server" UI to authenticate against the default NetBird - * management server with a setup key — exactly the flow a user would use, but - * automated. + * Login smoke test: drives the "Change server" UI to authenticate against the + * default (production) NetBird management server with a setup key — exactly the + * flow a user would use, but automated. + * + *

This test stops at a successful login (the success dialog). For the + * end-to-end "can actually reach a peer" check (doc test case 1) see + * {@link PeerConnectivityTest}. The shared login steps live in + * {@link LoginFlow}. * *

The setup key is read from an instrumentation runner argument so CI can * inject it as a secret without baking it into the APK: @@ -38,24 +29,13 @@ * ./gradlew connectedDebugAndroidTest \ * -Pandroid.testInstrumentationRunnerArguments.setupKey=<UUID> * - * - *

The test navigates straight to {@code nav_change_server} (skipping the - * first-install teaser screen), fills the setup key and taps the - * "Use NetBird" button, which uses the management URL hard-coded in the app - * ({@code Preferences.defaultServer()}). Then it waits for the success dialog. */ @RunWith(AndroidJUnit4.class) public class SetupKeyAuthTest { - - private static final String TAG = "NBSetupKeyAuthTest"; - private static final String PACKAGE = "io.netbird.client"; - private static final long UI_TIMEOUT_MS = 5_000; - private static final long LOGIN_TIMEOUT_MS = 15_000; - - @SuppressWarnings("deprecation") - @Rule - public ActivityTestRule activityRule = - new ActivityTestRule<>(MainActivity.class, true, true); + @Before + public void skipIfPreviousFailed() { + FailFast.skipIfAborted(); + } @Test public void loginWithSetupKeyViaUi() throws Exception { @@ -66,113 +46,6 @@ public void loginWithSetupKeyViaUi() throws Exception { assertTrue("setupKey must not be blank", !setupKey.trim().isEmpty()); UiDevice device = UiDevice.getInstance(InstrumentationRegistry.getInstrumentation()); - device.waitForIdle(); - - // Try the navigation up to 2 times — on first launch the MainActivity - // pushes firstInstallFragment after onCreate, which can race with our - // navigate() call from the test thread. - UiObject2 setupKeyLabel = null; - for (int attempt = 1; attempt <= 2 && setupKeyLabel == null; attempt++) { - Log.i(TAG, "navigateToChangeServer attempt " + attempt); - navigateToChangeServer(); - dismissConfirmChangeServerDialog(device); - setupKeyLabel = device.wait( - Until.findObject(By.res(PACKAGE, "text_setup_key_label")), UI_TIMEOUT_MS); - } - if (setupKeyLabel == null) { - dumpScreenshot(device, "navigation-failed"); - fail("text_setup_key_label not found after 2 navigation attempts"); - } - setupKeyLabel.click(); - - UiObject2 setupKeyField = device.wait( - Until.findObject(By.res(PACKAGE, "edit_text_setup_key")), UI_TIMEOUT_MS); - assertNotNull("edit_text_setup_key must be present", setupKeyField); - setupKeyField.setText(setupKey.trim()); - - // "Use NetBird" submits with the app's default management URL. - UiObject2 submit = device.wait( - Until.findObject(By.res(PACKAGE, "btn_use_netbird")), UI_TIMEOUT_MS); - assertNotNull("btn_use_netbird must be present", submit); - submit.click(); - - // Either the success dialog ("btn_close") shows up, or the form re-enables - // itself with an error. - long deadline = System.currentTimeMillis() + LOGIN_TIMEOUT_MS; - while (System.currentTimeMillis() < deadline) { - UiObject2 closeBtn = device.findObject(By.res(PACKAGE, "btn_close")); - if (closeBtn != null) { - Log.i(TAG, "Setup-key login succeeded"); - closeBtn.click(); - return; - } - // If the "Change server" button is enabled again, the request came - // back with an error. - UiObject2 submitAgain = device.findObject(By.res(PACKAGE, "btn_change_server")); - if (submitAgain != null && submitAgain.isEnabled()) { - fail("Login failed: submit button re-enabled without success dialog"); - } - Thread.sleep(200); - } - dumpScreenshot(device, "login-timeout"); - fail("Login did not complete within " + (LOGIN_TIMEOUT_MS / 1000) + "s"); - } - - /** - * Skip the first-install teaser and jump straight to the "Change server" screen. - * {@code hideAlert=true} suppresses the "are you sure?" warning dialog so this - * is non-interactive. - */ - private void navigateToChangeServer() throws InterruptedException { - MainActivity activity = activityRule.getActivity(); - assertNotNull("MainActivity must be available", activity); - - activity.runOnUiThread(() -> { - View host = activity.findViewById(R.id.nav_host_fragment_content_main); - NavController nav = Navigation.findNavController(host); - Bundle bundle = new Bundle(); - bundle.putBoolean(ChangeServerFragment.HideAlertBundleArg, true); - // Same nav options the FirstInstallFragment uses when the user taps - // its "change_server" link, so we land in the same place. - NavOptions opts = new NavOptions.Builder() - .setPopUpTo(R.id.firstInstallFragment, true) - .build(); - nav.navigate(R.id.nav_change_server, bundle, opts); - }); - // Let the fragment transaction commit before UiAutomator looks for views. - Thread.sleep(1500); - } - - /** - * The Change Server fragment shows a "this will erase the local config" - * confirmation dialog whenever it opens — even when the caller passed - * {@code hideAlert=true}, because that arg is not currently honoured by - * the fragment. Tap Yes to dismiss so we can interact with the form. - */ - private static void dismissConfirmChangeServerDialog(UiDevice device) { - UiObject2 yes = device.wait( - Until.findObject(By.res(PACKAGE, "btn_yes")), UI_TIMEOUT_MS); - if (yes != null) { - Log.i(TAG, "Dismissing change-server confirmation dialog"); - yes.click(); - device.waitForIdle(); - } - } - - /** - * Take a screenshot via UiAutomator and write it into the test runner's - * working dir (cwd is /data/local/tmp/io.netbird.client.test on most - * devices, which `adb pull` can read). - */ - private static void dumpScreenshot(UiDevice device, String name) { - try { - File png = new File("/sdcard/Pictures/" + name + ".png"); - //noinspection ResultOfMethodCallIgnored - png.getParentFile().mkdirs(); - boolean ok = device.takeScreenshot(png); - Log.i(TAG, "Screenshot " + (ok ? "saved to " : "FAILED for ") + png); - } catch (Throwable t) { - Log.w(TAG, "Failed to dump screenshot: " + t.getMessage()); - } + LoginFlow.loginWithSetupKey(E2eAppRule.activity(), device, setupKey); } } diff --git a/app/src/androidTest/java/io/netbird/client/VpnTestHarness.java b/app/src/androidTest/java/io/netbird/client/VpnTestHarness.java new file mode 100644 index 00000000..6d7ec523 --- /dev/null +++ b/app/src/androidTest/java/io/netbird/client/VpnTestHarness.java @@ -0,0 +1,276 @@ +package io.netbird.client; + +import android.app.Instrumentation; +import android.app.UiAutomation; +import android.os.ParcelFileDescriptor; +import android.util.Log; + +import androidx.test.platform.app.InstrumentationRegistry; +import androidx.test.uiautomator.UiDevice; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.net.HttpURLConnection; +import java.net.InetSocketAddress; +import java.net.Socket; +import java.net.URL; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Shared machinery for the on-device client e2e tests (the Android port of the + * Robot {@code client-tests.robot} suite). Each test creates one of these from + * its {@link MainActivity}, then composes the building blocks it needs: + * + *

    + *
  • {@link #grantVpnConsent()} — pre-approve the VPN so the engine starts + * without the system consent dialog (headless-friendly)
    • + *
  • {@link #connectAndAwait(long)} — start the engine via the app API and + * wait for {@code onConnected}, like the Robot {@code Wait For Peer + * Ready}
    • + *
  • {@link #waitForPing(String, long)} / {@link #pingOnce(String)} — the + * Android equivalent of the Robot {@code Get Ping Command And Regex} + * check (ICMP through the tunnel)
    • + *
  • {@link #tcpConnects(String, int, int)} — the equivalent of the Robot + * {@code Open Connection ... port=N} Telnet check (TCP reachability)
    • + *
+ * + *

Connecting goes through {@link MainActivity#switchConnection(boolean)} and + * a {@link StateListener}, not the Lottie connect button, so it is robust to UI + * timing. Network probes run as shell commands / plain sockets so they observe + * the tunnel exactly as a user's traffic would. + */ +final class VpnTestHarness { + + private static final String TAG = "NBVpnHarness"; + + private final MainActivity activity; + private final UiDevice device; + private final UiAutomation uiAutomation; + + VpnTestHarness(MainActivity activity) { + Instrumentation instrumentation = InstrumentationRegistry.getInstrumentation(); + this.activity = activity; + this.device = UiDevice.getInstance(instrumentation); + this.uiAutomation = instrumentation.getUiAutomation(); + } + + UiDevice device() { + return device; + } + + /** Show touch feedback on screen so recordings reveal what the test taps. */ + void enableTouchVisualization() { + shell("settings put system show_touches 1"); + shell("settings put system pointer_location 1"); + } + + void disableTouchVisualization() { + shell("settings put system show_touches 0"); + shell("settings put system pointer_location 0"); + } + + /** + * Pre-grant the VPN consent (ACTIVATE_VPN appop) for this package so the + * VpnService starts without a system dialog. Best-effort: if the command is + * unavailable the test still works as long as consent was granted before. + */ + void grantVpnConsent() { + String out = shell("appops set " + LoginFlow.PACKAGE + " ACTIVATE_VPN allow"); + Log.i(TAG, "appops ACTIVATE_VPN -> " + out.trim()); + } + + /** + * Start the engine via the app's own API and block until a state listener + * reports {@code onConnected}, or the timeout elapses. + * + * @return true if the engine reported connected within the timeout + */ + boolean connectAndAwait(long timeoutSec) throws InterruptedException { + CountDownLatch connectedLatch = new CountDownLatch(1); + + StateListener listener = new StateListener() { + @Override public void onEngineStarted() {} + @Override public void onEngineStopped() {} + @Override public void onAddressChanged(String fqdn, String ip) {} + @Override public void onConnected() { + Log.i(TAG, "Engine reported connected"); + connectedLatch.countDown(); + } + @Override public void onConnecting() {} + @Override public void onDisconnected() {} + @Override public void onDisconnecting() {} + @Override public void onPeersListChanged(long count) { + Log.i(TAG, "Peers list changed: " + count); + } + }; + + activity.runOnUiThread(() -> activity.registerServiceStateListener(listener)); + try { + activity.runOnUiThread(() -> activity.switchConnection(true)); + return connectedLatch.await(timeoutSec, TimeUnit.SECONDS); + } finally { + activity.runOnUiThread(() -> activity.unregisterServiceStateListener(listener)); + } + } + + /** Retry {@link #pingOnce(String)} until it succeeds or the timeout elapses. */ + boolean waitForPing(String target, long timeoutSec) throws InterruptedException { + long deadline = System.currentTimeMillis() + (timeoutSec * 1000L); + int attempt = 0; + while (System.currentTimeMillis() < deadline) { + attempt++; + if (pingOnce(target)) { + Log.i(TAG, "Ping to " + target + " succeeded on attempt " + attempt); + return true; + } + Thread.sleep(3000); + } + Log.w(TAG, "Ping to " + target + " failed after " + attempt + " attempts"); + return false; + } + + /** Ping a host (FQDN or IP) once through the tunnel. */ + boolean pingOnce(String target) { + String output = shell(String.format("ping -c 1 -W %d %s", PING_W_SEC, target)); + if (output.contains("1 received") || output.contains("1 packets received")) { + return true; + } + // Some ROMs print a different summary; fall back to a positive RTT line. + return output.contains("time=") && !output.contains("100% packet loss"); + } + + /** + * Try to open a TCP connection to {@code host:port}, the equivalent of the + * Robot {@code Open Connection ... port=N connection_timeout=1} Telnet + * check. Returns true if the socket connects within {@code timeoutMs}. + */ + boolean tcpConnects(String host, int port, int timeoutMs) { + try (Socket socket = new Socket()) { + socket.connect(new InetSocketAddress(host, port), timeoutMs); + Log.i(TAG, "TCP connect to " + host + ":" + port + " succeeded"); + return true; + } catch (IOException e) { + Log.i(TAG, "TCP connect to " + host + ":" + port + " failed: " + e.getMessage()); + return false; + } + } + + /** + * Resolve {@code host} via {@code ping}, which is the only resolver tool + * present on these devices (no nslookup/getent on Android 11). ping prints + * the resolved address in its first line — {@code PING host (1.2.3.4) ...} — + * and does the lookup through the device resolver / NetBird VpnService DNS, + * like a user's traffic. Returns that address, or null if it didn't resolve. + */ + String resolve(String host) { + String out = shell("ping -c 1 -W 2 " + host); + Matcher m = PING_RESOLVED_IP.matcher(out); + return m.find() ? m.group(1) : null; + } + + private static final Pattern PING_RESOLVED_IP = + Pattern.compile("\\(([0-9]{1,3}(?:\\.[0-9]{1,3}){3})\\)"); + + /** Retry {@link #resolve(String)} until it returns {@code expectedIp} or the timeout elapses. */ + boolean waitForResolve(String host, String expectedIp, long timeoutSec) throws InterruptedException { + long deadline = System.currentTimeMillis() + (timeoutSec * 1000L); + String last = null; + while (System.currentTimeMillis() < deadline) { + last = resolve(host); + if (expectedIp.equals(last)) { + Log.i(TAG, "Resolved " + host + " -> " + last); + return true; + } + Thread.sleep(3000); + } + Log.w(TAG, "Resolve " + host + " did not yield " + expectedIp + " (last: " + last + ")"); + return false; + } + + /** + * Issue a real HTTPS GET and return the response body, or null on failure. + * Used by the exit-node test to ask {@code https://api.ipify.org} what the + * public egress IP is — the Android equivalent of the Robot + * {@code GET https://api.ipify.org}. With the tunnel routing through an + * exit node, the returned IP is the exit node's, not the device's. + */ + String httpGet(String urlString) { + HttpURLConnection conn = null; + try { + URL url = new URL(urlString); + conn = (HttpURLConnection) url.openConnection(); + conn.setConnectTimeout(10_000); + conn.setReadTimeout(10_000); + conn.setRequestMethod("GET"); + int code = conn.getResponseCode(); + if (code != HttpURLConnection.HTTP_OK) { + Log.i(TAG, "GET " + urlString + " -> HTTP " + code); + return null; + } + try (BufferedReader reader = new BufferedReader( + new InputStreamReader(conn.getInputStream()))) { + StringBuilder body = new StringBuilder(); + String line; + while ((line = reader.readLine()) != null) { + body.append(line); + } + return body.toString(); + } + } catch (IOException e) { + Log.i(TAG, "GET " + urlString + " failed: " + e.getMessage()); + return null; + } finally { + if (conn != null) { + conn.disconnect(); + } + } + } + + /** + * Poll {@link #httpGet(String)} until the response contains {@code + * expectedSubstring} (word-boundaried for dotted-quad IPs) or the timeout + * elapses. + */ + boolean waitForHttpBodyContains(String urlString, String expectedSubstring, long timeoutSec) + throws InterruptedException { + Pattern p = Pattern.compile("(^|[^0-9.])" + Pattern.quote(expectedSubstring) + "([^0-9.]|$)"); + long deadline = System.currentTimeMillis() + (timeoutSec * 1000L); + String last = null; + while (System.currentTimeMillis() < deadline) { + last = httpGet(urlString); + if (last != null && p.matcher(last).find()) { + Log.i(TAG, "GET " + urlString + " body matched " + expectedSubstring); + return true; + } + Thread.sleep(3000); + } + Log.w(TAG, "GET " + urlString + " never matched " + expectedSubstring + " (last: " + last + ")"); + return false; + } + + /** Run a shell command via the instrumentation UiAutomation and return stdout. */ + String shell(String command) { + try { + ParcelFileDescriptor pfd = uiAutomation.executeShellCommand(command); + try (BufferedReader reader = new BufferedReader( + new InputStreamReader(new ParcelFileDescriptor.AutoCloseInputStream(pfd)))) { + StringBuilder output = new StringBuilder(); + String line; + while ((line = reader.readLine()) != null) { + output.append(line).append('\n'); + } + return output.toString(); + } + } catch (Exception e) { + Log.w(TAG, "Shell command failed: " + command + " - " + e.getMessage()); + return ""; + } + } + + /** Per-attempt ping timeout, in seconds. */ + private static final int PING_W_SEC = 5; +} From 4228e7cc9d77cb1c8fe5c2fa4f5e818ef01d6d50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Zolt=C3=A1n=20Papp?= Date: Tue, 16 Jun 2026 00:18:40 +0200 Subject: [PATCH 7/9] test: disable force-relay once for the whole e2e suite MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move the force-relay-off step from per-test calls into E2eSuite's @BeforeClass so it runs once, guaranteed, before any test — instead of each test toggling the global setting. --- .../io/netbird/client/DnsResolutionTest.java | 1 - .../java/io/netbird/client/E2eSuite.java | 14 +++++++++++ .../io/netbird/client/ExitNodeRouteTest.java | 1 - .../java/io/netbird/client/LoginFlow.java | 23 ++++++++----------- .../netbird/client/PeerConnectivityTest.java | 6 ++--- .../java/io/netbird/client/PortAclTest.java | 1 - 6 files changed, 25 insertions(+), 21 deletions(-) diff --git a/app/src/androidTest/java/io/netbird/client/DnsResolutionTest.java b/app/src/androidTest/java/io/netbird/client/DnsResolutionTest.java index 41edfeff..4bb5fc15 100644 --- a/app/src/androidTest/java/io/netbird/client/DnsResolutionTest.java +++ b/app/src/androidTest/java/io/netbird/client/DnsResolutionTest.java @@ -92,7 +92,6 @@ public void resolvesPeerNameThroughTunnel() throws Exception { harness.enableTouchVisualization(); harness.grantVpnConsent(); - LoginFlow.ensureForceRelayDisabled(activity, harness.device()); profileName = LoginFlow.createAndSwitchToFreshProfile(activity, harness.device(), "dns"); LoginFlow.loginWithSetupKey(activity, harness.device(), setupKey); diff --git a/app/src/androidTest/java/io/netbird/client/E2eSuite.java b/app/src/androidTest/java/io/netbird/client/E2eSuite.java index 279fab4b..8cbe946c 100644 --- a/app/src/androidTest/java/io/netbird/client/E2eSuite.java +++ b/app/src/androidTest/java/io/netbird/client/E2eSuite.java @@ -1,5 +1,9 @@ package io.netbird.client; +import androidx.test.platform.app.InstrumentationRegistry; +import androidx.test.uiautomator.UiDevice; + +import org.junit.BeforeClass; import org.junit.runner.RunWith; import org.junit.runners.Suite; @@ -23,4 +27,14 @@ ExitNodeRouteTest.class, }) public class E2eSuite { + + /** + * Force relay is a global setting that defaults ON and would stop the + * relay-less peer connecting. Turn it off ONCE, before any test runs. + */ + @BeforeClass + public static void disableForceRelay() throws Exception { + UiDevice device = UiDevice.getInstance(InstrumentationRegistry.getInstrumentation()); + LoginFlow.setForceRelay(E2eAppRule.activity(), device, false); + } } diff --git a/app/src/androidTest/java/io/netbird/client/ExitNodeRouteTest.java b/app/src/androidTest/java/io/netbird/client/ExitNodeRouteTest.java index 5e08b5c7..7b368174 100644 --- a/app/src/androidTest/java/io/netbird/client/ExitNodeRouteTest.java +++ b/app/src/androidTest/java/io/netbird/client/ExitNodeRouteTest.java @@ -85,7 +85,6 @@ public void egressGoesThroughExitNode() throws Exception { harness.enableTouchVisualization(); harness.grantVpnConsent(); - LoginFlow.ensureForceRelayDisabled(activity, harness.device()); profileName = LoginFlow.createAndSwitchToFreshProfile(activity, harness.device(), "exit-node"); LoginFlow.loginWithSetupKey(activity, harness.device(), setupKey); diff --git a/app/src/androidTest/java/io/netbird/client/LoginFlow.java b/app/src/androidTest/java/io/netbird/client/LoginFlow.java index 5cdcbc38..c827a58e 100644 --- a/app/src/androidTest/java/io/netbird/client/LoginFlow.java +++ b/app/src/androidTest/java/io/netbird/client/LoginFlow.java @@ -236,18 +236,14 @@ static void removeProfile(MainActivity activity, UiDevice device, String profile Log.i(TAG, "Removed profile " + profileName); } - private static boolean forceRelayDisabled = false; - /** - * Turn off the "Force relay connection" toggle in the Advanced screen so - * peers can connect directly (P2P). It defaults ON (a global setting), which - * prevents the relay-less peer from connecting. This is a one-time suite - * pre-step: every test's {@code @Before} calls it, but it only acts once. + * Set the "Force relay connection" toggle (a global setting) to {@code + * enabled} via the Advanced screen. Each test sets the value it needs rather + * than relying on the previous test's state, so the toggle is reset between + * tests. Only clicks when the current state differs. */ - static void ensureForceRelayDisabled(MainActivity activity, UiDevice device) throws InterruptedException { - if (forceRelayDisabled) { - return; - } + static void setForceRelay(MainActivity activity, UiDevice device, boolean enabled) + throws InterruptedException { navigateTo(activity, R.id.nav_advanced); // The switch is far down the scrollable Advanced screen — scroll to it. @@ -258,15 +254,14 @@ static void ensureForceRelayDisabled(MainActivity activity, UiDevice device) thr dumpScreenshot(device, "force-relay-switch-missing"); fail("Force relay connection switch not found in Advanced screen"); } - if (toggle.isChecked()) { + if (toggle.isChecked() != enabled) { toggle.click(); // Toggling pops a "reconnection needed" warning; dismiss it. confirmDialog(device); - Log.i(TAG, "Disabled force relay connection"); + Log.i(TAG, "Set force relay connection to " + enabled); } else { - Log.i(TAG, "Force relay connection already off"); + Log.i(TAG, "Force relay connection already " + enabled); } - forceRelayDisabled = true; } /** diff --git a/app/src/androidTest/java/io/netbird/client/PeerConnectivityTest.java b/app/src/androidTest/java/io/netbird/client/PeerConnectivityTest.java index 87683067..5c6151a3 100644 --- a/app/src/androidTest/java/io/netbird/client/PeerConnectivityTest.java +++ b/app/src/androidTest/java/io/netbird/client/PeerConnectivityTest.java @@ -95,7 +95,8 @@ public void connectsWithoutRelay() throws Exception { /** * Shared body: fresh profile → login → connect → ping {@code peerFqdn} * through the tunnel. This is the Android equivalent of the Robot - * {@code Try Peer Connectivity ${peer_name}} keyword. + * {@code Try Peer Connectivity ${peer_name}} keyword. Force-relay is + * disabled once for the whole suite (see {@link E2eSuite}). */ private void connectAndPing(String peerFqdn, String scenario) throws Exception { Bundle args = InstrumentationRegistry.getArguments(); @@ -109,9 +110,6 @@ private void connectAndPing(String peerFqdn, String scenario) throws Exception { harness.enableTouchVisualization(); harness.grantVpnConsent(); - // Global pre-step (once for the whole suite): peers can't connect P2P - // with force-relay on, which it is by default. - LoginFlow.ensureForceRelayDisabled(activity, harness.device()); // 1. Create a fresh, isolated profile (Android equivalent of the Robot // suite's `netbird profile add test-`), then log in into it. diff --git a/app/src/androidTest/java/io/netbird/client/PortAclTest.java b/app/src/androidTest/java/io/netbird/client/PortAclTest.java index e4da1692..83dc72c8 100644 --- a/app/src/androidTest/java/io/netbird/client/PortAclTest.java +++ b/app/src/androidTest/java/io/netbird/client/PortAclTest.java @@ -98,7 +98,6 @@ public void connectsOnlyToAllowedPorts() throws Exception { harness.enableTouchVisualization(); harness.grantVpnConsent(); - LoginFlow.ensureForceRelayDisabled(activity, harness.device()); // Fresh profile + login, like the Robot suite's per-test InitNetBird. profileName = LoginFlow.createAndSwitchToFreshProfile(activity, harness.device(), "port-acl"); From 1b9d8940475c689d511534b513ab081d3d1d3f6d Mon Sep 17 00:00:00 2001 From: Zoltan Papp Date: Tue, 4 Aug 2026 18:26:27 +0200 Subject: [PATCH 8/9] ci: move instrumented e2e run to the private mobile-e2e repo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The instrumented-tests job logs the emulator into the production NetBird API with a setup key and uploads logcat/screen recordings as artifacts — none of which belongs on a public repo. The workflow, the secrets and the runner script now live in netbirdio/mobile-e2e, which checks out this repo and builds it with the build-android action. The test sources stay here. Co-Authored-By: Claude Opus 5 (1M context) --- .github/scripts/run-instrumented-tests.sh | 58 ------------ .github/workflows/build-debug.yml | 104 ---------------------- 2 files changed, 162 deletions(-) delete mode 100755 .github/scripts/run-instrumented-tests.sh diff --git a/.github/scripts/run-instrumented-tests.sh b/.github/scripts/run-instrumented-tests.sh deleted file mode 100755 index 28c27258..00000000 --- a/.github/scripts/run-instrumented-tests.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env bash -# Runs connectedDebugAndroidTest while capturing the emulator screen and -# logcat in the background. screenrecord caps each clip at 180s, so we loop -# until the test finishes; logcat streams continuously to a file. Both end -# up in screen-recordings/ and are uploaded as an artifact from the workflow. -# -# Expects $INSTRUMENTATION_NB_SETUP_KEY in the environment. - -set +e - -mkdir -p screen-recordings -adb shell rm -rf /sdcard/recordings -adb shell mkdir -p /sdcard/recordings - -adb logcat -c -adb logcat -v threadtime > screen-recordings/logcat.log 2>&1 & -LOGCAT_PID=$! - -# Sentinel must exist before the background loop starts; otherwise the first -# iteration sees no file and exits immediately. -touch /tmp/record_active -( - i=0 - while [ -f /tmp/record_active ]; do - seg=$(printf "seg_%03d.mp4" "$i") - echo "[record] starting $seg" - adb shell screenrecord --time-limit 180 --bit-rate 4000000 "/sdcard/recordings/$seg" - echo "[record] $seg exited" - i=$((i + 1)) - done - echo "[record] loop ended" -) & -REC_LOOP_PID=$! - -GRADLE_ARGS=( - --no-daemon :app:connectedDebugAndroidTest - -Pandroid.testInstrumentationRunnerArguments.class=io.netbird.client.E2eSuite - -Pandroid.testInstrumentationRunnerArguments.listener=io.netbird.client.FailFast - -Pandroid.testInstrumentationRunnerArguments.setupKey="$INSTRUMENTATION_NB_SETUP_KEY" -) - -if [ -n "$INSTRUMENTATION_EXIT_NODE_SETUP_KEY" ]; then - GRADLE_ARGS+=(-Pandroid.testInstrumentationRunnerArguments.exitNodeSetupKey="$INSTRUMENTATION_EXIT_NODE_SETUP_KEY") -fi - -./gradlew "${GRADLE_ARGS[@]}" -TEST_EXIT=$? - -rm -f /tmp/record_active -adb shell pkill -SIGINT screenrecord 2>/dev/null || true -wait "$REC_LOOP_PID" 2>/dev/null || true -sleep 3 -adb pull /sdcard/recordings ./screen-recordings/ || true - -kill "$LOGCAT_PID" 2>/dev/null || true -wait "$LOGCAT_PID" 2>/dev/null || true - -exit $TEST_EXIT diff --git a/.github/workflows/build-debug.yml b/.github/workflows/build-debug.yml index 1c767776..be49b42f 100644 --- a/.github/workflows/build-debug.yml +++ b/.github/workflows/build-debug.yml @@ -85,107 +85,3 @@ jobs: app/build/reports/tests/ tool/build/reports/tests/ retention-days: 3 - - instrumented-tests: - needs: build-debug - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository - runs-on: ubuntu-latest - timeout-minutes: 30 - environment: instrumentation-test-secrets - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - submodules: recursive - - - name: Setup Java - uses: actions/setup-java@v4 - with: - java-version: "17" - distribution: "adopt" - cache: "gradle" - - - name: Download AAR artifact - uses: actions/download-artifact@v4 - with: - name: netbird-aar - path: gomobile - - - name: Verify required secrets - env: - INSTRUMENTATION_NB_SETUP_KEY: ${{ secrets.INSTRUMENTATION_NB_SETUP_KEY }} - run: | - if [ -z "$INSTRUMENTATION_NB_SETUP_KEY" ]; then - echo "::error::INSTRUMENTATION_NB_SETUP_KEY repository secret is not configured" - exit 1 - fi - - - name: Enable KVM group perms - run: | - echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules - sudo udevadm control --reload-rules - sudo udevadm trigger --name-match=kvm - - - name: AVD cache - id: avd-cache - uses: actions/cache@v4 - with: - # ANDROID_HOME is set by the runner image but not visible to - # ${{ env.X }} at expression-eval time; hardcoded to the path on - # ubuntu-latest. - path: | - ~/.android/avd/* - ~/.android/adb* - /usr/local/lib/android/sdk/system-images/android-30/google_apis/x86_64 - key: avd-api30-google_apis-x86_64-pixel_3a-v1 - - - name: Create AVD snapshot - if: steps.avd-cache.outputs.cache-hit != 'true' - uses: reactivecircus/android-emulator-runner@v2 - with: - api-level: 30 - target: google_apis - arch: x86_64 - profile: pixel_3a - disk-size: 4096M - heap-size: 512M - force-avd-creation: false - emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim - disable-animations: true - script: echo "Generated AVD snapshot for caching." - - - name: Run instrumented tests - uses: reactivecircus/android-emulator-runner@v2 - env: - INSTRUMENTATION_NB_SETUP_KEY: ${{ secrets.INSTRUMENTATION_NB_SETUP_KEY }} - INSTRUMENTATION_EXIT_NODE_SETUP_KEY: ${{ secrets.INSTRUMENTATION_EXIT_NODE_SETUP_KEY }} - with: - api-level: 30 - target: google_apis - arch: x86_64 - profile: pixel_3a - disk-size: 4096M - heap-size: 512M - force-avd-creation: false - emulator-options: -no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim - disable-animations: true - script: bash .github/scripts/run-instrumented-tests.sh - - - name: Upload test results - if: always() - uses: actions/upload-artifact@v4 - with: - name: instrumented-test-results - path: | - app/build/reports/androidTests/ - tool/build/reports/androidTests/ - retention-days: 3 - - - name: Upload screen recordings - if: always() - uses: actions/upload-artifact@v4 - with: - name: instrumented-test-screen-recordings - path: screen-recordings/ - if-no-files-found: warn - retention-days: 3 From 30fd2f1ff5a72e493e305351d6d3f5b3d6c7739c Mon Sep 17 00:00:00 2001 From: Zoltan Papp Date: Tue, 4 Aug 2026 18:43:16 +0200 Subject: [PATCH 9/9] test: split instrumented tests from the e2e suite MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move the tests that need external infrastructure or secrets (the E2eSuite with its helpers, plus NetworkConnectivityStressTest) into the io.netbird.client.e2e package. The classic instrumented tests stay in the root package and run again in CI via a secret-free instrumented-tests job filtered with notPackage=io.netbird.client.e2e — safe for fork PRs. The e2e package is executed from the private mobile-e2e repo, which selects it with class=io.netbird.client.e2e.E2eSuite. New tests sort themselves: anything placed in the e2e package is automatically excluded from the public CI run, no exclude list to maintain. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/build-debug.yml | 85 +++++++++++++++++++ app/src/androidTest/README.md | 2 +- .../client/{ => e2e}/DnsResolutionTest.java | 4 +- .../netbird/client/{ => e2e}/E2eAppRule.java | 4 +- .../io/netbird/client/{ => e2e}/E2eSuite.java | 6 +- .../client/{ => e2e}/ExitNodeRouteTest.java | 4 +- .../io/netbird/client/{ => e2e}/FailFast.java | 2 +- .../netbird/client/{ => e2e}/LoginFlow.java | 5 +- .../NetworkConnectivityStressTest.java | 8 +- .../{ => e2e}/PeerConnectivityTest.java | 4 +- .../netbird/client/{ => e2e}/PortAclTest.java | 4 +- .../client/{ => e2e}/SetupKeyAuthTest.java | 2 +- .../client/{ => e2e}/VpnTestHarness.java | 5 +- 13 files changed, 121 insertions(+), 14 deletions(-) rename app/src/androidTest/java/io/netbird/client/{ => e2e}/DnsResolutionTest.java (98%) rename app/src/androidTest/java/io/netbird/client/{ => e2e}/E2eAppRule.java (97%) rename app/src/androidTest/java/io/netbird/client/{ => e2e}/E2eSuite.java (93%) rename app/src/androidTest/java/io/netbird/client/{ => e2e}/ExitNodeRouteTest.java (98%) rename app/src/androidTest/java/io/netbird/client/{ => e2e}/FailFast.java (97%) rename app/src/androidTest/java/io/netbird/client/{ => e2e}/LoginFlow.java (99%) rename app/src/androidTest/java/io/netbird/client/{ => e2e}/NetworkConnectivityStressTest.java (99%) rename app/src/androidTest/java/io/netbird/client/{ => e2e}/PeerConnectivityTest.java (98%) rename app/src/androidTest/java/io/netbird/client/{ => e2e}/PortAclTest.java (98%) rename app/src/androidTest/java/io/netbird/client/{ => e2e}/SetupKeyAuthTest.java (98%) rename app/src/androidTest/java/io/netbird/client/{ => e2e}/VpnTestHarness.java (99%) diff --git a/.github/workflows/build-debug.yml b/.github/workflows/build-debug.yml index be49b42f..515e6f66 100644 --- a/.github/workflows/build-debug.yml +++ b/.github/workflows/build-debug.yml @@ -85,3 +85,88 @@ jobs: app/build/reports/tests/ tool/build/reports/tests/ retention-days: 3 + + # Classic instrumented tests: everything outside io.netbird.client.e2e. + # They need no secrets and no external infrastructure, so they can run on + # every PR, forks included. The e2e package (production API, setup keys) + # runs from the private mobile-e2e repo instead. + instrumented-tests: + needs: build-debug + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Setup Java + uses: actions/setup-java@v4 + with: + java-version: "17" + distribution: "adopt" + cache: "gradle" + + - name: Download AAR artifact + uses: actions/download-artifact@v4 + with: + name: netbird-aar + path: gomobile + + - name: Enable KVM group perms + run: | + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules + sudo udevadm control --reload-rules + sudo udevadm trigger --name-match=kvm + + - name: AVD cache + id: avd-cache + uses: actions/cache@v4 + with: + # ANDROID_HOME is set by the runner image but not visible to + # ${{ env.X }} at expression-eval time; hardcoded to the path on + # ubuntu-latest. + path: | + ~/.android/avd/* + ~/.android/adb* + /usr/local/lib/android/sdk/system-images/android-30/google_apis/x86_64 + key: avd-api30-google_apis-x86_64-pixel_3a-v1 + + - name: Create AVD snapshot + if: steps.avd-cache.outputs.cache-hit != 'true' + uses: reactivecircus/android-emulator-runner@v2 + with: + api-level: 30 + target: google_apis + arch: x86_64 + profile: pixel_3a + disk-size: 4096M + heap-size: 512M + force-avd-creation: false + emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim + disable-animations: true + script: echo "Generated AVD snapshot for caching." + + - name: Run instrumented tests + uses: reactivecircus/android-emulator-runner@v2 + with: + api-level: 30 + target: google_apis + arch: x86_64 + profile: pixel_3a + disk-size: 4096M + heap-size: 512M + force-avd-creation: false + emulator-options: -no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim + disable-animations: true + script: ./gradlew --no-daemon connectedDebugAndroidTest -Pandroid.testInstrumentationRunnerArguments.notPackage=io.netbird.client.e2e + + - name: Upload test results + if: always() + uses: actions/upload-artifact@v4 + with: + name: instrumented-test-results + path: | + app/build/reports/androidTests/ + tool/build/reports/androidTests/ + retention-days: 3 diff --git a/app/src/androidTest/README.md b/app/src/androidTest/README.md index 71c504a9..7c94b2ef 100644 --- a/app/src/androidTest/README.md +++ b/app/src/androidTest/README.md @@ -22,7 +22,7 @@ adb install -r -t app/build/outputs/apk/debug/app-debug.apk adb install -r -t app/build/outputs/apk/androidTest/debug/app-debug-androidTest.apk # 3. Run the test directly via adb -adb shell am instrument -w -e class io.netbird.client.NetworkConnectivityStressTest \ +adb shell am instrument -w -e class io.netbird.client.e2e.NetworkConnectivityStressTest \ io.netbird.client.test/androidx.test.runner.AndroidJUnitRunner ``` diff --git a/app/src/androidTest/java/io/netbird/client/DnsResolutionTest.java b/app/src/androidTest/java/io/netbird/client/e2e/DnsResolutionTest.java similarity index 98% rename from app/src/androidTest/java/io/netbird/client/DnsResolutionTest.java rename to app/src/androidTest/java/io/netbird/client/e2e/DnsResolutionTest.java index 4bb5fc15..3f0236ca 100644 --- a/app/src/androidTest/java/io/netbird/client/DnsResolutionTest.java +++ b/app/src/androidTest/java/io/netbird/client/e2e/DnsResolutionTest.java @@ -1,4 +1,6 @@ -package io.netbird.client; +package io.netbird.client.e2e; + +import io.netbird.client.MainActivity; import android.os.Bundle; import android.util.Log; diff --git a/app/src/androidTest/java/io/netbird/client/E2eAppRule.java b/app/src/androidTest/java/io/netbird/client/e2e/E2eAppRule.java similarity index 97% rename from app/src/androidTest/java/io/netbird/client/E2eAppRule.java rename to app/src/androidTest/java/io/netbird/client/e2e/E2eAppRule.java index 79826dbc..3c09cd2a 100644 --- a/app/src/androidTest/java/io/netbird/client/E2eAppRule.java +++ b/app/src/androidTest/java/io/netbird/client/e2e/E2eAppRule.java @@ -1,4 +1,6 @@ -package io.netbird.client; +package io.netbird.client.e2e; + +import io.netbird.client.MainActivity; import android.app.Activity; import android.app.Instrumentation; diff --git a/app/src/androidTest/java/io/netbird/client/E2eSuite.java b/app/src/androidTest/java/io/netbird/client/e2e/E2eSuite.java similarity index 93% rename from app/src/androidTest/java/io/netbird/client/E2eSuite.java rename to app/src/androidTest/java/io/netbird/client/e2e/E2eSuite.java index 8cbe946c..fdcd4384 100644 --- a/app/src/androidTest/java/io/netbird/client/E2eSuite.java +++ b/app/src/androidTest/java/io/netbird/client/e2e/E2eSuite.java @@ -1,4 +1,6 @@ -package io.netbird.client; +package io.netbird.client.e2e; + +import io.netbird.client.MainActivity; import androidx.test.platform.app.InstrumentationRegistry; import androidx.test.uiautomator.UiDevice; @@ -15,7 +17,7 @@ * *

Run with: * 

- *   -Pandroid.testInstrumentationRunnerArguments.class=io.netbird.client.E2eSuite
+ *   -Pandroid.testInstrumentationRunnerArguments.class=io.netbird.client.e2e.E2eSuite
  *
*/ @RunWith(Suite.class) diff --git a/app/src/androidTest/java/io/netbird/client/ExitNodeRouteTest.java b/app/src/androidTest/java/io/netbird/client/e2e/ExitNodeRouteTest.java similarity index 98% rename from app/src/androidTest/java/io/netbird/client/ExitNodeRouteTest.java rename to app/src/androidTest/java/io/netbird/client/e2e/ExitNodeRouteTest.java index 7b368174..191cabed 100644 --- a/app/src/androidTest/java/io/netbird/client/ExitNodeRouteTest.java +++ b/app/src/androidTest/java/io/netbird/client/e2e/ExitNodeRouteTest.java @@ -1,4 +1,6 @@ -package io.netbird.client; +package io.netbird.client.e2e; + +import io.netbird.client.MainActivity; import android.os.Bundle; import android.util.Log; diff --git a/app/src/androidTest/java/io/netbird/client/FailFast.java b/app/src/androidTest/java/io/netbird/client/e2e/FailFast.java similarity index 97% rename from app/src/androidTest/java/io/netbird/client/FailFast.java rename to app/src/androidTest/java/io/netbird/client/e2e/FailFast.java index d81ba059..7980fa13 100644 --- a/app/src/androidTest/java/io/netbird/client/FailFast.java +++ b/app/src/androidTest/java/io/netbird/client/e2e/FailFast.java @@ -1,4 +1,4 @@ -package io.netbird.client; +package io.netbird.client.e2e; import androidx.test.internal.runner.listener.InstrumentationRunListener; diff --git a/app/src/androidTest/java/io/netbird/client/LoginFlow.java b/app/src/androidTest/java/io/netbird/client/e2e/LoginFlow.java similarity index 99% rename from app/src/androidTest/java/io/netbird/client/LoginFlow.java rename to app/src/androidTest/java/io/netbird/client/e2e/LoginFlow.java index c827a58e..d9823576 100644 --- a/app/src/androidTest/java/io/netbird/client/LoginFlow.java +++ b/app/src/androidTest/java/io/netbird/client/e2e/LoginFlow.java @@ -1,4 +1,7 @@ -package io.netbird.client; +package io.netbird.client.e2e; + +import io.netbird.client.MainActivity; +import io.netbird.client.R; import android.os.Bundle; import android.util.Log; diff --git a/app/src/androidTest/java/io/netbird/client/NetworkConnectivityStressTest.java b/app/src/androidTest/java/io/netbird/client/e2e/NetworkConnectivityStressTest.java similarity index 99% rename from app/src/androidTest/java/io/netbird/client/NetworkConnectivityStressTest.java rename to app/src/androidTest/java/io/netbird/client/e2e/NetworkConnectivityStressTest.java index a7b1931b..3ba6514b 100644 --- a/app/src/androidTest/java/io/netbird/client/NetworkConnectivityStressTest.java +++ b/app/src/androidTest/java/io/netbird/client/e2e/NetworkConnectivityStressTest.java @@ -1,4 +1,8 @@ -package io.netbird.client; +package io.netbird.client.e2e; + +import io.netbird.client.MainActivity; +import io.netbird.client.StateListener; +import io.netbird.client.StateListenerAdapter; import android.app.Instrumentation; import android.app.UiAutomation; @@ -60,7 +64,7 @@ *

Run with

* 
  * ./gradlew connectedAndroidTest \
- *   -Pandroid.testInstrumentationRunnerArguments.class=io.netbird.client.NetworkConnectivityStressTest
+ *   -Pandroid.testInstrumentationRunnerArguments.class=io.netbird.client.e2e.NetworkConnectivityStressTest
  *
*/ @RunWith(AndroidJUnit4.class) diff --git a/app/src/androidTest/java/io/netbird/client/PeerConnectivityTest.java b/app/src/androidTest/java/io/netbird/client/e2e/PeerConnectivityTest.java similarity index 98% rename from app/src/androidTest/java/io/netbird/client/PeerConnectivityTest.java rename to app/src/androidTest/java/io/netbird/client/e2e/PeerConnectivityTest.java index 5c6151a3..11cd3f96 100644 --- a/app/src/androidTest/java/io/netbird/client/PeerConnectivityTest.java +++ b/app/src/androidTest/java/io/netbird/client/e2e/PeerConnectivityTest.java @@ -1,4 +1,6 @@ -package io.netbird.client; +package io.netbird.client.e2e; + +import io.netbird.client.MainActivity; import android.os.Bundle; import android.util.Log; diff --git a/app/src/androidTest/java/io/netbird/client/PortAclTest.java b/app/src/androidTest/java/io/netbird/client/e2e/PortAclTest.java similarity index 98% rename from app/src/androidTest/java/io/netbird/client/PortAclTest.java rename to app/src/androidTest/java/io/netbird/client/e2e/PortAclTest.java index 83dc72c8..3d486ab3 100644 --- a/app/src/androidTest/java/io/netbird/client/PortAclTest.java +++ b/app/src/androidTest/java/io/netbird/client/e2e/PortAclTest.java @@ -1,4 +1,6 @@ -package io.netbird.client; +package io.netbird.client.e2e; + +import io.netbird.client.MainActivity; import android.os.Bundle; import android.util.Log; diff --git a/app/src/androidTest/java/io/netbird/client/SetupKeyAuthTest.java b/app/src/androidTest/java/io/netbird/client/e2e/SetupKeyAuthTest.java similarity index 98% rename from app/src/androidTest/java/io/netbird/client/SetupKeyAuthTest.java rename to app/src/androidTest/java/io/netbird/client/e2e/SetupKeyAuthTest.java index f53a204f..7ab55dea 100644 --- a/app/src/androidTest/java/io/netbird/client/SetupKeyAuthTest.java +++ b/app/src/androidTest/java/io/netbird/client/e2e/SetupKeyAuthTest.java @@ -1,4 +1,4 @@ -package io.netbird.client; +package io.netbird.client.e2e; import android.os.Bundle; diff --git a/app/src/androidTest/java/io/netbird/client/VpnTestHarness.java b/app/src/androidTest/java/io/netbird/client/e2e/VpnTestHarness.java similarity index 99% rename from app/src/androidTest/java/io/netbird/client/VpnTestHarness.java rename to app/src/androidTest/java/io/netbird/client/e2e/VpnTestHarness.java index 6d7ec523..77732a42 100644 --- a/app/src/androidTest/java/io/netbird/client/VpnTestHarness.java +++ b/app/src/androidTest/java/io/netbird/client/e2e/VpnTestHarness.java @@ -1,4 +1,7 @@ -package io.netbird.client; +package io.netbird.client.e2e; + +import io.netbird.client.MainActivity; +import io.netbird.client.StateListener; import android.app.Instrumentation; import android.app.UiAutomation;