diff --git a/.azure-pipelines/1es-integration-tests-private.yml b/.azure-pipelines/1es-integration-tests-private.yml new file mode 100644 index 000000000..751f04aef --- /dev/null +++ b/.azure-pipelines/1es-integration-tests-private.yml @@ -0,0 +1,155 @@ +# Private-cluster integration test (1ES pipeline). +# +# Runs the k8s-deploy action against a real private AKS cluster. Lives in Azure +# DevOps (not GitHub Actions) because GitHub-repo Federated Identity Credentials +# are being removed; auth uses a governed WIF service connection instead. +# +# The node24 JS action is invoked as `node lib/index.js` with inputs supplied via +# INPUT_* env vars. There is no action.yml default injection here, so keep INPUT_* +# in sync with action.yml; skip-tls-verify (getBooleanInput) and GITHUB_WORKFLOW +# must be set or the action throws. +# +# The pipeline does not create/delete the resource group (it must already exist); +# it only creates and deletes a per-build cluster inside it. + +trigger: + branches: + include: + - main + +pr: + branches: + include: + - releases/* + +resources: + repositories: + - repository: 1esPipelines + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release + +variables: + serviceConnection: k8s-deploy-intg-test-svc-conn + resourceGroup: k8s-deploy-intg-rg + location: eastus2 + clusterName: test-$(Build.BuildId) + namespace: test-$(Build.BuildId) + workflowFriendlyName: k8s-deploy-integration-tests-private + +extends: + template: v1/1ES.Unofficial.PipelineTemplate.yml@1esPipelines + parameters: + pool: + name: staging-pool-amd64-mariner-2 + image: 1es-azlinux-3-amd64-custom-disk + os: linux + hostArchitecture: amd64 + sdl: + sourceAnalysisPool: + name: staging-pool-amd64-mariner-2 + image: azcu-agent-amd64-windows-22-img + os: windows + hostArchitecture: amd64 + stages: + - stage: integration_test + displayName: Private cluster integration tests + jobs: + - job: run_integration_test + displayName: Run Private Cluster Integration Tests + timeoutInMinutes: 60 + steps: + - task: NodeTool@0 + displayName: Install Node.js + inputs: + versionSpec: '24.x' + + - script: | + rm -rf node_modules/ + npm install + displayName: Install dependencies + + - script: npm run build + displayName: Build + + - task: AzureCLI@2 + displayName: Create private AKS cluster and set context + inputs: + azureSubscription: $(serviceConnection) + scriptType: bash + scriptLocation: inlineScript + inlineScript: | + set -e + set +x + # RG must already exist; do not run `az group create`. + az aks create --name $(clusterName) --resource-group $(resourceGroup) --enable-private-cluster --generate-ssh-keys + az aks get-credentials --resource-group $(resourceGroup) --name $(clusterName) + + - task: AzureCLI@2 + displayName: Create namespace to run tests + inputs: + azureSubscription: $(serviceConnection) + scriptType: bash + scriptLocation: inlineScript + inlineScript: | + set -e + az aks command invoke --resource-group $(resourceGroup) --name $(clusterName) --command "kubectl create ns $(namespace)" + + - task: UsePythonVersion@0 + displayName: Install Python + inputs: + versionSpec: '3.x' + + - task: AzureCLI@2 + displayName: Executing deploy action for pod + inputs: + azureSubscription: $(serviceConnection) + scriptType: bash + scriptLocation: inlineScript + inlineScript: | + set -e + node lib/index.js + env: + KUBECONFIG: $(HOME)/.kube/config + # Inputs supplied via INPUT_* (no action.yml injection here). + INPUT_ACTION: deploy + INPUT_STRATEGY: basic + INPUT_TRAFFIC-SPLIT-METHOD: pod + INPUT_SKIP-TLS-VERIFY: 'false' + INPUT_NAMESPACE: $(namespace) + INPUT_IMAGES: nginx:1.14.2 + INPUT_MANIFESTS: | + test/integration/manifests/test.yml + test/integration/manifests/test2.yml + INPUT_PRIVATE-CLUSTER: 'true' + INPUT_RESOURCE-GROUP: $(resourceGroup) + INPUT_NAME: $(clusterName) + INPUT_RESOURCE-TYPE: Microsoft.ContainerService/managedClusters + # Required by the action; also the workflowFriendlyName label. + GITHUB_WORKFLOW: $(workflowFriendlyName) + + - task: AzureCLI@2 + displayName: Checking if deployments and services were created + inputs: + azureSubscription: $(serviceConnection) + scriptType: bash + scriptLocation: inlineScript + inlineScript: | + set -e + python test/integration/k8s-deploy-test.py private=$(clusterName) resourceGroup=$(resourceGroup) namespace=$(namespace) kind=Deployment name=nginx-deployment containerName=nginx:1.14.2 labels=app:nginx,workflow:actions.github.com-k8s-deploy,workflowFriendlyName:$(workflowFriendlyName) selectorLabels=app:nginx + python test/integration/k8s-deploy-test.py private=$(clusterName) resourceGroup=$(resourceGroup) namespace=$(namespace) kind=Service name=nginx-service labels=workflow:actions.github.com-k8s-deploy,workflowFriendlyName:$(workflowFriendlyName) selectorLabels=app:nginx + + python test/integration/k8s-deploy-test.py private=$(clusterName) resourceGroup=$(resourceGroup) namespace=$(namespace) kind=Deployment name=nginx-deployment2 containerName=nginx:1.14.2 labels=app:nginx2,workflow:actions.github.com-k8s-deploy,workflowFriendlyName:$(workflowFriendlyName) selectorLabels=app:nginx2 + python test/integration/k8s-deploy-test.py private=$(clusterName) resourceGroup=$(resourceGroup) namespace=$(namespace) kind=Service name=nginx-service2 labels=workflow:actions.github.com-k8s-deploy,workflowFriendlyName:$(workflowFriendlyName) selectorLabels=app:nginx2 + + - task: AzureCLI@2 + displayName: Clean up AKS cluster + condition: always() + inputs: + azureSubscription: $(serviceConnection) + scriptType: bash + scriptLocation: inlineScript + inlineScript: | + # Delete only the cluster; the shared RG persists. + echo "deleting AKS cluster $(clusterName)" + az aks delete --yes --resource-group $(resourceGroup) --name $(clusterName) || true diff --git a/.github/workflows/run-integration-tests-private.yml b/.github/workflows/run-integration-tests-private.yml deleted file mode 100644 index 3e6ac5988..000000000 --- a/.github/workflows/run-integration-tests-private.yml +++ /dev/null @@ -1,85 +0,0 @@ -name: Cluster Integration Tests - private cluster -on: - pull_request: - branches: - - 'releases/*' - push: - branches: - - main - workflow_dispatch: - -jobs: - run-integration-test: - name: Run Minikube Integration Tests - runs-on: ubuntu-22.04 - env: - KUBECONFIG: /home/runner/.kube/config - NAMESPACE: test-${{ github.run_id }} - permissions: - contents: read - id-token: write - steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - - - name: Install dependencies - run: | - rm -rf node_modules/ - npm install - - - name: Build - run: npm run build - - - name: Azure login - uses: azure/login@v3.0.0 - with: - client-id: ${{ secrets.AZURE_CLIENT_ID }} - tenant-id: ${{ secrets.AZURE_TENANT_ID }} - subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} - - - uses: Azure/setup-kubectl@829323503d1be3d00ca8346e5391ca0b07a9ab0d # v5.1.0 - name: Install Kubectl - - - name: Create private AKS cluster and set context - run: | - set +x - # create cluster - az group create --location eastus2 --name ${{ env.NAMESPACE }} - az aks create --name ${{ env.NAMESPACE }} --resource-group ${{ env.NAMESPACE }} --enable-private-cluster --generate-ssh-keys - az aks get-credentials --resource-group ${{ env.NAMESPACE }} --name ${{ env.NAMESPACE }} - - - name: Create namespace to run tests - run: | - az aks command invoke --resource-group ${{ env.NAMESPACE }} --name ${{ env.NAMESPACE }} --command "kubectl create ns ${{ env.NAMESPACE }}" - - - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # 6.3.0 - name: Install Python - with: - python-version: '3.x' - - - name: Executing deploy action for pod - uses: ./ - with: - namespace: ${{ env.NAMESPACE }} - images: nginx:1.14.2 - manifests: | - test/integration/manifests/test.yml - test/integration/manifests/test2.yml - action: deploy - private-cluster: true - resource-group: ${{ env.NAMESPACE }} - name: ${{ env.NAMESPACE }} - - - name: Checking if deployments and services were created - run: | - python test/integration/k8s-deploy-test.py private=${{ env.NAMESPACE }} namespace=${{ env.NAMESPACE }} kind=Deployment name=nginx-deployment containerName=nginx:1.14.2 labels=app:nginx,workflow:actions.github.com-k8s-deploy,workflowFriendlyName:Cluster_Integration_Tests_-_private_cluster selectorLabels=app:nginx - python test/integration/k8s-deploy-test.py private=${{ env.NAMESPACE }} namespace=${{ env.NAMESPACE }} kind=Service name=nginx-service labels=workflow:actions.github.com-k8s-deploy,workflowFriendlyName:Cluster_Integration_Tests_-_private_cluster selectorLabels=app:nginx - - python test/integration/k8s-deploy-test.py private=${{ env.NAMESPACE }} namespace=${{ env.NAMESPACE }} kind=Deployment name=nginx-deployment2 containerName=nginx:1.14.2 labels=app:nginx2,workflow:actions.github.com-k8s-deploy,workflowFriendlyName:Cluster_Integration_Tests_-_private_cluster selectorLabels=app:nginx2 - python test/integration/k8s-deploy-test.py private=${{ env.NAMESPACE }} namespace=${{ env.NAMESPACE }} kind=Service name=nginx-service2 labels=workflow:actions.github.com-k8s-deploy,workflowFriendlyName:Cluster_Integration_Tests_-_private_cluster selectorLabels=app:nginx2 - - - name: Clean up AKS cluster - if: ${{ always() }} - run: | - echo "deleting AKS cluster and resource group" - az aks delete --yes --resource-group ${{ env.NAMESPACE }} --name ${{ env.NAMESPACE }} - az group delete --resource-group ${{ env.NAMESPACE }} --yes diff --git a/test/integration/k8s-deploy-test.py b/test/integration/k8s-deploy-test.py index 20c0ce7a5..cf07e5d47 100644 --- a/test/integration/k8s-deploy-test.py +++ b/test/integration/k8s-deploy-test.py @@ -21,6 +21,7 @@ ingressServicesKey = "ingressServices" tsServicesKey = "tsServices" privateKey = "private" +resourceGroupKey = "resourceGroup" def parseArgs(sysArgs): @@ -211,7 +212,12 @@ def main(): try: if privateKey in parsedArgs: uniqueName = parsedArgs[privateKey] - azPrefix = f"az aks command invoke --resource-group {uniqueName} --name {uniqueName} --command " + # The resource group defaults to the cluster's unique name for + # backward compatibility (older workflows named the RG and cluster + # identically). When the RG is shared/pre-created and differs from + # the cluster name, pass it explicitly via resourceGroup=. + resourceGroup = parsedArgs.get(resourceGroupKey, uniqueName) + azPrefix = f"az aks command invoke --resource-group {resourceGroup} --name {uniqueName} --command " cmd = azPrefix + "'" + cmd + "'" outputString = os.popen(cmd).read() successExit = "exitcode=0"