From c02666bca2920c226d31a1be51e8d602e1e8983c Mon Sep 17 00:00:00 2001 From: gangwgr Date: Wed, 12 Aug 2026 17:09:20 +0530 Subject: [PATCH] Add TLSAdherence support hypershift tls scanner job --- .../openshift-tls-scanner-main.yaml | 4 + .../hypershift-modern-tls-commands.sh | 298 ++++++++++++++++-- .../modern-tls/hypershift-modern-tls-ref.yaml | 46 ++- .../tls-scanner-hypershift-aws-workflow.yaml | 6 + 4 files changed, 330 insertions(+), 24 deletions(-) diff --git a/ci-operator/config/openshift/tls-scanner/openshift-tls-scanner-main.yaml b/ci-operator/config/openshift/tls-scanner/openshift-tls-scanner-main.yaml index decde528141ab..d98912e5def74 100644 --- a/ci-operator/config/openshift/tls-scanner/openshift-tls-scanner-main.yaml +++ b/ci-operator/config/openshift/tls-scanner/openshift-tls-scanner-main.yaml @@ -148,7 +148,11 @@ tests: steps: cluster_profile: hypershift-aws env: + GUEST_FEATURE_SET: TechPreviewNoUpgrade PQC_CHECK: "true" + TECH_PREVIEW_NO_UPGRADE: "true" + TLS_13_ENABLE_TLS_ADHERENCE: "true" + TLS_13_TLS_ADHERENCE_POLICY: StrictAllComponents workflow: tls-scanner-hypershift-aws zz_generated_metadata: branch: main diff --git a/ci-operator/step-registry/hypershift/modern-tls/hypershift-modern-tls-commands.sh b/ci-operator/step-registry/hypershift/modern-tls/hypershift-modern-tls-commands.sh index ba72b68c5dcbc..28454270c84de 100755 --- a/ci-operator/step-registry/hypershift/modern-tls/hypershift-modern-tls-commands.sh +++ b/ci-operator/step-registry/hypershift/modern-tls/hypershift-modern-tls-commands.sh @@ -26,21 +26,115 @@ fi echo "Found HostedCluster: ${HOSTED_CLUSTER_NAME} in namespace ${HOSTED_CLUSTER_NAMESPACE}" echo "Hosted control plane namespace: ${HCP_NAMESPACE}" +NODEPOOL_CONFIG_VERSION_ANNOTATION="hypershift.openshift.io/nodePoolCurrentConfigVersion" +NODEPOOL_CONFIG_ANNOTATION="hypershift.openshift.io/nodePoolCurrentConfig" + +nodepool_annotation() { + local nodepool_name=$1 + local annotation_key=$2 + oc get "nodepool/${nodepool_name}" -n "${HOSTED_CLUSTER_NAMESPACE}" \ + -o go-template="{{index .metadata.annotations \"${annotation_key}\"}}" +} + +nodepool_config_version() { + nodepool_annotation "$1" "${NODEPOOL_CONFIG_VERSION_ANNOTATION}" +} + +nodepool_current_config() { + nodepool_annotation "$1" "${NODEPOOL_CONFIG_ANNOTATION}" +} + +nodepool_condition_status() { + local nodepool_name=$1 + local condition_type=$2 + oc get "nodepool/${nodepool_name}" -n "${HOSTED_CLUSTER_NAMESPACE}" \ + -o jsonpath="{.status.conditions[?(@.type==\"${condition_type}\")].status}" 2>/dev/null || true +} + +mapfile -t NODEPOOLS < <(oc get nodepool -n "${HOSTED_CLUSTER_NAMESPACE}" \ + -o jsonpath="{range .items[?(@.spec.clusterName==\"${HOSTED_CLUSTER_NAME}\")]}{.metadata.name}{\"\\n\"}{end}") +if [[ ${#NODEPOOLS[@]} -eq 0 ]]; then + mapfile -t NODEPOOLS < <(oc get nodepool -n "${HOSTED_CLUSTER_NAMESPACE}" -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}') +fi + +declare -A NODEPOOL_BASELINE_CONFIG_VERSIONS=() +declare -A NODEPOOL_BASELINE_CONFIG_HASHES=() +if [[ ${#NODEPOOLS[@]} -eq 0 ]]; then + echo "Warning: no NodePools found for HostedCluster ${HOSTED_CLUSTER_NAME}" +else + echo "Recording NodePool config versions before TLS change..." + for np in "${NODEPOOLS[@]}"; do + baseline_version="" + baseline_config="" + for _ in $(seq 1 40); do + baseline_version="$(nodepool_config_version "${np}")" + baseline_config="$(nodepool_current_config "${np}")" + if [[ -n "${baseline_version}" ]]; then + break + fi + echo " NodePool/${np}: waiting for ${NODEPOOL_CONFIG_VERSION_ANNOTATION} to be set..." + sleep 15 + done + if [[ -z "${baseline_version}" ]]; then + echo "Error: could not read ${NODEPOOL_CONFIG_VERSION_ANNOTATION} on NodePool/${np} before TLS change" + oc get "nodepool/${np}" -n "${HOSTED_CLUSTER_NAMESPACE}" -o yaml || true + exit 1 + fi + NODEPOOL_BASELINE_CONFIG_VERSIONS["${np}"]="${baseline_version}" + NODEPOOL_BASELINE_CONFIG_HASHES["${np}"]="${baseline_config}" + echo " NodePool/${np}: ${NODEPOOL_CONFIG_VERSION_ANNOTATION}='${baseline_version}', ${NODEPOOL_CONFIG_ANNOTATION}='${baseline_config}'" + done +fi + kas_generation="$(oc get deployment -n "${HCP_NAMESPACE}" kube-apiserver -o jsonpath='{.metadata.generation}')" +case "${TLS_13_ENABLE_TLS_ADHERENCE:-}" in +true) + case "${TLS_13_TLS_ADHERENCE_POLICY}" in + LegacyAdheringComponentsOnly|StrictAllComponents) ;; + *) + echo "Invalid TLS_13_TLS_ADHERENCE_POLICY='${TLS_13_TLS_ADHERENCE_POLICY}' (expected LegacyAdheringComponentsOnly or StrictAllComponents)" + exit 1 + ;; + esac + ;; +false|"") + ;; +*) + echo "Invalid TLS_13_ENABLE_TLS_ADHERENCE='${TLS_13_ENABLE_TLS_ADHERENCE}' (expected literal \"true\" or \"false\")" + exit 1 + ;; +esac + echo "Applying Modern TLS Security Profile to HostedCluster..." -oc patch hostedcluster -n "${HOSTED_CLUSTER_NAMESPACE}" "${HOSTED_CLUSTER_NAME}" --type=merge -p '{ - "spec": { - "configuration": { - "apiServer": { - "tlsSecurityProfile": { - "type": "Modern", - "modern": {} +if [[ "${TLS_13_ENABLE_TLS_ADHERENCE:-}" == "true" ]]; then + oc patch hostedcluster -n "${HOSTED_CLUSTER_NAMESPACE}" "${HOSTED_CLUSTER_NAME}" --type=merge -p "{ + \"spec\": { + \"configuration\": { + \"apiServer\": { + \"tlsAdherence\": \"${TLS_13_TLS_ADHERENCE_POLICY}\", + \"tlsSecurityProfile\": { + \"type\": \"Modern\", + \"modern\": {} + } } } } - } -}' + }" +else + oc patch hostedcluster -n "${HOSTED_CLUSTER_NAMESPACE}" "${HOSTED_CLUSTER_NAME}" --type=merge -p '{ + "spec": { + "configuration": { + "apiServer": { + "tlsSecurityProfile": { + "type": "Modern", + "modern": {} + } + } + } + } + }' +fi hc_tls_profile=$(oc get hostedcluster -n "${HOSTED_CLUSTER_NAMESPACE}" "${HOSTED_CLUSTER_NAME}" -o jsonpath='{.spec.configuration.apiServer.tlsSecurityProfile.type}') if [[ "${hc_tls_profile}" != "Modern" ]]; then @@ -49,6 +143,15 @@ if [[ "${hc_tls_profile}" != "Modern" ]]; then fi echo "✓ HostedCluster spec.configuration.apiServer.tlsSecurityProfile.type is Modern" +if [[ "${TLS_13_ENABLE_TLS_ADHERENCE:-}" == "true" ]]; then + hc_tls_adherence=$(oc get hostedcluster -n "${HOSTED_CLUSTER_NAMESPACE}" "${HOSTED_CLUSTER_NAME}" -o jsonpath='{.spec.configuration.apiServer.tlsAdherence}') + if [[ "${hc_tls_adherence}" != "${TLS_13_TLS_ADHERENCE_POLICY}" ]]; then + echo "Error: HostedCluster tlsAdherence is '${hc_tls_adherence}', expected '${TLS_13_TLS_ADHERENCE_POLICY}'" + exit 1 + fi + echo "✓ HostedCluster spec.configuration.apiServer.tlsAdherence is ${hc_tls_adherence}" +fi + echo "Waiting for kube-apiserver to reconcile the TLS profile..." rollout_deadline=$((SECONDS + 300)) while (( SECONDS < rollout_deadline )); do @@ -74,6 +177,76 @@ for dep in ${HCP_DEPLOYMENTS}; do done echo "All control plane deployments rolled out." +wait_nodepool_rollout() { + if [[ ${#NODEPOOLS[@]} -eq 0 ]]; then + return 0 + fi + + echo "Waiting for NodePool config rollout after HostedCluster TLS configuration change..." + local timeout_seconds + timeout_seconds="${NODEPOOL_ROLLOUT_TIMEOUT_SECONDS:-3600}" + local deadline=$((SECONDS + timeout_seconds)) + + for np in "${NODEPOOLS[@]}"; do + local baseline_version="${NODEPOOL_BASELINE_CONFIG_VERSIONS[${np}]}" + local baseline_config="${NODEPOOL_BASELINE_CONFIG_HASHES[${np}]}" + local rollout_complete=false + local saw_updating_config=false + + echo " Waiting for NodePool/${np} config rollout (version baseline='${baseline_version}', config baseline='${baseline_config}')..." + while (( SECONDS < deadline )); do + local current_version current_config updating_config all_healthy + current_version="$(nodepool_config_version "${np}")" + current_config="$(nodepool_current_config "${np}")" + updating_config="$(nodepool_condition_status "${np}" "UpdatingConfig")" + all_healthy="$(nodepool_condition_status "${np}" "AllNodesHealthy")" + + if [[ "${updating_config}" == "True" ]]; then + saw_updating_config=true + fi + + if [[ -n "${current_version}" && "${current_version}" != "${baseline_version}" ]]; then + echo " ✓ NodePool/${np} config version rolled out: '${baseline_version}' -> '${current_version}'" + rollout_complete=true + break + fi + if [[ -n "${baseline_config}" && -n "${current_config}" && "${current_config}" != "${baseline_config}" ]]; then + echo " ✓ NodePool/${np} config hash rolled out: '${baseline_config}' -> '${current_config}'" + rollout_complete=true + break + fi + if [[ "${saw_updating_config}" == "true" && "${updating_config}" == "False" && "${all_healthy}" == "True" ]]; then + echo " ✓ NodePool/${np} finished UpdatingConfig with AllNodesHealthy=True (version='${current_version}')" + rollout_complete=true + break + fi + sleep 15 + done + + if [[ "${rollout_complete}" != "true" ]]; then + local current_version current_config updating_config all_healthy + current_version="$(nodepool_config_version "${np}")" + current_config="$(nodepool_current_config "${np}")" + updating_config="$(nodepool_condition_status "${np}" "UpdatingConfig")" + all_healthy="$(nodepool_condition_status "${np}" "AllNodesHealthy")" + if [[ "${updating_config}" == "False" && "${all_healthy}" == "True" && "${current_version}" == "${baseline_version}" ]]; then + echo " Warning: NodePool/${np} ${NODEPOOL_CONFIG_VERSION_ANNOTATION} remained '${current_version}' but NodePool is healthy; continuing" + else + echo "Error: timed out waiting for NodePool/${np} config rollout (version='${current_version:-}', UpdatingConfig='${updating_config}', AllNodesHealthy='${all_healthy}')" + oc get "nodepool/${np}" -n "${HOSTED_CLUSTER_NAMESPACE}" -o yaml || true + return 1 + fi + fi + + echo " Waiting for NodePool/${np} AllNodesHealthy=True..." + oc wait "nodepool/${np}" -n "${HOSTED_CLUSTER_NAMESPACE}" \ + --for=condition=AllNodesHealthy=True --timeout="${NODEPOOL_ROLLOUT_TIMEOUT}" + done + echo "✓ NodePool config rollout complete" +} + +wait_nodepool_rollout + verify_modern_tls_endpoint() { local api_server api_host api_port @@ -94,31 +267,116 @@ verify_modern_tls_endpoint() { echo "✓ API server endpoint enforces Modern TLS (TLS 1.3 only)" } +wait_guest_machine_config_pools() { + echo "Checking guest cluster MachineConfigPools..." + local mcp_names + if ! mcp_names="$(oc get mcp -o name 2>&1)"; then + if [[ "${mcp_names}" == *"(NotFound)"* ]]; then + echo "MachineConfigPool API is not available on guest cluster; skipping MCP wait" + return 0 + fi + echo "Error: failed to list MachineConfigPools: ${mcp_names}" >&2 + return 1 + fi + + if [[ -z "${mcp_names//[$'\t\r\n ']/}" ]]; then + echo "No MachineConfigPools on guest cluster; skipping MCP wait (node rollout validated via NodePool)" + return 0 + fi + + echo "Waiting for all MachineConfigPools to finish updating on guest cluster..." + local deadline=$((SECONDS + MCP_ROLLOUT_TIMEOUT_SECONDS)) + while (( SECONDS < deadline )); do + if oc wait mcp --all --for=condition=UPDATED=True --timeout=30s && \ + oc wait mcp --all --for=condition=UPDATING=False --timeout=30s && \ + oc wait mcp --all --for=condition=DEGRADED=False --timeout=30s; then + echo "✓ All MachineConfigPools are updated" + return 0 + fi + echo "MachineConfigPools still updating..." + sleep 15 + done + echo "Error: timed out waiting for MachineConfigPools" + oc get mcp -o wide || true + return 1 +} + +wait_guest_tuned_daemonset() { + echo "Waiting for tuned DaemonSet on all guest nodes..." + local tuned_lookup + if ! tuned_lookup="$(oc -n openshift-cluster-node-tuning get ds/tuned -o name 2>&1)"; then + if [[ "${tuned_lookup}" == *"(NotFound)"* ]]; then + echo "Warning: tuned DaemonSet not found in openshift-cluster-node-tuning; skipping tuned readiness check" + return 0 + fi + echo "Error: failed to look up tuned DaemonSet: ${tuned_lookup}" >&2 + return 1 + fi + + oc wait --for=condition=Available ds/tuned -n openshift-cluster-node-tuning --timeout="${TUNED_ROLLOUT_TIMEOUT}" + + local desired ready + desired=$(oc get ds/tuned -n openshift-cluster-node-tuning -o jsonpath='{.status.desiredNumberScheduled}') + ready=$(oc get ds/tuned -n openshift-cluster-node-tuning -o jsonpath='{.status.numberReady}') + if [[ -z "${desired}" || "${desired}" -eq 0 ]]; then + echo "Warning: tuned DaemonSet has no scheduled pods" + return 0 + fi + if [[ "${ready}" -lt "${desired}" ]]; then + echo "Error: tuned DaemonSet has ${ready}/${desired} ready pods" + oc get pods -n openshift-cluster-node-tuning -o wide || true + return 1 + fi + echo "✓ tuned DaemonSet ready on ${ready}/${desired} nodes" +} + export KUBECONFIG=${SHARED_DIR}/nested_kubeconfig echo "Waiting for guest cluster APIServer to reflect Modern TLS profile..." +guest_tls_verified=false for i in {1..40}; do tls_profile=$(oc get apiserver/cluster -o jsonpath='{.spec.tlsSecurityProfile.type}' 2>/dev/null || echo "") if [[ "$tls_profile" == "Modern" ]]; then echo "✓ Guest cluster APIServer tlsSecurityProfile.type is Modern" - echo "✓ Modern TLS Security Profile successfully applied" - exit 0 + guest_tls_verified=true + break fi if verify_modern_tls_endpoint; then echo "Guest cluster APIServer tlsSecurityProfile.type is '${tls_profile}' (HyperShift may not mirror this field)" - echo "✓ Modern TLS Security Profile successfully applied" - exit 0 + guest_tls_verified=true + break fi echo "Waiting for Modern TLS profile to propagate (attempt $i/40)..." sleep 15 done -tls_profile=$(oc get apiserver/cluster -o jsonpath='{.spec.tlsSecurityProfile.type}' 2>/dev/null || echo "NotFound") -if [[ "$tls_profile" == "Modern" ]]; then - echo "✓ Modern TLS Security Profile successfully applied" - exit 0 +if [[ "${guest_tls_verified}" != "true" ]]; then + tls_profile=$(oc get apiserver/cluster -o jsonpath='{.spec.tlsSecurityProfile.type}' 2>/dev/null || echo "NotFound") + echo "Guest cluster APIServer tlsSecurityProfile.type is '${tls_profile}'" + verify_modern_tls_endpoint +fi + +if oc adm wait-for-stable-cluster --help >/dev/null 2>&1; then + echo "Waiting for guest cluster to become stable..." + oc adm wait-for-stable-cluster --minimum-stable-period=2m --timeout="${GUEST_STABLE_CLUSTER_TIMEOUT}" +else + echo "Warning: oc adm wait-for-stable-cluster is not available; skipping cluster stability wait" +fi + +wait_guest_machine_config_pools +wait_guest_tuned_daemonset + +if [[ "${TLS_13_ENABLE_TLS_ADHERENCE:-}" == "true" ]]; then + guest_tls_adherence=$(oc get apiserver/cluster -o jsonpath='{.spec.tlsAdherence}' 2>/dev/null || echo "") + if [[ -n "${guest_tls_adherence}" && "${guest_tls_adherence}" != "${TLS_13_TLS_ADHERENCE_POLICY}" ]]; then + echo "Error: guest APIServer tlsAdherence is '${guest_tls_adherence}', expected '${TLS_13_TLS_ADHERENCE_POLICY}'" + exit 1 + fi + if [[ "${guest_tls_adherence}" == "${TLS_13_TLS_ADHERENCE_POLICY}" ]]; then + echo "✓ Guest cluster APIServer tlsAdherence is ${guest_tls_adherence}" + else + echo "Guest cluster APIServer tlsAdherence is not mirrored yet (HyperShift may only enforce via node rollout)" + fi fi -echo "Guest cluster APIServer tlsSecurityProfile.type is '${tls_profile}'" -verify_modern_tls_endpoint -echo "✓ Modern TLS Security Profile successfully applied" +echo "✓ Modern TLS Security Profile successfully applied and node rollout complete" diff --git a/ci-operator/step-registry/hypershift/modern-tls/hypershift-modern-tls-ref.yaml b/ci-operator/step-registry/hypershift/modern-tls/hypershift-modern-tls-ref.yaml index 919a4eefcac4c..5ddb02daf0dd6 100644 --- a/ci-operator/step-registry/hypershift/modern-tls/hypershift-modern-tls-ref.yaml +++ b/ci-operator/step-registry/hypershift/modern-tls/hypershift-modern-tls-ref.yaml @@ -5,6 +5,40 @@ ref: name: "5.0" tag: cli commands: hypershift-modern-tls-commands.sh + env: + - name: TLS_13_ENABLE_TLS_ADHERENCE + default: "false" + documentation: |- + When true, patch the HostedCluster to set spec.configuration.apiServer.tlsAdherence + using TLS_13_TLS_ADHERENCE_POLICY. Requires TechPreviewNoUpgrade on the guest cluster. + - name: TLS_13_TLS_ADHERENCE_POLICY + default: StrictAllComponents + documentation: |- + Value for HostedCluster spec.configuration.apiServer.tlsAdherence when + TLS_13_ENABLE_TLS_ADHERENCE is true. + - name: NODEPOOL_ROLLOUT_TIMEOUT + default: 60m + documentation: |- + Timeout passed to oc wait while waiting for NodePool AllNodesHealthy after the + config version annotation changes. + - name: NODEPOOL_ROLLOUT_TIMEOUT_SECONDS + default: "3600" + documentation: |- + Maximum time in seconds to wait for NodePool + hypershift.openshift.io/nodePoolCurrentConfigVersion to change after the + HostedCluster TLS configuration is updated. + - name: GUEST_STABLE_CLUSTER_TIMEOUT + default: 3h + documentation: |- + Timeout for oc adm wait-for-stable-cluster on the guest cluster after TLS changes. + - name: MCP_ROLLOUT_TIMEOUT_SECONDS + default: "3600" + documentation: |- + Maximum time in seconds to wait for guest MachineConfigPools to reach UPDATED=True. + - name: TUNED_ROLLOUT_TIMEOUT + default: 30m + documentation: |- + Timeout for the guest tuned DaemonSet to become Available on all nodes. resources: requests: cpu: 100m @@ -13,10 +47,14 @@ ref: Configures Modern TLS Security Profile on a HyperShift hosted cluster. This step patches the HostedCluster CR to enable Modern TLS profile, which - enforces TLS 1.3 for all API server connections. It waits for the hosted - control plane kube-apiserver to roll out on the management cluster, then - verifies the profile on the guest APIServer object or via a TLS 1.3 endpoint - probe when HyperShift does not mirror the field into the guest cluster. + enforces TLS 1.3 for all API server connections. When TLS_13_ENABLE_TLS_ADHERENCE + is true, it also sets spec.configuration.apiServer.tlsAdherence (for example + StrictAllComponents). It waits for the hosted control plane kube-apiserver to + roll out on the management cluster, records each NodePool's + hypershift.openshift.io/nodePoolCurrentConfigVersion annotation before the TLS + change, then waits for that annotation to change (HyperShift only updates it after + the config has rolled out to the NodePool) and for AllNodesHealthy=True before + verifying guest cluster TLS settings and node-level rollout. Note: Due to current HyperShift limitations, the Modern TLS profile may not propagate to etcd and some other control plane components. This is a known diff --git a/ci-operator/step-registry/tls/scanner/hypershift-aws/tls-scanner-hypershift-aws-workflow.yaml b/ci-operator/step-registry/tls/scanner/hypershift-aws/tls-scanner-hypershift-aws-workflow.yaml index a74fd5b2fdcd2..ad5e4624f9c5c 100644 --- a/ci-operator/step-registry/tls/scanner/hypershift-aws/tls-scanner-hypershift-aws-workflow.yaml +++ b/ci-operator/step-registry/tls/scanner/hypershift-aws/tls-scanner-hypershift-aws-workflow.yaml @@ -14,6 +14,11 @@ workflow: The workflow configures Modern TLS (TLS 1.3) on the hosted cluster and scans TLS endpoints in both the management control plane and the guest cluster. + When TLS_13_ENABLE_TLS_ADHERENCE is true, the workflow also sets + spec.configuration.apiServer.tlsAdherence on the HostedCluster (for example + StrictAllComponents). TechPreviewNoUpgrade must be enabled on the HyperShift + operator (hypershift install --tech-preview-no-upgrade) and guest cluster. + The management cluster is hosted in the `osd-hypershift` AWS account and destroyed after the scan completes. steps: @@ -21,6 +26,7 @@ workflow: pre: - ref: ipi-install-rbac - chain: hypershift-setup-root-management-cluster + - ref: hypershift-install - chain: hypershift-aws-create - ref: hypershift-modern-tls test: