diff --git a/vgpu-manager/rhel10/Dockerfile b/vgpu-manager/rhel10/Dockerfile new file mode 100644 index 000000000..3fb9824e5 --- /dev/null +++ b/vgpu-manager/rhel10/Dockerfile @@ -0,0 +1,59 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +FROM registry.access.redhat.com/ubi10/ubi:10.2-1784668814 + +ARG DRIVER_VERSION +ENV DRIVER_VERSION=$DRIVER_VERSION +ARG DRIVER_ARCH=x86_64 +ENV DRIVER_ARCH=$DRIVER_ARCH +ARG GIT_COMMIT="" + +# Optional: inject custom CA certificates so package managers (dnf/yum) can +# verify TLS when building behind a corporate MITM proxy. Drop one or more +# *.crt files into the build context's "certs/" directory (the default), or +# override CUSTOM_CA_CERTS_DIR with a path to a different directory inside +# the build context. When the directory is empty, this is a no-op. +ARG CUSTOM_CA_CERTS_DIR=certs +COPY ${CUSTOM_CA_CERTS_DIR}/ /etc/pki/ca-trust/source/anchors/ +RUN update-ca-trust + +RUN mkdir -p /driver +WORKDIR /driver +COPY NVIDIA-Linux-${DRIVER_ARCH}-${DRIVER_VERSION}-vgpu-kvm.run . +RUN chmod +x NVIDIA-Linux-${DRIVER_ARCH}-${DRIVER_VERSION}-vgpu-kvm.run + +COPY nvidia-driver /usr/local/bin +COPY ocp_dtk_entrypoint /usr/local/bin + +RUN dnf install -y pciutils && \ + dnf clean all && \ + rm -rf /var/cache/dnf/* + +LABEL io.k8s.display-name="NVIDIA vGPU Manager Container" +LABEL name="NVIDIA vGPU Manager Container" +LABEL vendor="NVIDIA" +LABEL version="${DRIVER_VERSION}" +LABEL vcs-ref="${GIT_COMMIT}" +LABEL release="N/A" +LABEL summary="Provision the NVIDIA vGPU Manager through containers" +LABEL description="See summary" + +# Install / upgrade packages here that are required to resolve CVEs +ARG CVE_UPDATES +RUN if [ -n "${CVE_UPDATES}" ]; then \ + dnf update -y ${CVE_UPDATES} && \ + dnf clean all; \ + fi + +ENTRYPOINT ["nvidia-driver", "init"] diff --git a/vgpu-manager/rhel10/certs/.gitkeep b/vgpu-manager/rhel10/certs/.gitkeep new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/vgpu-manager/rhel10/certs/.gitkeep @@ -0,0 +1 @@ + diff --git a/vgpu-manager/rhel10/nvidia-driver b/vgpu-manager/rhel10/nvidia-driver new file mode 100755 index 000000000..f3d5d505c --- /dev/null +++ b/vgpu-manager/rhel10/nvidia-driver @@ -0,0 +1,335 @@ +#!/bin/bash +# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -xe + +DRIVER_VERSION=${DRIVER_VERSION:?"Missing driver version"} +DRIVER_RESET_RETRIES=10 +DELAY_BEFORE_VF_CREATION=${DELAY_BEFORE_VF_CREATION:-15} +RUN_DIR=/run/nvidia +NVIDIA_MODULE_PARAMS=() +MODPROBE_CONFIG_DIR="/etc/modprobe.d" + +# Requires the pod to run with hostPID: true. +_host_selinux_enabled() { + [ -r /proc/1/mounts ] && + grep -qsw "selinuxfs" /proc/1/mounts && + [ -f /proc/1/root/sys/fs/selinux/enforce ] +} + +# Mount the driver rootfs into the run directory with the exception of sysfs. +_mount_rootfs() { + echo "Mounting NVIDIA driver rootfs..." + mount --make-runbindable /sys + mount --make-private /sys + mkdir -p ${RUN_DIR}/driver + mount --rbind / ${RUN_DIR}/driver + + echo "Check host SELinux status" + if _host_selinux_enabled; then + echo "Host SELinux is enabled" + echo "Change device files security context for selinux compatibility" + chcon -R -t container_file_t ${RUN_DIR}/driver/dev + else + echo "Host SELinux is disabled, skipping..." + fi +} + +# Unmount the driver rootfs from the run directory. +_unmount_rootfs() { + echo "Unmounting NVIDIA driver rootfs..." + if findmnt -r -o TARGET | grep "${RUN_DIR}/driver" > /dev/null; then + umount -l -R ${RUN_DIR}/driver + fi +} + +# Create /dev/char directory if it doesn't exist inside the container. +# Without this directory, nvidia-vgpu-mgr will fail to create symlinks +# under /dev/char for new devices nodes. +_create_dev_char_directory() { + if [ ! -d "/dev/char" ]; then + echo "Creating '/dev/char' directory" + mkdir -p /dev/char + fi +} + +_set_fw_search_path() { + local nv_fw_search_path="$RUN_DIR/driver/lib/firmware" + local fw_path_config_file="/sys/module/firmware_class/parameters/path" + + if [[ ! -z $(grep '[^[:space:]]' $fw_path_config_file) ]]; then + echo "WARNING: A search path is already configured in $fw_path_config_file" + echo " Retaining the current configuration. Note, GSP firmware may not be found and thus won't be used by the NVIDIA driver." + return + fi + + echo "Configuring the following firmware search path in '$fw_path_config_file': $nv_fw_search_path" + echo -n "$nv_fw_search_path" > $fw_path_config_file +} + +# For each kernel module configuration file mounted into the container, +# parse the file contents and extract the custom module parameters that +# are to be passed as input to 'modprobe'. +# +# Assumptions: +# - Configuration file is named nvidia.conf +# - Configuration file is mounted inside the container at /drivers. +# - Each line in the file contains at least one parameter, where parameters on the same line +# are space delimited. It is up to the user to properly format the file to ensure +# the correct set of parameters are passed to 'modprobe'. +_get_module_params() { + local base_path="/drivers" + # nvidia + if [ -f "${base_path}/nvidia.conf" ]; then + while IFS="" read -r param || [ -n "$param" ]; do + NVIDIA_MODULE_PARAMS+=("$param") + done <"${base_path}/nvidia.conf" + echo "Module parameters provided for nvidia: ${NVIDIA_MODULE_PARAMS[@]}" + fi +} + +_install_driver() { + local tmp_dir=$(mktemp -d) + local install_args=() + + # Specify the --skip-module-load flag for versions of the nvidia-installer that + # support it. From the nvidia-installer help output: + # + # --skip-module-load + # Skip the test load of the NVIDIA kernel modules after the modules are built, + # and skip loading them after installation is complete. + # + # Without this flag, a subtle bug can occur if the nvidia-installer fails to unload + # the NVIDIA kernel modules after the test load. The modules will remain loaded and + # any custom NVIDIA module parameters configured as input to the driver container + # will not be applied. + # + DRIVER_BRANCH=$(echo ${DRIVER_VERSION} | cut -d. -f1) + if [ "${DRIVER_BRANCH}" -ge "550" ]; then + install_args+=("--skip-module-load") + fi + sh NVIDIA-Linux-${DRIVER_ARCH}-${DRIVER_VERSION}-vgpu-kvm.run --ui=none --no-questions --tmpdir ${tmp_dir} --no-systemd ${install_args[@]+"${install_args[@]}"} +} + +_create_module_params_conf() { + echo "Parsing kernel module parameters..." + _get_module_params + + if [ ${#NVIDIA_MODULE_PARAMS[@]} -gt 0 ]; then + echo "Configuring nvidia module parameters in ${MODPROBE_CONFIG_DIR}/nvidia.conf" + echo "options nvidia ${NVIDIA_MODULE_PARAMS[@]}" > ${MODPROBE_CONFIG_DIR}/nvidia.conf + fi +} + +# Load NVIDIA driver kernel modules with custom parameters and start vGPU daemons +_load_driver() { + echo "Loading NVIDIA driver kernel modules..." + set -o xtrace +o nounset + modprobe nvidia + modprobe nvidia_vgpu_vfio + set +o xtrace -o nounset + + # Start vGPU daemons + /usr/bin/nvidia-vgpud + /usr/bin/nvidia-vgpu-mgr & + + # check nvidia drivers are loaded + if [ ! -f /sys/module/nvidia_vgpu_vfio/refcnt ] || [ ! -f /sys/module/nvidia/refcnt ]; then + echo "Failed to load nvidia driver" + return 1 + fi + return 0 +} + +# Enable virtual functions for all physical GPUs on the node that support SR-IOV. +# Retry logic is to account for when the driver is busy (i.e. during driver initialization) +_enable_vfs() { + # Wait before attempting to create VFs to ensure the driver has finished initializing. + # This is a WAR for a bug in vGPU 17.2 where sriov-manage does not return a non-zero + # exit code even though VF creation fails. + sleep $DELAY_BEFORE_VF_CREATION + + local retry + for ((retry = 0 ; retry <= $DRIVER_RESET_RETRIES ; retry++)); do + if /usr/lib/nvidia/sriov-manage -e ALL; then + return 0 + fi + if [ $retry == $DRIVER_RESET_RETRIES ]; then + echo "Failed to enable VFs" + fi + done + return 1 +} + +# Disable virtual functions for all physical GPUs on the node that support SR-IOV. +# Retry logic is to account for when the driver is busy (i.e. during driver initialization) +_disable_vfs() { + local retry + for ((retry = 0 ; retry <= $DRIVER_RESET_RETRIES ; retry++)); do + if /usr/lib/nvidia/sriov-manage -d ALL; then + return 0 + fi + if [ $retry == $DRIVER_RESET_RETRIES ]; then + echo "Failed to disable VFs" + fi + done + return 1 +} + +_unload_driver() { + local rmmod_args=() + local nvidia_deps=0 + local nvidia_refs=0 + local nvidia_vgpu_vfio_refs=0 + + if [ -f /var/run/nvidia-vgpu-mgr/nvidia-vgpu-mgr.pid ]; then + echo "Stopping NVIDIA vGPU Manager..." + local pid=$(< /var/run/nvidia-vgpu-mgr/nvidia-vgpu-mgr.pid) + + kill -TERM "${pid}" + for i in $(seq 1 50); do + kill -0 "${pid}" 2> /dev/null || break + sleep 0.1 + done + if [ $i -eq 50 ]; then + echo "Could not stop NVIDIA vGPU Manager" >&2 + return 1 + fi + fi + + echo "Unloading NVIDIA driver kernel modules..." + if [ -f /sys/module/nvidia_vgpu_vfio/refcnt ]; then + nvidia_vgpu_vfio_refs=$(< /sys/module/nvidia_vgpu_vfio/refcnt) + rmmod_args+=("nvidia_vgpu_vfio") + ((++nvidia_deps)) + fi + if [ -f /sys/module/nvidia/refcnt ]; then + nvidia_refs=$(< /sys/module/nvidia/refcnt) + rmmod_args+=("nvidia") + fi + + # TODO: check if nvidia module is in use by checking refcnt + + if [ ${#rmmod_args[@]} -gt 0 ]; then + rmmod ${rmmod_args[@]} + if [ "$?" != "0" ]; then + return 1 + fi + fi + return 0 +} + +_shutdown() { + if _disable_vfs && _unload_driver; then + _unmount_rootfs + return 0 + fi + echo "Failed to cleanup driver" + return 1 +} + +build() { + echo "build() not implemented" +} + +load() { + echo "load() not implemented" +} + +update() { + echo "update() not implemented" +} + +init() { + trap "echo 'Caught signal'; exit 1" HUP INT QUIT PIPE TERM + trap "_shutdown" EXIT + + if ! _unload_driver; then + echo "Previous NVIDIA driver installation cannot be removed. Exiting" + exit 1 + fi + _unmount_rootfs + _create_dev_char_directory + _set_fw_search_path + _create_module_params_conf + _install_driver + _load_driver || exit 1 + _mount_rootfs + _enable_vfs + + # In certain scenarios, /sys/class/mdev_bus is not populated with the correct list of devices (PFs and possible VFs) at this point. + # Re-run nvdidia-vgpud to ensure /sys/class/mdev_bus is populated correctly. And restart nvidia-vgpu-mgr if previously killed. + nvidia-vgpud & + pgrep nvidia-vgpu-mgr >/dev/null || (echo "Restarting nvidia-vgpu-mgr after previously killed" && nvidia-vgpu-mgr &) + + set +x + echo "Done, now waiting for signal" + trap "echo 'Caught signal'; _shutdown; trap - EXIT; exit" HUP INT QUIT PIPE TERM + + while true; do + sleep 15 + pgrep nvidia-vgpu-mgr >/dev/null || (echo "ERROR: nvidia-vgpu-mgr daemon is no longer running. Exiting." && exit 1) + done +} + + +usage() { + cat >&2 < "$DRIVER_TOOLKIT_SHARED_DIR/env" + + touch "$DRIVER_TOOLKIT_SHARED_DIR/dir_prepared" + fi + + set +x + while [[ ! -f "$DRIVER_TOOLKIT_SHARED_DIR/driver_build_started" ]]; do + if [[ -f "$DRIVER_TOOLKIT_SHARED_DIR/driver_toolkit_broken" ]]; then + echo "WARNING: broken driver toolkit detected" + exit 1 + # TODO: use entitlement based fallback + #exec bash -x nvidia-driver init + fi + echo "$(date) Waiting for openshift-driver-toolkit-ctr container to start ..." + sleep 15 + done + + echo "$(date) openshift-driver-toolkit-ctr started." + + # TODO: Currently dtk-build-driver will actually install and load the driver as well. + # Uncomment the following if/when dtk-build-driver only builds precompiled driver. + #while [[ ! -f "$DRIVER_TOOLKIT_SHARED_DIR/driver_built" ]]; do + # echo "$(date) Waiting for openshift-driver-toolkit-ctr container to build the precompiled driver ..." + # sleep 15 + #done + + #echo "$(date) openshift-driver-toolkit-ctr finished building driver." + set -x + sleep infinity +} + +dtk-build-driver() { + if [[ "${RHCOS_IMAGE_MISSING:-}" == "true" ]]; then + echo "WARNING: 'istag/driver-toolkit:${RHCOS_VERSION} -n openshift' missing, nothing to do in openshift-driver-toolkit-ctr container" + sleep +inf + fi + + if ! [[ -f "/lib/modules/$(uname -r)/vmlinuz" ]]; then + echo "WARNING: broken Driver Toolkit image detected:" + echo "- Node kernel: $(uname -r)" + echo "- Kernel package: $(rpm -q --qf "%{VERSION}-%{RELEASE}.%{ARCH}" kernel-core)" + + # TODO: log entitlement based fallback + #echo "INFO: informing nvidia-driver-ctr to fallback on entitled-build." + touch "$DRIVER_TOOLKIT_SHARED_DIR/driver_toolkit_broken" + echo "INFO: nothing else to do in openshift-driver-toolkit-ctr container, sleeping forever." + sleep +inf + fi + + # Shared directory is prepared before entering this script. See + # 'until [ -f /mnt/shared-nvidia-driver-toolkit/dir_prepared ] ...' + # in the Pod command/args + + touch "$DRIVER_TOOLKIT_SHARED_DIR/driver_build_started" + + set -x + set -o allexport + source "${DRIVER_TOOLKIT_SHARED_DIR}/env" + set +o allexport; + + # if this directory already exists, + # NVIDIA-Linux-$DRIVER_ARCH-$DRIVER_VERSION.run fails to run + # and doesn't create its files. This may happen when the + # container fails and restart its execution, leading to + # hard-to-understand "unrelated" errors in the following of the script execution + + rm -rf "${DRIVER_TOOLKIT_SHARED_DIR}/driver/NVIDIA-Linux-${DRIVER_ARCH}-${DRIVER_VERSION}"; + + mkdir "${DRIVER_TOOLKIT_SHARED_DIR}/bin" -p + + cp -v \ + "$DRIVER_TOOLKIT_SHARED_DIR/nvidia-driver" \ + "$DRIVER_TOOLKIT_SHARED_DIR/lspci" \ + "$DRIVER_TOOLKIT_SHARED_DIR/setpci" \ + "${DRIVER_TOOLKIT_SHARED_DIR}/bin" + + export PATH="${DRIVER_TOOLKIT_SHARED_DIR}/bin:$PATH"; + + mkdir "${DRIVER_TOOLKIT_SHARED_DIR}/lib" -p + cp -v \ + "$DRIVER_TOOLKIT_SHARED_DIR"/libpci.so.* \ + "${DRIVER_TOOLKIT_SHARED_DIR}/lib" + export LD_LIBRARY_PATH="${DRIVER_TOOLKIT_SHARED_DIR}/lib:$LD_LIBRARY_PATH"; + + # ensure lspci is installed, as 'sriov-manage' script requires it + if ! $(lspci >/dev/null); then + dnf install -y pciutils && rm -rf /var/cache/yum/* + fi + + # upon catching a signal, terminate child process to trigger driver cleanup + trap 'echo "Caught signal"; kill "${child_pid}"; wait "${child_pid}"; exit' HUP INT QUIT PIPE TERM + cd "${DRIVER_TOOLKIT_SHARED_DIR}/driver"; + echo "#" + echo "# Executing nvidia-driver install script ..." + echo "#" + bash -x "${DRIVER_TOOLKIT_SHARED_DIR}/nvidia-driver" init & + + child_pid="$!" + wait "${child_pid}" + + # TODO: only build driver in the dtk, and let main container load. + # 'nvidia-driver init' will only exit if it fails + echo "Driver installation failed. Exiting ..." + exit 1 +} + +usage() { + cat >&2 <